feat(dns): add dnscrypt and dns over tcp
This commit is contained in:
369
scripts/tools/add_extra_metrics_to_csv.go
Normal file
369
scripts/tools/add_extra_metrics_to_csv.go
Normal file
@@ -0,0 +1,369 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/layers"
|
||||
"github.com/google/gopacket/pcapgo"
|
||||
)
|
||||
|
||||
type QueryRecord struct {
|
||||
Domain string
|
||||
QueryType string
|
||||
Protocol string
|
||||
DNSSec string
|
||||
AuthDNSSec string
|
||||
KeepAlive string
|
||||
DNSServer string
|
||||
Timestamp string
|
||||
DurationNs int64
|
||||
DurationMs float64
|
||||
RequestSizeBytes int
|
||||
ResponseSizeBytes int
|
||||
ResponseCode string
|
||||
Error string
|
||||
BytesSent int64
|
||||
BytesReceived int64
|
||||
PacketsSent int64
|
||||
PacketsReceived int64
|
||||
TotalBytes int64
|
||||
}
|
||||
|
||||
func parseRFC3339Nano(ts string) (time.Time, error) {
|
||||
return time.Parse(time.RFC3339Nano, ts)
|
||||
}
|
||||
|
||||
func processProviderFolder(providerPath string) error {
|
||||
providerName := filepath.Base(providerPath)
|
||||
fmt.Printf("\n=== Processing provider: %s ===\n", providerName)
|
||||
|
||||
files, err := os.ReadDir(providerPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
processed := 0
|
||||
skipped := 0
|
||||
errors := 0
|
||||
|
||||
for _, file := range files {
|
||||
if !strings.HasSuffix(file.Name(), ".csv") {
|
||||
continue
|
||||
}
|
||||
|
||||
csvPath := filepath.Join(providerPath, file.Name())
|
||||
pcapPath := strings.Replace(csvPath, ".csv", ".pcap", 1)
|
||||
|
||||
// Check if PCAP exists
|
||||
if _, err := os.Stat(pcapPath); os.IsNotExist(err) {
|
||||
fmt.Printf(" ⊗ Skipping: %s (no matching PCAP)\n", file.Name())
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if already processed (has backup)
|
||||
backupPath := csvPath + ".bak"
|
||||
if _, err := os.Stat(backupPath); err == nil {
|
||||
fmt.Printf(" ⊙ Skipping: %s (already processed, backup exists)\n", file.Name())
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Printf(" ↻ Processing: %s ... ", file.Name())
|
||||
if err := processPair(csvPath, pcapPath); err != nil {
|
||||
fmt.Printf("ERROR\n")
|
||||
log.Printf(" Error: %v\n", err)
|
||||
errors++
|
||||
} else {
|
||||
fmt.Printf("✓\n")
|
||||
processed++
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf(" Summary: %d processed, %d skipped, %d errors\n", processed, skipped, errors)
|
||||
return nil
|
||||
}
|
||||
|
||||
func processPair(csvPath, pcapPath string) error {
|
||||
// Create backup
|
||||
backupPath := csvPath + ".bak"
|
||||
input, err := os.ReadFile(csvPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup read failed: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(backupPath, input, 0644); err != nil {
|
||||
return fmt.Errorf("backup write failed: %w", err)
|
||||
}
|
||||
|
||||
// Read CSV records
|
||||
records, err := readCSV(csvPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("CSV read failed: %w", err)
|
||||
}
|
||||
|
||||
if len(records) == 0 {
|
||||
return fmt.Errorf("no records in CSV")
|
||||
}
|
||||
|
||||
// Read and parse PCAP
|
||||
packets, err := readPCAPGo(pcapPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("PCAP read failed: %w", err)
|
||||
}
|
||||
|
||||
// Enrich records with bandwidth data
|
||||
enrichRecords(records, packets)
|
||||
|
||||
// Write enriched CSV
|
||||
if err := writeCSV(csvPath, records); err != nil {
|
||||
return fmt.Errorf("CSV write failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func readCSV(path string) ([]*QueryRecord, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
r := csv.NewReader(f)
|
||||
rows, err := r.ReadAll()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(rows) < 2 {
|
||||
return nil, fmt.Errorf("CSV has no data rows")
|
||||
}
|
||||
|
||||
records := make([]*QueryRecord, 0, len(rows)-1)
|
||||
for i := 1; i < len(rows); i++ {
|
||||
row := rows[i]
|
||||
if len(row) < 14 {
|
||||
log.Printf(" Warning: Skipping malformed row %d", i+1)
|
||||
continue
|
||||
}
|
||||
|
||||
durationNs, _ := strconv.ParseInt(row[8], 10, 64)
|
||||
durationMs, _ := strconv.ParseFloat(row[9], 64)
|
||||
reqSize, _ := strconv.Atoi(row[10])
|
||||
respSize, _ := strconv.Atoi(row[11])
|
||||
|
||||
records = append(records, &QueryRecord{
|
||||
Domain: row[0],
|
||||
QueryType: row[1],
|
||||
Protocol: row[2],
|
||||
DNSSec: row[3],
|
||||
AuthDNSSec: row[4],
|
||||
KeepAlive: row[5],
|
||||
DNSServer: row[6],
|
||||
Timestamp: row[7],
|
||||
DurationNs: durationNs,
|
||||
DurationMs: durationMs,
|
||||
RequestSizeBytes: reqSize,
|
||||
ResponseSizeBytes: respSize,
|
||||
ResponseCode: row[12],
|
||||
Error: row[13],
|
||||
})
|
||||
}
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
type PacketInfo struct {
|
||||
Timestamp time.Time
|
||||
Size int
|
||||
IsSent bool
|
||||
}
|
||||
|
||||
func readPCAPGo(path string) ([]PacketInfo, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
reader, err := pcapgo.NewReader(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var packets []PacketInfo
|
||||
packetSource := gopacket.NewPacketSource(reader, reader.LinkType())
|
||||
|
||||
for packet := range packetSource.Packets() {
|
||||
if packet.NetworkLayer() == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
isDNS := false
|
||||
isSent := false
|
||||
|
||||
// Check UDP layer (DNS, DoQ, DoH3)
|
||||
if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer != nil {
|
||||
udp := udpLayer.(*layers.UDP)
|
||||
isDNS = udp.SrcPort == 53 || udp.DstPort == 53 ||
|
||||
udp.SrcPort == 853 || udp.DstPort == 853 ||
|
||||
udp.SrcPort == 443 || udp.DstPort == 443
|
||||
isSent = udp.DstPort == 53 || udp.DstPort == 853 || udp.DstPort == 443
|
||||
}
|
||||
|
||||
// Check TCP layer (DoT, DoH)
|
||||
if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
|
||||
tcp := tcpLayer.(*layers.TCP)
|
||||
isDNS = tcp.SrcPort == 53 || tcp.DstPort == 53 ||
|
||||
tcp.SrcPort == 853 || tcp.DstPort == 853 ||
|
||||
tcp.SrcPort == 443 || tcp.DstPort == 443
|
||||
isSent = tcp.DstPort == 53 || tcp.DstPort == 853 || tcp.DstPort == 443
|
||||
}
|
||||
|
||||
if isDNS {
|
||||
packets = append(packets, PacketInfo{
|
||||
Timestamp: packet.Metadata().Timestamp,
|
||||
Size: len(packet.Data()),
|
||||
IsSent: isSent,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return packets, nil
|
||||
}
|
||||
|
||||
func enrichRecords(records []*QueryRecord, packets []PacketInfo) {
|
||||
for _, rec := range records {
|
||||
ts, err := parseRFC3339Nano(rec.Timestamp)
|
||||
if err != nil {
|
||||
log.Printf(" Warning: Failed to parse timestamp: %s", rec.Timestamp)
|
||||
continue
|
||||
}
|
||||
|
||||
// Define time window for this query
|
||||
windowStart := ts
|
||||
windowEnd := ts.Add(time.Duration(rec.DurationNs))
|
||||
|
||||
var sent, recv, pktSent, pktRecv int64
|
||||
|
||||
// Match packets within the time window
|
||||
for _, pkt := range packets {
|
||||
if (pkt.Timestamp.Equal(windowStart) || pkt.Timestamp.After(windowStart)) &&
|
||||
pkt.Timestamp.Before(windowEnd) {
|
||||
if pkt.IsSent {
|
||||
sent += int64(pkt.Size)
|
||||
pktSent++
|
||||
} else {
|
||||
recv += int64(pkt.Size)
|
||||
pktRecv++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rec.BytesSent = sent
|
||||
rec.BytesReceived = recv
|
||||
rec.PacketsSent = pktSent
|
||||
rec.PacketsReceived = pktRecv
|
||||
rec.TotalBytes = sent + recv
|
||||
}
|
||||
}
|
||||
|
||||
func writeCSV(path string, records []*QueryRecord) error {
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
w := csv.NewWriter(f)
|
||||
defer w.Flush()
|
||||
|
||||
// Write header
|
||||
header := []string{
|
||||
"domain", "query_type", "protocol", "dnssec", "auth_dnssec",
|
||||
"keep_alive", "dns_server", "timestamp", "duration_ns", "duration_ms",
|
||||
"request_size_bytes", "response_size_bytes", "response_code", "error",
|
||||
"bytes_sent", "bytes_received", "packets_sent", "packets_received", "total_bytes",
|
||||
}
|
||||
if err := w.Write(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write data rows
|
||||
for _, rec := range records {
|
||||
row := []string{
|
||||
rec.Domain,
|
||||
rec.QueryType,
|
||||
rec.Protocol,
|
||||
rec.DNSSec,
|
||||
rec.AuthDNSSec,
|
||||
rec.KeepAlive,
|
||||
rec.DNSServer,
|
||||
rec.Timestamp,
|
||||
strconv.FormatInt(rec.DurationNs, 10),
|
||||
strconv.FormatFloat(rec.DurationMs, 'f', -1, 64),
|
||||
strconv.Itoa(rec.RequestSizeBytes),
|
||||
strconv.Itoa(rec.ResponseSizeBytes),
|
||||
rec.ResponseCode,
|
||||
rec.Error,
|
||||
strconv.FormatInt(rec.BytesSent, 10),
|
||||
strconv.FormatInt(rec.BytesReceived, 10),
|
||||
strconv.FormatInt(rec.PacketsSent, 10),
|
||||
strconv.FormatInt(rec.PacketsReceived, 10),
|
||||
strconv.FormatInt(rec.TotalBytes, 10),
|
||||
}
|
||||
if err := w.Write(row); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
resultsDir := "results"
|
||||
providers := []string{"adguard", "cloudflare", "google", "quad9"}
|
||||
|
||||
fmt.Println("╔═══════════════════════════════════════════════╗")
|
||||
fmt.Println("║ DNS PCAP Preprocessor v1.0 ║")
|
||||
fmt.Println("║ Enriching ALL CSVs with bandwidth metrics ║")
|
||||
fmt.Println("╚═══════════════════════════════════════════════╝")
|
||||
|
||||
totalProcessed := 0
|
||||
totalSkipped := 0
|
||||
totalErrors := 0
|
||||
|
||||
for _, provider := range providers {
|
||||
providerPath := filepath.Join(resultsDir, provider)
|
||||
if _, err := os.Stat(providerPath); os.IsNotExist(err) {
|
||||
fmt.Printf("\n⚠ Provider folder not found: %s\n", provider)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := processProviderFolder(providerPath); err != nil {
|
||||
log.Printf("Error processing %s: %v\n", provider, err)
|
||||
totalErrors++
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("\n╔═══════════════════════════════════════════════╗")
|
||||
fmt.Println("║ Preprocessing Complete! ║")
|
||||
fmt.Println("╚═══════════════════════════════════════════════╝")
|
||||
fmt.Printf("\nAll CSV files now have 5 additional columns:\n")
|
||||
fmt.Printf(" • bytes_sent - Total bytes sent to DNS server\n")
|
||||
fmt.Printf(" • bytes_received - Total bytes received from DNS server\n")
|
||||
fmt.Printf(" • packets_sent - Number of packets sent\n")
|
||||
fmt.Printf(" • packets_received - Number of packets received\n")
|
||||
fmt.Printf(" • total_bytes - Sum of sent + received bytes\n")
|
||||
fmt.Printf("\n📁 Backups saved as: *.csv.bak\n")
|
||||
fmt.Printf("\n💡 Tip: The analysis script will filter which files to visualize,\n")
|
||||
fmt.Printf(" but all files now have complete bandwidth metrics!\n")
|
||||
}
|
||||
Reference in New Issue
Block a user