summaryrefslogtreecommitdiff
path: root/internal/http
diff options
context:
space:
mode:
Diffstat (limited to 'internal/http')
-rw-r--r--internal/http/compression.go142
-rw-r--r--internal/http/compression_test.go152
-rw-r--r--internal/http/headers.go54
-rw-r--r--internal/http/headers_test.go117
-rw-r--r--internal/http/rawrequest.go327
-rw-r--r--internal/http/rawrequest_test.go254
6 files changed, 1046 insertions, 0 deletions
diff --git a/internal/http/compression.go b/internal/http/compression.go
new file mode 100644
index 0000000..aba8ef1
--- /dev/null
+++ b/internal/http/compression.go
@@ -0,0 +1,142 @@
+package http
+
+import (
+ "bytes"
+ "compress/flate"
+ "compress/gzip"
+ "compress/zlib"
+ "io"
+ "net/http"
+ "strings"
+
+ "github.com/andybalholm/brotli"
+ "github.com/klauspost/compress/zstd"
+)
+
+const MaxDisplayBytes = 64 * 1024 // 64 KB
+
+// Decompress decompresses body according to the Content-Encoding header.
+// Returns the original body unchanged if encoding is unknown or decompression fails.
+//
+// Deflate note: servers claiming "deflate" Content-Encoding almost universally
+// send zlib-wrapped data (RFC 1950), not raw DEFLATE (RFC 1951). We try zlib
+// first and fall back to raw deflate so both are handled correctly.
+func Decompress(header http.Header, body []byte) []byte {
+ encoding := strings.ToLower(strings.TrimSpace(header.Get("Content-Encoding")))
+
+ switch encoding {
+ case "gzip":
+ gr, err := gzip.NewReader(bytes.NewReader(body))
+ if err != nil {
+ return body
+ }
+ defer gr.Close()
+ out, err := io.ReadAll(gr)
+ if err != nil {
+ return body
+ }
+ return out
+
+ case "deflate":
+ // Try zlib (RFC 1950) first — the de-facto standard despite the name.
+ if out, err := decompressZlib(body); err == nil {
+ return out
+ }
+ // Fall back to raw DEFLATE (RFC 1951).
+ out, err := io.ReadAll(flate.NewReader(bytes.NewReader(body)))
+ if err != nil {
+ return body
+ }
+ return out
+
+ case "br":
+ out, err := io.ReadAll(brotli.NewReader(bytes.NewReader(body)))
+ if err != nil {
+ return body
+ }
+ return out
+
+ case "zstd":
+ dec, err := zstd.NewReader(bytes.NewReader(body))
+ if err != nil {
+ return body
+ }
+ defer dec.Close()
+ out, err := io.ReadAll(dec)
+ if err != nil {
+ return body
+ }
+ return out
+
+ case "", "identity":
+ // No encoding or explicit identity — return as-is.
+ return body
+ }
+
+ // Unknown encoding — return original bytes unchanged.
+ return body
+}
+
+// decompressZlib attempts to decompress body as zlib (RFC 1950).
+func decompressZlib(body []byte) ([]byte, error) {
+ r, err := zlib.NewReader(bytes.NewReader(body))
+ if err != nil {
+ return nil, err
+ }
+ defer r.Close()
+ return io.ReadAll(r)
+}
+
+// IsBinary returns true if the response body should not be stored or displayed as text.
+//
+// If Content-Encoding is a known compression we support, the body will be
+// decompressed to text before this is called, so those are not binary.
+// Unknown or future Content-Encoding values are treated as binary since we
+// cannot safely decode them.
+func IsBinary(header http.Header) bool {
+ encoding := strings.ToLower(strings.TrimSpace(header.Get("Content-Encoding")))
+ switch encoding {
+ case "", "identity", "gzip", "deflate", "br", "zstd":
+ // Known encodings we can decompress — fall through to Content-Type check.
+ default:
+ // Unknown encoding — we can't decode it, treat as binary.
+ return true
+ }
+
+ ct := strings.ToLower(header.Get("Content-Type"))
+ if ct == "" {
+ // No Content-Type — assume text so we don't silently discard bodies.
+ return false
+ }
+
+ textPrefixes := []string{
+ "text/",
+ "application/json",
+ "application/xml",
+ "application/javascript",
+ "application/x-www-form-urlencoded",
+ "application/xhtml",
+ "application/ld+json",
+ "application/graphql",
+ "application/x-yaml",
+ "application/yaml",
+ }
+ for _, prefix := range textPrefixes {
+ if strings.Contains(ct, prefix) {
+ return false
+ }
+ }
+ return true
+}
+
+// TruncateBody returns body truncated to MaxDisplayBytes with a notice appended.
+func TruncateBody(body []byte) []byte {
+ if len(body) <= MaxDisplayBytes {
+ return body
+ }
+ notice := []byte("\n... truncated")
+ out := make([]byte, MaxDisplayBytes+len(notice))
+ copy(out, body[:MaxDisplayBytes])
+ copy(out[MaxDisplayBytes:], notice)
+ return out
+}
diff --git a/internal/http/compression_test.go b/internal/http/compression_test.go
new file mode 100644
index 0000000..b675341
--- /dev/null
+++ b/internal/http/compression_test.go
@@ -0,0 +1,152 @@
+package http
+
+import (
+ "bytes"
+ "compress/flate"
+ "compress/gzip"
+ "net/http"
+ "strings"
+ "testing"
+
+ "github.com/andybalholm/brotli"
+ "github.com/klauspost/compress/zstd"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// ── Decompress ────────────────────────────────────────────────────────────────
+
+func TestDecompress_NoEncoding(t *testing.T) {
+ body := []byte("plain")
+ assert.Equal(t, body, Decompress(http.Header{}, body))
+}
+
+func TestDecompress_UnknownEncoding(t *testing.T) {
+ body := []byte("mystery")
+ assert.Equal(t, body, Decompress(http.Header{"Content-Encoding": []string{"identity"}}, body))
+}
+
+func TestDecompress_Gzip(t *testing.T) {
+ var buf bytes.Buffer
+ w := gzip.NewWriter(&buf)
+ _, err := w.Write([]byte("gzip content"))
+ require.NoError(t, err)
+ w.Close()
+ assert.Equal(t, "gzip content", string(Decompress(http.Header{"Content-Encoding": []string{"gzip"}}, buf.Bytes())))
+}
+
+func TestDecompress_Gzip_Corrupt(t *testing.T) {
+ body := []byte("not gzip")
+ assert.Equal(t, body, Decompress(http.Header{"Content-Encoding": []string{"gzip"}}, body))
+}
+
+func TestDecompress_Deflate(t *testing.T) {
+ var buf bytes.Buffer
+ w, err := flate.NewWriter(&buf, flate.DefaultCompression)
+ require.NoError(t, err)
+ w.Write([]byte("deflate content"))
+ w.Close()
+ assert.Equal(t, "deflate content", string(Decompress(http.Header{"Content-Encoding": []string{"deflate"}}, buf.Bytes())))
+}
+
+func TestDecompress_Deflate_Corrupt(t *testing.T) {
+ body := []byte("not deflate")
+ assert.Equal(t, body, Decompress(http.Header{"Content-Encoding": []string{"deflate"}}, body))
+}
+
+func TestDecompress_Brotli(t *testing.T) {
+ var buf bytes.Buffer
+ w := brotli.NewWriter(&buf)
+ w.Write([]byte("brotli content"))
+ w.Close()
+ assert.Equal(t, "brotli content", string(Decompress(http.Header{"Content-Encoding": []string{"br"}}, buf.Bytes())))
+}
+
+func TestDecompress_Brotli_Corrupt(t *testing.T) {
+ body := []byte("not brotli")
+ assert.Equal(t, body, Decompress(http.Header{"Content-Encoding": []string{"br"}}, body))
+}
+
+func TestDecompress_Zstd(t *testing.T) {
+ enc, err := zstd.NewWriter(nil)
+ require.NoError(t, err)
+ compressed := enc.EncodeAll([]byte("zstd content"), nil)
+ assert.Equal(t, "zstd content", string(Decompress(http.Header{"Content-Encoding": []string{"zstd"}}, compressed)))
+}
+
+func TestDecompress_Zstd_Corrupt(t *testing.T) {
+ body := []byte("not zstd")
+ assert.Equal(t, body, Decompress(http.Header{"Content-Encoding": []string{"zstd"}}, body))
+}
+
+func TestDecompress_CaseInsensitive(t *testing.T) {
+ var buf bytes.Buffer
+ w := gzip.NewWriter(&buf)
+ w.Write([]byte("case"))
+ w.Close()
+ assert.Equal(t, "case", string(Decompress(http.Header{"Content-Encoding": []string{"GZIP"}}, buf.Bytes())))
+}
+
+func TestDecompress_EmptyBody(t *testing.T) {
+ assert.Equal(t, []byte{}, Decompress(http.Header{"Content-Encoding": []string{"gzip"}}, []byte{}))
+}
+
+func TestDecompress_LargeBody(t *testing.T) {
+ large := []byte(strings.Repeat("a", 100000))
+ var buf bytes.Buffer
+ w := gzip.NewWriter(&buf)
+ w.Write(large)
+ w.Close()
+ out := Decompress(http.Header{"Content-Encoding": []string{"gzip"}}, buf.Bytes())
+ assert.Equal(t, large, out)
+}
+
+// ── IsBinary ──────────────────────────────────────────────────────────────────
+
+func TestIsBinary_KnownEncodingsNotBinary(t *testing.T) {
+ // Known encodings we can decompress fall through to the Content-Type check.
+ // image/png is still binary even with a known encoding — the encoding tells
+ // us we can decompress it, but the content type determines binary/text.
+ for _, enc := range []string{"gzip", "deflate", "br", "zstd"} {
+ h := http.Header{
+ "Content-Encoding": []string{enc},
+ "Content-Type": []string{"text/plain"},
+ }
+ assert.False(t, IsBinary(h), "encoding %s with text/plain should not be binary", enc)
+ }
+ // image/png with a known encoding is still binary.
+ for _, enc := range []string{"gzip", "deflate", "br", "zstd"} {
+ h := http.Header{
+ "Content-Encoding": []string{enc},
+ "Content-Type": []string{"image/png"},
+ }
+ assert.True(t, IsBinary(h), "encoding %s with image/png should be binary", enc)
+ }
+}
+
+func TestIsBinary_BinaryContentTypes(t *testing.T) {
+ for _, ct := range []string{"image/png", "image/jpeg", "application/octet-stream", "video/mp4", "audio/mpeg"} {
+ h := http.Header{"Content-Type": []string{ct}}
+ assert.True(t, IsBinary(h), "%s should be binary", ct)
+ }
+}
+
+func TestIsBinary_TextContentTypes(t *testing.T) {
+ for _, ct := range []string{
+ "text/plain", "text/html", "text/css",
+ "application/json", "application/xml",
+ "application/javascript", "application/x-www-form-urlencoded",
+ "application/xhtml+xml", "application/ld+json",
+ } {
+ h := http.Header{"Content-Type": []string{ct}}
+ assert.False(t, IsBinary(h), "%s should not be binary", ct)
+ }
+}
+
+func TestIsBinary_EmptyContentType(t *testing.T) {
+ assert.False(t, IsBinary(http.Header{"Content-Type": []string{""}}))
+}
+
+func TestIsBinary_NoHeaders(t *testing.T) {
+ assert.False(t, IsBinary(http.Header{}))
+}
diff --git a/internal/http/headers.go b/internal/http/headers.go
new file mode 100644
index 0000000..d06d584
--- /dev/null
+++ b/internal/http/headers.go
@@ -0,0 +1,54 @@
+package http
+
+import (
+ "net"
+ "net/http"
+)
+
+// NormalizeHost strips the port from a Host header value when it is the
+// default for the scheme (443 for HTTPS, 80 for HTTP), so that we never
+// send "Host: example.com:443" to an upstream server.
+func NormalizeHost(host string, tls bool) string {
+ h, port, err := net.SplitHostPort(host)
+ if err != nil {
+ return host
+ }
+ if (tls && port == "443") || (!tls && port == "80") {
+ return h
+ }
+ return host
+}
+
+// StripRequestCacheHeaders removes conditional-request headers so the
+// upstream always returns a fresh response.
+func StripRequestCacheHeaders(h http.Header) {
+ h.Del("If-None-Match")
+ h.Del("If-Modified-Since")
+ h.Del("If-Range")
+ h.Del("If-Match")
+ h.Del("If-Unmodified-Since")
+}
+
+// StripResponseCacheHeaders removes caching directives from a response and
+// sets no-store so the browser never serves a stale cached copy.
+func StripResponseCacheHeaders(h http.Header) {
+ h.Del("Cache-Control")
+ h.Del("Expires")
+ h.Del("ETag")
+ h.Del("Last-Modified")
+ h.Del("Pragma")
+ h.Set("Cache-Control", "no-store, no-cache, must-revalidate")
+}
+
+// HopByHop is the set of headers that must not be forwarded to the upstream.
+var HopByHop = map[string]bool{
+ "Connection": true,
+ "Proxy-Connection": true,
+ "Keep-Alive": true,
+ "Proxy-Authenticate": true,
+ "Proxy-Authorization": true,
+ "Te": true,
+ "Trailers": true,
+ "Transfer-Encoding": true,
+ "Upgrade": true,
+}
diff --git a/internal/http/headers_test.go b/internal/http/headers_test.go
new file mode 100644
index 0000000..2378da6
--- /dev/null
+++ b/internal/http/headers_test.go
@@ -0,0 +1,117 @@
+package http
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+// ── NormalizeHost ─────────────────────────────────────────────────────────────
+
+func TestNormalizeHost_TLS_443_Stripped(t *testing.T) {
+ assert.Equal(t, "example.com", NormalizeHost("example.com:443", true))
+}
+
+func TestNormalizeHost_Plain_80_Stripped(t *testing.T) {
+ assert.Equal(t, "example.com", NormalizeHost("example.com:80", false))
+}
+
+func TestNormalizeHost_TLS_NonDefault_Kept(t *testing.T) {
+ assert.Equal(t, "example.com:8443", NormalizeHost("example.com:8443", true))
+}
+
+func TestNormalizeHost_Plain_NonDefault_Kept(t *testing.T) {
+ assert.Equal(t, "example.com:8080", NormalizeHost("example.com:8080", false))
+}
+
+func TestNormalizeHost_Plain_443_Kept(t *testing.T) {
+ // port 443 on plain (non-TLS) should not be stripped
+ assert.Equal(t, "example.com:443", NormalizeHost("example.com:443", false))
+}
+
+func TestNormalizeHost_TLS_80_Kept(t *testing.T) {
+ // port 80 on TLS should not be stripped
+ assert.Equal(t, "example.com:80", NormalizeHost("example.com:80", true))
+}
+
+func TestNormalizeHost_NoPort_ReturnedAsIs(t *testing.T) {
+ assert.Equal(t, "example.com", NormalizeHost("example.com", true))
+}
+
+func TestNormalizeHost_IP_WithPort443(t *testing.T) {
+ assert.Equal(t, "127.0.0.1", NormalizeHost("127.0.0.1:443", true))
+}
+
+// ── StripRequestCacheHeaders ──────────────────────────────────────────────────
+
+func TestStripRequestCacheHeaders_AllStripped(t *testing.T) {
+ h := http.Header{
+ "If-None-Match": []string{`"etag"`},
+ "If-Modified-Since": []string{"Mon, 01 Jan 2024 00:00:00 GMT"},
+ "If-Range": []string{`"etag"`},
+ "If-Match": []string{`"etag"`},
+ "If-Unmodified-Since": []string{"Mon, 01 Jan 2024 00:00:00 GMT"},
+ "Authorization": []string{"Bearer token"},
+ }
+ StripRequestCacheHeaders(h)
+
+ assert.Empty(t, h.Get("If-None-Match"))
+ assert.Empty(t, h.Get("If-Modified-Since"))
+ assert.Empty(t, h.Get("If-Range"))
+ assert.Empty(t, h.Get("If-Match"))
+ assert.Empty(t, h.Get("If-Unmodified-Since"))
+ assert.Equal(t, "Bearer token", h.Get("Authorization"))
+}
+
+func TestStripRequestCacheHeaders_AlreadyAbsent(t *testing.T) {
+ h := http.Header{"Content-Type": []string{"application/json"}}
+ assert.NotPanics(t, func() { StripRequestCacheHeaders(h) })
+ assert.Equal(t, "application/json", h.Get("Content-Type"))
+}
+
+// ── StripResponseCacheHeaders ─────────────────────────────────────────────────
+
+func TestStripResponseCacheHeaders_AllStripped(t *testing.T) {
+ h := http.Header{
+ "Cache-Control": []string{"max-age=3600"},
+ "Expires": []string{"Mon, 01 Jan 2025 00:00:00 GMT"},
+ "ETag": []string{`"etag"`},
+ "Last-Modified": []string{"Mon, 01 Jan 2024 00:00:00 GMT"},
+ "Pragma": []string{"no-cache"},
+ "Content-Type": []string{"text/html"},
+ }
+ StripResponseCacheHeaders(h)
+
+ assert.Equal(t, "no-store, no-cache, must-revalidate", h.Get("Cache-Control"))
+ assert.Empty(t, h.Get("Expires"))
+ assert.Empty(t, h.Get("ETag"))
+ assert.Empty(t, h.Get("Last-Modified"))
+ assert.Empty(t, h.Get("Pragma"))
+ assert.Equal(t, "text/html", h.Get("Content-Type"))
+}
+
+func TestStripResponseCacheHeaders_SetEvenWhenAbsent(t *testing.T) {
+ h := http.Header{}
+ StripResponseCacheHeaders(h)
+ assert.Equal(t, "no-store, no-cache, must-revalidate", h.Get("Cache-Control"))
+}
+
+// ── HopByHop ──────────────────────────────────────────────────────────────────
+
+func TestHopByHop_ContainsExpectedHeaders(t *testing.T) {
+ expected := []string{
+ "Connection", "Proxy-Connection", "Keep-Alive",
+ "Proxy-Authenticate", "Proxy-Authorization",
+ "Te", "Trailers", "Transfer-Encoding", "Upgrade",
+ }
+ for _, h := range expected {
+ assert.True(t, HopByHop[h], "expected %s in HopByHop", h)
+ }
+}
+
+func TestHopByHop_DoesNotContainRegularHeaders(t *testing.T) {
+ assert.False(t, HopByHop["Content-Type"])
+ assert.False(t, HopByHop["Authorization"])
+ assert.False(t, HopByHop["Host"])
+}
diff --git a/internal/http/rawrequest.go b/internal/http/rawrequest.go
new file mode 100644
index 0000000..090787a
--- /dev/null
+++ b/internal/http/rawrequest.go
@@ -0,0 +1,327 @@
+package http
+
+import (
+ "bufio"
+ "crypto/tls"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// RawRequestOptions controls how SendRaw dials and sends.
+type RawRequestOptions struct {
+ Host string
+ Port int
+ TLS bool
+ RawReq string
+ CookieJar map[string]string // optional; merged into Cookie header
+ TimeoutMs int // 0 means use defaults (10s dial, 30s total)
+}
+
+// RawResponse is returned by SendRaw.
+type RawResponse struct {
+ Raw string // full response as text (status line + headers + body)
+ StatusCode int
+ Headers http.Header
+ Body []byte
+ Cookies []*http.Cookie
+}
+
+// SendRaw dials host:port, sends rawReq over a plain or TLS connection, and
+// returns the decompressed, decoded response. It is intentionally low-level
+// so the caller (Repeater, Intruder) controls the exact bytes on the wire.
+//
+// The request line is always rewritten to HTTP/1.1 — H2 frames cannot be
+// sent as raw text, and a raw socket is always HTTP/1.1.
+func SendRaw(opts RawRequestOptions) (*RawResponse, error) {
+ addr := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port))
+
+ timeout := 30 * time.Second
+ if opts.TimeoutMs > 0 {
+ timeout = time.Duration(opts.TimeoutMs) * time.Millisecond
+ }
+
+ var conn net.Conn
+ var err error
+ if opts.TLS {
+ dialer := &net.Dialer{Timeout: timeout}
+ tlsCfg := &tls.Config{
+ ServerName: opts.Host,
+ InsecureSkipVerify: true, //nolint:gosec
+ }
+ conn, err = tls.DialWithDialer(dialer, "tcp", addr, tlsCfg)
+ } else {
+ conn, err = net.DialTimeout("tcp", addr, timeout)
+ }
+ if err != nil {
+ return nil, fmt.Errorf("dial %s: %w", addr, err)
+ }
+ defer conn.Close()
+
+ if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil {
+ return nil, fmt.Errorf("set deadline: %w", err)
+ }
+
+ raw := NormalizeLineEndings(opts.RawReq)
+ raw = rewriteRequestLine(raw)
+ raw = rewriteHeaders(raw, opts.CookieJar)
+
+ if _, err := fmt.Fprint(conn, raw); err != nil {
+ return nil, fmt.Errorf("write request: %w", err)
+ }
+
+ httpResp, err := http.ReadResponse(bufio.NewReader(conn), nil)
+ if err != nil {
+ return nil, fmt.Errorf("read response: %w", err)
+ }
+ defer httpResp.Body.Close()
+
+ body, err := io.ReadAll(io.LimitReader(httpResp.Body, 2<<20))
+ if err != nil {
+ return nil, fmt.Errorf("read response body: %w", err)
+ }
+ body = Decompress(httpResp.Header, body)
+ body = TruncateBody(body)
+ rawResp := buildRawResponse(httpResp, body)
+
+ return &RawResponse{
+ Raw: rawResp,
+ StatusCode: httpResp.StatusCode,
+ Headers: httpResp.Header,
+ Body: body,
+ Cookies: httpResp.Cookies(),
+ }, nil
+}
+
+// rewriteRequestLine rewrites the first line of a raw HTTP request to use
+// HTTP/1.1, regardless of what was captured. HTTP/2 cannot be sent as raw
+// text over a socket.
+func rewriteRequestLine(raw string) string {
+ idx := strings.Index(raw, "\r\n")
+ if idx < 0 {
+ return raw
+ }
+ firstLine := raw[:idx]
+ parts := strings.Fields(firstLine)
+ if len(parts) < 2 {
+ return raw
+ }
+ // Rewrite: METHOD PATH HTTP/1.1
+ newFirstLine := parts[0] + " " + parts[1] + " HTTP/1.1"
+ return newFirstLine + raw[idx:]
+}
+
+// ParseHostFromRaw extracts host, port, and TLS flag from the Host header
+// in a raw HTTP request string. The caller should pass the known TLS state
+// from the captured transaction when available, as the Host header alone
+// cannot distinguish HTTP from HTTPS.
+func ParseHostFromRaw(raw string) (host string, port int, useTLS bool) {
+ // Check the scheme in the request line first (e.g. absolute-form URLs).
+ firstLine := strings.SplitN(raw, "\n", 2)[0]
+ parts := strings.Fields(firstLine)
+ if len(parts) >= 2 {
+ url := parts[1]
+ if strings.HasPrefix(url, "https://") {
+ useTLS = true
+ } else if strings.HasPrefix(url, "http://") {
+ useTLS = false
+ }
+ }
+
+ for line := range strings.SplitSeq(raw, "\n") {
+ line = strings.TrimSpace(line)
+ if !strings.HasPrefix(strings.ToLower(line), "host:") {
+ continue
+ }
+ hostVal := strings.TrimSpace(line[5:])
+ if h, portStr, err := net.SplitHostPort(hostVal); err == nil {
+ host = h
+ port, _ = strconv.Atoi(portStr)
+ // Port in Host header is the definitive signal.
+ useTLS = port == 443
+ } else {
+ host = hostVal
+ if net.ParseIP(hostVal) != nil {
+ // Bare IP without port — can't tell, default to plaintext.
+ port = 80
+ useTLS = false
+ } else {
+ // Domain without port — use whatever we detected from the
+ // request line scheme, or fall back to HTTPS.
+ if port == 0 {
+ if useTLS {
+ port = 443
+ } else {
+ port = 80
+ }
+ }
+ // If we got here without a scheme signal, default to HTTPS
+ // since most modern interception is TLS.
+ if port == 443 {
+ useTLS = true
+ }
+ }
+ }
+ return
+ }
+ return "", 0, false
+}
+
+// ParseRawHeaders parses the headers section of a raw HTTP response string
+// into an http.Header map.
+func ParseRawHeaders(raw string) http.Header {
+ headers := http.Header{}
+ lines := strings.Split(raw, "\r\n")
+ if len(lines) == 0 {
+ return headers
+ }
+ // Skip the status/request line.
+ for _, line := range lines[1:] {
+ if line == "" {
+ break
+ }
+ parts := strings.SplitN(line, ": ", 2)
+ if len(parts) == 2 {
+ headers.Add(parts[0], parts[1])
+ }
+ }
+ return headers
+}
+
+// SplitRaw splits a raw HTTP message into its header section and body at the
+// first blank line. Handles both CRLF and LF delimiters. The returned header
+// section does not include the blank-line separator.
+func SplitRaw(raw string) (header string, body []byte) {
+ if idx := strings.Index(raw, "\r\n\r\n"); idx >= 0 {
+ return raw[:idx], []byte(raw[idx+4:])
+ }
+ if idx := strings.Index(raw, "\n\n"); idx >= 0 {
+ return raw[:idx], []byte(raw[idx+2:])
+ }
+ return raw, nil
+}
+
+// NormalizeLineEndings ensures CRLF throughout the header section only.
+// It does NOT touch the body to avoid corrupting binary or multi-byte content.
+func NormalizeLineEndings(raw string) string {
+ header, body := SplitRaw(raw)
+ header = strings.ReplaceAll(header, "\r\n", "\n")
+ header = strings.ReplaceAll(header, "\n", "\r\n")
+ return header + "\r\n\r\n" + string(body)
+}
+
+// rewriteHeaders rewrites the header section of a raw request:
+// - Injects Connection: close (required for single-use raw TCP connections)
+// - Strips Accept-Encoding (we handle decompression ourselves)
+// - Strips Content-Length (recalculated below)
+// - Strips Transfer-Encoding (not valid in HTTP/1.1 raw send context)
+// - Merges Cookie header from jar (replaces existing if jar is non-empty)
+// - Strips default port from Host when it matches the scheme
+// - Appends correct Content-Length based on byte length of body
+func rewriteHeaders(raw string, jar map[string]string) string {
+ // Split on the canonical blank line separator (normalizeLineEndings runs first).
+ parts := strings.SplitN(raw, "\r\n\r\n", 2)
+ headerSection := parts[0]
+ body := ""
+ if len(parts) == 2 {
+ body = parts[1]
+ // Trim trailing CRLF from body only if it was added by normalisation,
+ // not if the user intentionally added trailing newlines.
+ // We use the raw byte length for Content-Length regardless.
+ }
+
+ lines := strings.Split(headerSection, "\r\n")
+ out := make([]string, 0, len(lines)+3)
+ connectionSeen := false
+
+ for _, line := range lines {
+ lower := strings.ToLower(line)
+ switch {
+ case strings.HasPrefix(lower, "cookie:"):
+ // Drop original Cookie header only if jar has entries to replace it.
+ if len(jar) == 0 {
+ out = append(out, line)
+ }
+ case strings.HasPrefix(lower, "accept-encoding:"):
+ // Drop — we decompress ourselves so we want uncompressed bytes.
+ continue
+ case strings.HasPrefix(lower, "content-length:"):
+ // Drop — we recalculate below from actual body byte length.
+ continue
+ case strings.HasPrefix(lower, "transfer-encoding:"):
+ // Not valid to send in this raw H1 context.
+ continue
+ case strings.HasPrefix(lower, "connection:"):
+ // Replace whatever was there with Connection: close.
+ out = append(out, "Connection: close")
+ connectionSeen = true
+ case strings.HasPrefix(lower, "host:"):
+ hostVal := strings.TrimSpace(line[5:])
+ // Strip default ports from Host header.
+ if h, port, err := net.SplitHostPort(hostVal); err == nil {
+ if port == "443" || port == "80" {
+ line = "Host: " + h
+ }
+ }
+ out = append(out, line)
+ default:
+ out = append(out, line)
+ }
+ }
+
+ // Ensure Connection: close is present even if the original had no Connection header.
+ if !connectionSeen {
+ out = append(out, "Connection: close")
+ }
+
+ // Append merged Cookie header from jar.
+ if len(jar) > 0 {
+ cookies := make([]string, 0, len(jar))
+ for k, v := range jar {
+ cookies = append(cookies, k+"="+v)
+ }
+ out = append(out, "Cookie: "+strings.Join(cookies, "; "))
+ }
+
+ // Append Content-Length based on actual byte length of the body.
+ if len(body) > 0 {
+ out = append(out, fmt.Sprintf("Content-Length: %d", len(body)))
+ }
+
+ return strings.Join(out, "\r\n") + "\r\n\r\n" + body
+}
+
+func buildRawResponse(resp *http.Response, body []byte) string {
+ var b strings.Builder
+ fmt.Fprintf(&b, "HTTP/1.1 %d %s\r\n", resp.StatusCode, http.StatusText(resp.StatusCode))
+ for k, vals := range resp.Header {
+ for _, v := range vals {
+ fmt.Fprintf(&b, "%s: %s\r\n", k, v)
+ }
+ }
+ b.WriteString("\r\n")
+ b.Write(body)
+ return b.String()
+}
+
+// ExtractURL extracts the path from the first line of a raw HTTP request.
+func ExtractURL(rawRequest string) string {
+ parts := strings.Fields(strings.SplitN(rawRequest, "\n", 2)[0])
+ if len(parts) >= 2 {
+ return parts[1]
+ }
+ return "/"
+}
+
+// ExtractMethod extracts the HTTP method from the first line of a raw HTTP request.
+func ExtractMethod(rawRequest string) string {
+ parts := strings.Fields(strings.SplitN(rawRequest, "\n", 2)[0])
+ if len(parts) >= 1 {
+ return parts[0]
+ }
+ return "GET"
+}
diff --git a/internal/http/rawrequest_test.go b/internal/http/rawrequest_test.go
new file mode 100644
index 0000000..5dd411c
--- /dev/null
+++ b/internal/http/rawrequest_test.go
@@ -0,0 +1,254 @@
+package http
+
+import (
+ "fmt"
+ "net"
+ "net/http"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// ── ParseHostFromRaw ──────────────────────────────────────────────────────────
+
+func TestParseHostFromRaw_HostOnly(t *testing.T) {
+ host, port, useTLS := ParseHostFromRaw("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
+ assert.Equal(t, "example.com", host)
+ assert.Equal(t, 80, port)
+ assert.False(t, useTLS)
+}
+
+func TestParseHostFromRaw_HostWithPort443(t *testing.T) {
+ host, port, useTLS := ParseHostFromRaw("GET / HTTP/1.1\r\nHost: example.com:443\r\n\r\n")
+ assert.Equal(t, "example.com", host)
+ assert.Equal(t, 443, port)
+ assert.True(t, useTLS)
+}
+
+func TestParseHostFromRaw_HostWithPort80(t *testing.T) {
+ host, port, useTLS := ParseHostFromRaw("GET / HTTP/1.1\r\nHost: example.com:80\r\n\r\n")
+ assert.Equal(t, "example.com", host)
+ assert.Equal(t, 80, port)
+ assert.False(t, useTLS)
+}
+
+func TestParseHostFromRaw_CustomPort(t *testing.T) {
+ host, port, useTLS := ParseHostFromRaw("GET / HTTP/1.1\r\nHost: example.com:8080\r\n\r\n")
+ assert.Equal(t, "example.com", host)
+ assert.Equal(t, 8080, port)
+ assert.False(t, useTLS)
+}
+
+func TestParseHostFromRaw_IP(t *testing.T) {
+ host, port, useTLS := ParseHostFromRaw("GET / HTTP/1.1\r\nHost: 1.2.3.4\r\n\r\n")
+ assert.Equal(t, "1.2.3.4", host)
+ assert.Equal(t, 80, port)
+ assert.False(t, useTLS)
+}
+
+func TestParseHostFromRaw_NoHostHeader(t *testing.T) {
+ host, port, useTLS := ParseHostFromRaw("GET / HTTP/1.1\r\nContent-Type: text/plain\r\n\r\n")
+ assert.Empty(t, host)
+ assert.Equal(t, 0, port)
+ assert.False(t, useTLS)
+}
+
+func TestParseHostFromRaw_CaseInsensitive(t *testing.T) {
+ host, _, _ := ParseHostFromRaw("GET / HTTP/1.1\r\nHOST: example.com\r\n\r\n")
+ assert.Equal(t, "example.com", host)
+}
+
+func TestParseHostFromRaw_LFOnly(t *testing.T) {
+ host, port, _ := ParseHostFromRaw("GET / HTTP/1.1\nHost: example.com\n\n")
+ assert.Equal(t, "example.com", host)
+ assert.Equal(t, 80, port)
+}
+
+// ── ParseRawHeaders ───────────────────────────────────────────────────────────
+
+func TestParseRawHeaders_Single(t *testing.T) {
+ raw := "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n"
+ h := ParseRawHeaders(raw)
+ assert.Equal(t, "text/html", h.Get("Content-Type"))
+}
+
+func TestParseRawHeaders_Multiple(t *testing.T) {
+ raw := "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nX-Custom: value\r\n\r\n"
+ h := ParseRawHeaders(raw)
+ assert.Equal(t, "text/html", h.Get("Content-Type"))
+ assert.Equal(t, "value", h.Get("X-Custom"))
+}
+
+func TestParseRawHeaders_StopsAtBlankLine(t *testing.T) {
+ raw := "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\nbody here"
+ h := ParseRawHeaders(raw)
+ assert.Equal(t, "text/html", h.Get("Content-Type"))
+ assert.Empty(t, h.Get("body here"))
+}
+
+func TestParseRawHeaders_Empty(t *testing.T) {
+ h := ParseRawHeaders("HTTP/1.1 200 OK\r\n\r\n")
+ assert.Equal(t, http.Header{}, h)
+}
+
+// ── normalizeLineEndings ──────────────────────────────────────────────────────
+
+func TestNormalizeLineEndings_LFtoCRLF(t *testing.T) {
+ out := NormalizeLineEndings("GET / HTTP/1.1\nHost: example.com\n\n")
+ assert.Contains(t, out, "\r\n")
+ assert.NotContains(t, out, "\n\n")
+}
+
+func TestNormalizeLineEndings_AlreadyCRLF(t *testing.T) {
+ in := "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
+ out := NormalizeLineEndings(in)
+ assert.Equal(t, in, out)
+}
+
+func TestNormalizeLineEndings_FoldsContinuation(t *testing.T) {
+ // Continuation line folding was removed to avoid corrupting body content.
+ // normalizeLineEndings only touches the header section and converts LF to CRLF.
+ // Continuation lines are preserved as-is.
+ in := "GET / HTTP/1.1\r\nHost: example.com\r\n continued\r\n\r\n"
+ out := NormalizeLineEndings(in)
+ assert.Equal(t, in, out)
+}
+
+// ── rewriteHeaders ────────────────────────────────────────────────────────────
+
+func TestRewriteHeaders_StripsAcceptEncoding(t *testing.T) {
+ raw := "GET / HTTP/1.1\r\nHost: example.com\r\nAccept-Encoding: gzip\r\n\r\n"
+ out := rewriteHeaders(raw, nil)
+ assert.NotContains(t, out, "Accept-Encoding")
+}
+
+func TestRewriteHeaders_StripsContentLength(t *testing.T) {
+ raw := "POST / HTTP/1.1\r\nHost: example.com\r\nContent-Length: 999\r\n\r\nbody"
+ out := rewriteHeaders(raw, nil)
+ assert.NotContains(t, out, "Content-Length: 999")
+ assert.Contains(t, out, fmt.Sprintf("Content-Length: %d", len("body")))
+}
+
+func TestRewriteHeaders_PreservesCookieWhenNoJar(t *testing.T) {
+ raw := "GET / HTTP/1.1\r\nHost: example.com\r\nCookie: session=abc\r\n\r\n"
+ out := rewriteHeaders(raw, nil)
+ assert.Contains(t, out, "Cookie: session=abc")
+}
+
+func TestRewriteHeaders_ReplacesCookieFromJar(t *testing.T) {
+ raw := "GET / HTTP/1.1\r\nHost: example.com\r\nCookie: session=old\r\n\r\n"
+ out := rewriteHeaders(raw, map[string]string{"session": "new"})
+ assert.NotContains(t, out, "session=old")
+ assert.Contains(t, out, "session=new")
+}
+
+func TestRewriteHeaders_StripsDefaultPort443(t *testing.T) {
+ raw := "GET / HTTP/1.1\r\nHost: example.com:443\r\n\r\n"
+ out := rewriteHeaders(raw, nil)
+ assert.Contains(t, out, "Host: example.com\r\n")
+ assert.NotContains(t, out, "Host: example.com:443")
+}
+
+func TestRewriteHeaders_StripsDefaultPort80(t *testing.T) {
+ raw := "GET / HTTP/1.1\r\nHost: example.com:80\r\n\r\n"
+ out := rewriteHeaders(raw, nil)
+ assert.Contains(t, out, "Host: example.com\r\n")
+ assert.NotContains(t, out, "Host: example.com:80")
+}
+
+func TestRewriteHeaders_KeepsNonDefaultPort(t *testing.T) {
+ raw := "GET / HTTP/1.1\r\nHost: example.com:8080\r\n\r\n"
+ out := rewriteHeaders(raw, nil)
+ assert.Contains(t, out, "Host: example.com:8080")
+}
+
+// ── SendRaw (mock TCP server) ─────────────────────────────────────────────────
+
+func startMockServer(t *testing.T, response string) (string, int) {
+ t.Helper()
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+ go func() {
+ conn, err := ln.Accept()
+ if err != nil {
+ return
+ }
+ defer conn.Close()
+ buf := make([]byte, 4096)
+ conn.Read(buf)
+ fmt.Fprint(conn, response)
+ }()
+ t.Cleanup(func() { ln.Close() })
+ addr := ln.Addr().(*net.TCPAddr)
+ return "127.0.0.1", addr.Port
+}
+
+func TestSendRaw_200OK(t *testing.T) {
+ host, port := startMockServer(t, "HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello")
+ raw := fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s:%d\r\n\r\n", host, port)
+ result, err := SendRaw(RawRequestOptions{Host: host, Port: port, RawReq: raw})
+ require.NoError(t, err)
+ assert.Equal(t, 200, result.StatusCode)
+ assert.Equal(t, "hello", string(result.Body))
+}
+
+func TestSendRaw_404(t *testing.T) {
+ host, port := startMockServer(t, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")
+ raw := fmt.Sprintf("GET /missing HTTP/1.1\r\nHost: %s:%d\r\n\r\n", host, port)
+ result, err := SendRaw(RawRequestOptions{Host: host, Port: port, RawReq: raw})
+ require.NoError(t, err)
+ assert.Equal(t, 404, result.StatusCode)
+}
+
+func TestSendRaw_ConnectionRefused(t *testing.T) {
+ _, err := SendRaw(RawRequestOptions{
+ Host: "127.0.0.1", Port: 1,
+ RawReq: "GET / HTTP/1.1\r\nHost: 127.0.0.1:1\r\n\r\n",
+ })
+ assert.Error(t, err)
+}
+
+func TestSendRaw_CookieJarApplied(t *testing.T) {
+ received := make(chan string, 1)
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+ t.Cleanup(func() { ln.Close() })
+ go func() {
+ conn, _ := ln.Accept()
+ defer conn.Close()
+ buf := make([]byte, 4096)
+ n, _ := conn.Read(buf)
+ received <- string(buf[:n])
+ fmt.Fprint(conn, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
+ }()
+ addr := ln.Addr().(*net.TCPAddr)
+ raw := fmt.Sprintf("GET / HTTP/1.1\r\nHost: 127.0.0.1:%d\r\n\r\n", addr.Port)
+ SendRaw(RawRequestOptions{
+ Host: "127.0.0.1", Port: addr.Port,
+ RawReq: raw,
+ CookieJar: map[string]string{"session": "abc123"},
+ })
+ req := <-received
+ assert.Contains(t, req, "Cookie: session=abc123")
+}
+
+func TestSendRaw_AcceptEncodingStripped(t *testing.T) {
+ received := make(chan string, 1)
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+ t.Cleanup(func() { ln.Close() })
+ go func() {
+ conn, _ := ln.Accept()
+ defer conn.Close()
+ buf := make([]byte, 4096)
+ n, _ := conn.Read(buf)
+ received <- string(buf[:n])
+ fmt.Fprint(conn, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
+ }()
+ addr := ln.Addr().(*net.TCPAddr)
+ raw := fmt.Sprintf("GET / HTTP/1.1\r\nHost: 127.0.0.1:%d\r\nAccept-Encoding: gzip\r\n\r\n", addr.Port)
+ SendRaw(RawRequestOptions{Host: "127.0.0.1", Port: addr.Port, RawReq: raw})
+ req := <-received
+ assert.NotContains(t, req, "Accept-Encoding")
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-10-23 01:08:38 +0000