diff --git a/authbridge/authlib/config/config.go b/authbridge/authlib/config/config.go index 36b388ff8..349b9970a 100644 --- a/authbridge/authlib/config/config.go +++ b/authbridge/authlib/config/config.go @@ -317,6 +317,19 @@ type ListenerConfig struct { ReverseProxyAddr string `yaml:"reverse_proxy_addr" json:"reverse_proxy_addr"` ReverseProxyBackend string `yaml:"reverse_proxy_backend" json:"reverse_proxy_backend"` + // TransparentProxyAddr is the bind address for the outbound transparent + // listener used by proxy-sidecar enforce-redirect mode: iptables REDIRECTs + // the agent's bypass egress here, and the listener recovers the original + // destination via SO_ORIGINAL_DST and tunnels it through the same outbound + // pipeline as the forward proxy. The proxy-sidecar / lite presets default it + // to ":8082", so for those shapes the listener is effectively always on — + // binding is harmless when nothing is redirected to it (cooperative + // HTTP_PROXY deployments simply never receive connections on it). An empty + // value only disables the listener for modes that have no preset default for + // this field (e.g. waypoint / envoy-sidecar); under proxy-sidecar / lite the + // preset refills it, matching the always-on enforce-redirect design. + TransparentProxyAddr string `yaml:"transparent_proxy_addr" json:"transparent_proxy_addr"` + // SessionAPIAddr is the bind address for the session events HTTP server // (JSON snapshots + SSE stream consumed by abctl or curl). Default per // mode preset is ":9094". Set to empty string to disable the endpoint. diff --git a/authbridge/authlib/config/presets.go b/authbridge/authlib/config/presets.go index 0359046f0..3162879cc 100644 --- a/authbridge/authlib/config/presets.go +++ b/authbridge/authlib/config/presets.go @@ -16,6 +16,10 @@ func ApplyPreset(cfg *Config) { case ModeProxySidecar: setDefault(&cfg.Listener.ReverseProxyAddr, ":8080") setDefault(&cfg.Listener.ForwardProxyAddr, ":8081") + // Outbound transparent listener for enforce-redirect mode. Binding it + // is harmless when nothing is redirected here (cooperative HTTP_PROXY + // deployments simply never receive connections on it). + setDefault(&cfg.Listener.TransparentProxyAddr, ":8082") } // Session events API is default-on for every mode. Operators who diff --git a/authbridge/authlib/go.mod b/authbridge/authlib/go.mod index 8381e0242..dd1627835 100644 --- a/authbridge/authlib/go.mod +++ b/authbridge/authlib/go.mod @@ -9,6 +9,7 @@ require ( github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/open-policy-agent/opa v1.4.2 github.com/spiffe/go-spiffe/v2 v2.6.0 + golang.org/x/sys v0.42.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 google.golang.org/grpc v1.81.1 gopkg.in/yaml.v3 v3.0.1 @@ -68,7 +69,6 @@ require ( golang.org/x/crypto v0.48.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.12.0 // indirect google.golang.org/protobuf v1.36.11 // indirect diff --git a/authbridge/authlib/listener/forwardproxy/server.go b/authbridge/authlib/listener/forwardproxy/server.go index 1713c5127..cc2e6e356 100644 --- a/authbridge/authlib/listener/forwardproxy/server.go +++ b/authbridge/authlib/listener/forwardproxy/server.go @@ -738,41 +738,11 @@ func (s *Server) handleConnect(w http.ResponseWriter, r *http.Request) { // Record a SessionRequest event so /v1/sessions and abctl show that // a tunnel was opened. Mirrors the HTTP path's post-Allow recording - // (see handleRequest above). The MCP / Inference snapshots are nil - // by definition (CONNECT bytes are opaque), but Invocations from - // gate plugins (ibac, token-exchange's skip/no_route, etc.) and - // any plugin-public Plugins entries are still meaningful. - if s.Sessions != nil { - sid := s.Sessions.ActiveSession() - if sid == "" { - sid = session.DefaultSessionID - } - plugins := pipeline.SnapshotPlugins(pctx.Extensions.Custom) - ev := pipeline.SessionEvent{ - At: time.Now(), - Direction: pipeline.Outbound, - Phase: pipeline.SessionRequest, - Invocations: pipeline.SnapshotInvocations(pctx.Extensions.Invocations, pipeline.InvocationPhaseRequest), - Plugins: plugins, - Identity: pipeline.SnapshotIdentity(pctx), - Host: pctx.Host, - } - if ev.Invocations != nil || plugins != nil { - s.Sessions.Append(sid, ev) - } - } + // (see handleRequest above). Shared with the transparent-redirect path. + s.recordTunnelOpened(pctx) - // Bidirectional copy. When either side closes, propagate the close - // to the other so both io.Copy goroutines exit. Close-on-each-side - // is idempotent on net.Conn. - go func() { - _, _ = io.Copy(upstream, clientConn) - _ = upstream.Close() - _ = clientConn.Close() - }() - _, _ = io.Copy(clientConn, upstream) - _ = clientConn.Close() - _ = upstream.Close() + // Bidirectional copy until either side closes. + tunnel(clientConn, upstream) } // writeSSEFrame writes one SSE event built from a sseframe-decoded diff --git a/authbridge/authlib/listener/forwardproxy/sniff.go b/authbridge/authlib/listener/forwardproxy/sniff.go new file mode 100644 index 000000000..73bf6687a --- /dev/null +++ b/authbridge/authlib/listener/forwardproxy/sniff.go @@ -0,0 +1,168 @@ +package forwardproxy + +import ( + "bufio" + "bytes" + cryptotls "crypto/tls" + "errors" + "io" + "net" + "net/http" + "time" +) + +// Captured (iptables-REDIRECTed) connections carry no CONNECT line, so the +// destination hostname must be recovered from the connection's own first bytes: +// the TLS ClientHello SNI for HTTPS, or the HTTP Host header for plaintext HTTP. +// This gives policy parity with the explicit-proxy path (which reads r.Host). +// The recovered name is used ONLY as the policy key (pctx.Host); the dial target +// stays the SO_ORIGINAL_DST IP. See HandleTransparentConn. + +const ( + // sniffBufSize bounds how much of the leading bytes we buffer to find the + // SNI / Host header. Real ClientHellos and request header blocks fit well + // within this; anything larger falls back to the IP. + sniffBufSize = 8192 + // sniffTimeout bounds the peek so a client that connects but sends nothing + // (or a server-first protocol) can't pin a goroutine. Only relevant on the + // sniffed ports, where the client speaks first, so it is rarely hit. + sniffTimeout = 5 * time.Second +) + +// errSniffDone aborts the throwaway TLS handshake once we have the SNI. +var errSniffDone = errors.New("forwardproxy: sni sniff complete") + +// shouldSniff reports whether dst's port is one where the client speaks first +// with an HTTP/TLS preamble we can parse. Gating on these ports avoids adding +// peek latency to non-HTTP, often server-first protocols (SSH:22, SMTP:25, ...) +// that we would only ever blind-tunnel anyway. +func shouldSniff(dst string) bool { + _, port, err := net.SplitHostPort(dst) + if err != nil { + return false + } + switch port { + case "80", "443", "8080", "8443": + return true + default: + return false + } +} + +// sniffHost peeks the leading bytes of conn to recover the destination hostname +// (TLS SNI or HTTP Host header) without consuming them: it returns the hostname +// (without port; "" if none could be recovered) and a net.Conn that replays the +// peeked bytes so the downstream tunnel still forwards the handshake/request +// verbatim. A read deadline bounds the peek and is cleared before returning. +func sniffHost(conn net.Conn) (string, net.Conn) { + br := bufio.NewReaderSize(conn, sniffBufSize) + wrapped := &peekedConn{Conn: conn, r: br} + + _ = conn.SetReadDeadline(time.Now().Add(sniffTimeout)) + defer func() { _ = conn.SetReadDeadline(time.Time{}) }() + + first, err := br.Peek(1) + if err != nil || len(first) == 0 { + return "", wrapped + } + switch { + case first[0] == 0x16: // TLS handshake record + return stripPort(sniffTLSSNI(br)), wrapped + case first[0] >= 'A' && first[0] <= 'Z': // ASCII HTTP method + return stripPort(sniffHTTPHost(br)), wrapped + default: + return "", wrapped + } +} + +// sniffTLSSNI peeks the first TLS record (the ClientHello) and extracts the SNI. +func sniffTLSSNI(br *bufio.Reader) string { + hdr, err := br.Peek(5) + if err != nil || len(hdr) < 5 { + return "" + } + end := 5 + (int(hdr[3])<<8 | int(hdr[4])) + if end > br.Size() { + end = br.Size() + } + full, _ := br.Peek(end) // best effort: parse whatever is buffered + return extractSNI(full) +} + +// extractSNI parses the SNI out of a buffered ClientHello by driving a throwaway +// server-side handshake over the bytes and capturing ServerName in the config +// callback, then aborting. Leans on crypto/tls's hardened parser rather than a +// hand-rolled one. Returns "" if the bytes are not a parseable ClientHello. +func extractSNI(clientHello []byte) string { + var sni string + _ = cryptotls.Server(readOnlyConn{r: bytes.NewReader(clientHello)}, &cryptotls.Config{ + GetConfigForClient: func(chi *cryptotls.ClientHelloInfo) (*cryptotls.Config, error) { + sni = chi.ServerName + return nil, errSniffDone + }, + }).Handshake() + return sni +} + +// sniffHTTPHost peeks the request header block and returns the Host header. +func sniffHTTPHost(br *bufio.Reader) string { + end := br.Buffered() + if end < 1 { + end = 1 + } + for { + buf, err := br.Peek(end) + if i := bytes.Index(buf, []byte("\r\n\r\n")); i >= 0 { + return parseHTTPHost(buf[:i+4]) + } + if err != nil || end >= br.Size() { + return parseHTTPHost(buf) // best effort on what we have + } + end++ + } +} + +func parseHTTPHost(headerBytes []byte) string { + req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(headerBytes))) + if err != nil { + return "" + } + return req.Host +} + +// stripPort drops a trailing :port if present (HTTP Host headers may carry one; +// SNI never does). The real port comes from the SO_ORIGINAL_DST destination. +func stripPort(host string) string { + if host == "" { + return "" + } + if h, _, err := net.SplitHostPort(host); err == nil { + return h + } + return host +} + +// peekedConn is a net.Conn whose Read replays bytes buffered by a bufio.Reader +// during sniffing, then continues from the underlying conn. Writes, Close, +// deadlines, and addresses delegate to the embedded conn. +type peekedConn struct { + net.Conn + r *bufio.Reader +} + +func (c *peekedConn) Read(p []byte) (int, error) { return c.r.Read(p) } + +// readOnlyConn adapts a byte buffer to net.Conn for crypto/tls's server-side +// parser. Reads come from the buffer; writes are discarded (the throwaway +// handshake never needs to send), and the parser aborts via errSniffDone before +// any write would matter. +type readOnlyConn struct{ r io.Reader } + +func (c readOnlyConn) Read(p []byte) (int, error) { return c.r.Read(p) } +func (c readOnlyConn) Write(p []byte) (int, error) { return len(p), nil } +func (c readOnlyConn) Close() error { return nil } +func (c readOnlyConn) LocalAddr() net.Addr { return nil } +func (c readOnlyConn) RemoteAddr() net.Addr { return nil } +func (c readOnlyConn) SetDeadline(time.Time) error { return nil } +func (c readOnlyConn) SetReadDeadline(time.Time) error { return nil } +func (c readOnlyConn) SetWriteDeadline(time.Time) error { return nil } diff --git a/authbridge/authlib/listener/forwardproxy/sniff_test.go b/authbridge/authlib/listener/forwardproxy/sniff_test.go new file mode 100644 index 000000000..abd479ad4 --- /dev/null +++ b/authbridge/authlib/listener/forwardproxy/sniff_test.go @@ -0,0 +1,144 @@ +package forwardproxy + +import ( + cryptotls "crypto/tls" + "io" + "net" + "testing" + "time" +) + +// readAll drains a conn until EOF/close, returning what was read. Used to prove +// the sniffer replays the peeked bytes verbatim into the tunnel. +func readAll(t *testing.T, c net.Conn) []byte { + t.Helper() + _ = c.SetReadDeadline(time.Now().Add(2 * time.Second)) + b, _ := io.ReadAll(c) + return b +} + +// sniffServerSide accepts one connection on a fresh loopback listener, runs the +// client closure against its address, and returns sniffHost's result plus the +// bytes the wrapped conn yields (to verify replay). +func sniffServerSide(t *testing.T, client func(addr string)) (string, []byte) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer func() { _ = ln.Close() }() + + type result struct { + host string + replay []byte + } + ch := make(chan result, 1) + go func() { + conn, err := ln.Accept() + if err != nil { + ch <- result{} + return + } + host, wrapped := sniffHost(conn) + ch <- result{host: host, replay: readAll(t, wrapped)} + }() + + client(ln.Addr().String()) + + select { + case r := <-ch: + return r.host, r.replay + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for sniff result") + return "", nil + } +} + +// A real TLS ClientHello (generated by crypto/tls) with SNI must yield the SNI, +// and the handshake bytes must still be replayed for the tunnel. +func TestSniffHost_TLS_SNI(t *testing.T) { + host, replay := sniffServerSide(t, func(addr string) { + c, err := net.Dial("tcp", addr) + if err != nil { + t.Errorf("dial: %v", err) + return + } + // Drive a real ClientHello with ServerName set. The handshake won't + // complete (server side just sniffs and closes); we only need the + // ClientHello on the wire. + tlsConn := cryptotls.Client(c, &cryptotls.Config{ + ServerName: "api.openai.com", + InsecureSkipVerify: true, //nolint:gosec // test-only; no real verification + }) + _ = tlsConn.SetDeadline(time.Now().Add(1 * time.Second)) + _ = tlsConn.Handshake() // expected to fail; ClientHello is what matters + _ = c.Close() + }) + + if host != "api.openai.com" { + t.Errorf("SNI host = %q, want api.openai.com", host) + } + if len(replay) == 0 || replay[0] != 0x16 { + t.Errorf("replay did not start with a TLS record (0x16); got %d bytes", len(replay)) + } +} + +// A plaintext HTTP request must yield the Host header, port stripped, and the +// full request must be replayed. +func TestSniffHost_HTTP_Host(t *testing.T) { + req := "GET /v1/models HTTP/1.1\r\nHost: api.example.org:8080\r\nUser-Agent: x\r\n\r\n" + host, replay := sniffServerSide(t, func(addr string) { + c, err := net.Dial("tcp", addr) + if err != nil { + t.Errorf("dial: %v", err) + return + } + _, _ = c.Write([]byte(req)) + _ = c.Close() + }) + + if host != "api.example.org" { + t.Errorf("HTTP host = %q, want api.example.org (port stripped)", host) + } + if string(replay) != req { + t.Errorf("replay = %q, want the original request verbatim", replay) + } +} + +// Non-HTTP, non-TLS bytes yield no hostname but are still replayed verbatim. +func TestSniffHost_Opaque(t *testing.T) { + payload := []byte{0x00, 0x01, 0x02, 0x03, 0xff} + host, replay := sniffServerSide(t, func(addr string) { + c, err := net.Dial("tcp", addr) + if err != nil { + t.Errorf("dial: %v", err) + return + } + _, _ = c.Write(payload) + _ = c.Close() + }) + + if host != "" { + t.Errorf("opaque host = %q, want empty", host) + } + if string(replay) != string(payload) { + t.Errorf("replay = %v, want %v", replay, payload) + } +} + +func TestShouldSniff(t *testing.T) { + cases := map[string]bool{ + "1.2.3.4:443": true, + "1.2.3.4:80": true, + "1.2.3.4:8443": true, + "1.2.3.4:22": false, + "1.2.3.4:5432": false, + "[::1]:443": true, + "garbage": false, + } + for in, want := range cases { + if got := shouldSniff(in); got != want { + t.Errorf("shouldSniff(%q) = %v, want %v", in, got, want) + } + } +} diff --git a/authbridge/authlib/listener/forwardproxy/transparent.go b/authbridge/authlib/listener/forwardproxy/transparent.go new file mode 100644 index 000000000..5a43d0d93 --- /dev/null +++ b/authbridge/authlib/listener/forwardproxy/transparent.go @@ -0,0 +1,158 @@ +package forwardproxy + +import ( + "context" + "io" + "log/slog" + "net" + "net/http" + "time" + + "github.com/kagenti/kagenti-extensions/authbridge/authlib/pipeline" + "github.com/kagenti/kagenti-extensions/authbridge/authlib/session" +) + +// HandleTransparentConn processes one outbound connection captured by an +// iptables REDIRECT (proxy-sidecar enforce-redirect mode). It is the +// transparent-listener analogue of handleConnect, and shares its semantics: +// the same outbound pipeline gates the connection on destination/identity, and +// the bytes are then blind-tunnelled, preserving the agent's end-to-end TLS +// (token-exchange and protocol parsers are no-ops on opaque TLS, exactly as on +// the CONNECT path). +// +// The crucial difference from handleConnect: there is NO HTTP CONNECT request. +// The agent believes it is talking directly to dst, so the proxy must emit no +// protocol bytes back — no "200 Connection Established", no hijack. It simply +// gates, dials dst, and copies bytes both ways. dst is "host:port" recovered +// from SO_ORIGINAL_DST by the transparent listener. +// +// HOSTNAME RECOVERY: CONNECT carries a hostname in r.Host, but SO_ORIGINAL_DST +// yields only an IP:port. To give host/domain egress policy parity with the +// CONNECT path, we sniff the connection's first bytes for the destination name +// — the TLS ClientHello SNI for HTTPS, or the HTTP Host header for plaintext +// HTTP — and use it as pctx.Host. If neither can be recovered we fall back to +// the IP. The dial target ALWAYS stays the SO_ORIGINAL_DST IP (dst); the name is +// only the policy key. +// +// Trust caveat (relevant before enforce-redirect goes always-on): for captured +// traffic the agent controls both the SNI/Host and, separately, the IP the +// bytes actually go to, so a *malicious* agent could present an allowed name +// while connecting to another IP. Name-based policy here is therefore reliable +// against a cooperative/misconfigured agent (the motivating case) but is not a +// hard control against a hostile one — only the IP is ground truth. Hard +// enforcement would need IP-set allowlists or SNI/cert cross-checks. +// +// HandleTransparentConn owns clientConn's lifecycle and always closes it. +func (s *Server) HandleTransparentConn(clientConn net.Conn, dst string) { + defer func() { _ = clientConn.Close() }() + + // Keepalive on the raw client conn before sniffing wraps it (the wrapper is + // not a *net.TCPConn, so enableKeepalive would no-op on it). + enableKeepalive(clientConn) + + // Recover the destination hostname for policy parity with CONNECT. Gated to + // HTTP/TLS ports so non-HTTP protocols are not delayed by the peek. The dial + // target stays dst (the IP); only pctx.Host gets the recovered name. + host := dst + if shouldSniff(dst) { + name, wrapped := sniffHost(clientConn) + clientConn = wrapped + if name != "" { + if _, port, err := net.SplitHostPort(dst); err == nil { + host = net.JoinHostPort(name, port) + } + slog.Debug("transparent-proxy: recovered destination host for policy", + "host", name, "dst", dst) + } + } + + // Background context: there is no inbound *http.Request to tie cancellation + // to. Tunnel teardown (either side closing) is what ends the connection; + // the pipeline Run/Finish calls are short and don't need request scoping. + ctx := context.Background() + + pctx := &pipeline.Context{ + Direction: pipeline.Outbound, + Method: http.MethodConnect, // synthetic: opaque tunnel, parity with handleConnect + Scheme: "tcp", // marker: bytes are opaque, not HTTP + Host: host, + Headers: http.Header{}, + Shared: s.Shared, + StartedAt: time.Now(), + } + defer func() { + s.OutboundPipeline.RunFinish(ctx, pctx, pipeline.OutcomeFromContext(pctx)) + }() + + if s.Sessions != nil { + if aid := s.Sessions.ActiveSession(); aid != "" { + pctx.Session = s.Sessions.View(aid) + } + } + + // Gate on host/identity before opening the tunnel — identical to the + // CONNECT path. Parsers see no body and degrade gracefully. + action := s.OutboundPipeline.Run(ctx, pctx) + if action.Type == pipeline.Reject { + s.recordOutboundReject(pctx, action) + slog.Warn("transparent-proxy: outbound rejected by policy", "host", host) + return + } + + // Always dial the original IP (dst), never the sniffed name — the agent + // already chose the IP, and re-resolving the name could diverge from it. + upstream, err := net.DialTimeout("tcp", dst, connectDialTimeout) + if err != nil { + slog.Warn("transparent-proxy: upstream dial failed", "host", host, "dst", dst, "error", err) + return + } + defer func() { _ = upstream.Close() }() + + enableKeepalive(upstream) + + s.recordTunnelOpened(pctx) + tunnel(clientConn, upstream) +} + +// recordTunnelOpened emits the SessionRequest event for an opened opaque +// tunnel (CONNECT or transparent-redirect). Shared by handleConnect and +// HandleTransparentConn. MCP/Inference snapshots are nil by definition (the +// bytes are opaque); Invocations from gate plugins and plugin-public Plugins +// entries are still meaningful. +func (s *Server) recordTunnelOpened(pctx *pipeline.Context) { + if s.Sessions == nil { + return + } + sid := s.Sessions.ActiveSession() + if sid == "" { + sid = session.DefaultSessionID + } + plugins := pipeline.SnapshotPlugins(pctx.Extensions.Custom) + ev := pipeline.SessionEvent{ + At: time.Now(), + Direction: pipeline.Outbound, + Phase: pipeline.SessionRequest, + Invocations: pipeline.SnapshotInvocations(pctx.Extensions.Invocations, pipeline.InvocationPhaseRequest), + Plugins: plugins, + Identity: pipeline.SnapshotIdentity(pctx), + Host: pctx.Host, + } + if ev.Invocations != nil || plugins != nil { + s.Sessions.Append(sid, ev) + } +} + +// tunnel bidirectionally copies between two connections until either side +// closes, then propagates the close to the other so both io.Copy goroutines +// exit. Close-on-each-side is idempotent on net.Conn. Shared by handleConnect +// and HandleTransparentConn. +func tunnel(a, b net.Conn) { + go func() { + _, _ = io.Copy(b, a) + _ = b.Close() + _ = a.Close() + }() + _, _ = io.Copy(a, b) + _ = a.Close() + _ = b.Close() +} diff --git a/authbridge/authlib/listener/forwardproxy/transparent_test.go b/authbridge/authlib/listener/forwardproxy/transparent_test.go new file mode 100644 index 000000000..0933b39d0 --- /dev/null +++ b/authbridge/authlib/listener/forwardproxy/transparent_test.go @@ -0,0 +1,70 @@ +package forwardproxy + +import ( + "io" + "net" + "testing" + "time" + + "github.com/kagenti/kagenti-extensions/authbridge/authlib/pipeline" + "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/plugintesting" +) + +// HandleTransparentConn gates then blind-tunnels: with an allow-all pipeline it +// must dial the recovered destination and copy bytes both ways, emitting no +// proxy-protocol bytes of its own (the agent thinks it's talking to dst). +func TestHandleTransparentConn_Tunnels(t *testing.T) { + const banner = "UPSTREAM-HELLO\n" + + upstream, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen upstream: %v", err) + } + defer func() { _ = upstream.Close() }() + go func() { + c, err := upstream.Accept() + if err != nil { + return + } + defer func() { _ = c.Close() }() + _, _ = c.Write([]byte(banner)) + buf := make([]byte, 4) + if _, err := io.ReadFull(c, buf); err != nil { + return + } + _, _ = c.Write(buf) // echo + }() + + p, err := plugintesting.BuildPipeline(nil) // allow-all + if err != nil { + t.Fatalf("build pipeline: %v", err) + } + srv := &Server{OutboundPipeline: pipeline.NewHolder(p)} + + agentSide, proxySide := net.Pipe() + go srv.HandleTransparentConn(proxySide, upstream.Addr().String()) + + _ = agentSide.SetDeadline(time.Now().Add(5 * time.Second)) + + // The first bytes the agent sees must be the upstream's banner, NOT a + // "200 Connection Established" — proving no proxy protocol leaked. + got := make([]byte, len(banner)) + if _, err := io.ReadFull(agentSide, got); err != nil { + t.Fatalf("read banner: %v", err) + } + if string(got) != banner { + t.Fatalf("first bytes = %q, want upstream banner %q", got, banner) + } + + if _, err := agentSide.Write([]byte("ping")); err != nil { + t.Fatalf("write: %v", err) + } + echo := make([]byte, 4) + if _, err := io.ReadFull(agentSide, echo); err != nil { + t.Fatalf("read echo: %v", err) + } + if string(echo) != "ping" { + t.Fatalf("echo = %q, want %q", echo, "ping") + } + _ = agentSide.Close() +} diff --git a/authbridge/authlib/listener/transparentproxy/origdst_linux.go b/authbridge/authlib/listener/transparentproxy/origdst_linux.go new file mode 100644 index 000000000..2ea3cf55b --- /dev/null +++ b/authbridge/authlib/listener/transparentproxy/origdst_linux.go @@ -0,0 +1,78 @@ +//go:build linux + +package transparentproxy + +import ( + "fmt" + "net" + "unsafe" + + "golang.org/x/sys/unix" +) + +// soOriginalDst is netfilter's SO_ORIGINAL_DST (IPv4, SOL_IP) and +// IP6T_SO_ORIGINAL_DST (IPv6, SOL_IPV6) option number. After an iptables +// REDIRECT/DNAT, conntrack records the pre-NAT destination and exposes it +// to the receiving socket via getsockopt with this option. +const soOriginalDst = 80 + +// originalDst recovers the pre-REDIRECT destination of a connection accepted on +// a transparent (iptables-REDIRECTed) listener, via the SO_ORIGINAL_DST socket +// option. It tries IPv4 (SOL_IP) first, then IPv6 (SOL_IPV6). Returns the +// destination as "host:port". +// +// This relies on REDIRECT/DNAT (conntrack-backed) — the same mechanism Envoy's +// original_dst listener filter uses — NOT TPROXY (which would instead read the +// socket's local address). +func originalDst(conn *net.TCPConn) (string, error) { + raw, err := conn.SyscallConn() + if err != nil { + return "", fmt.Errorf("transparentproxy: SyscallConn: %w", err) + } + + var ( + dst string + innerErr error + ) + ctrlErr := raw.Control(func(fd uintptr) { + dst, innerErr = getOrigDst(fd) + }) + if ctrlErr != nil { + return "", fmt.Errorf("transparentproxy: RawConn.Control: %w", ctrlErr) + } + return dst, innerErr +} + +// getOrigDst performs the raw getsockopt for SO_ORIGINAL_DST on fd. The buffer +// is sized for sockaddr_in6 (the larger of the two); parseSockaddr reads only +// the bytes the kernel reports via the (in/out) optlen. +func getOrigDst(fd uintptr) (string, error) { + var buf [unix.SizeofSockaddrInet6]byte + size := uint32(len(buf)) + + // IPv4 (SOL_IP) first — the common case in kagenti clusters. + if errno := getsockopt(fd, unix.SOL_IP, soOriginalDst, &buf[0], &size); errno == 0 { + return parseSockaddr(buf[:size]) + } + + // IPv6 (SOL_IPV6). + size = uint32(len(buf)) + if errno := getsockopt(fd, unix.SOL_IPV6, soOriginalDst, &buf[0], &size); errno == 0 { + return parseSockaddr(buf[:size]) + } + + return "", fmt.Errorf("transparentproxy: getsockopt SO_ORIGINAL_DST failed (not a REDIRECTed connection?)") +} + +func getsockopt(fd uintptr, level, name int, val *byte, length *uint32) unix.Errno { + _, _, errno := unix.Syscall6( + unix.SYS_GETSOCKOPT, + fd, + uintptr(level), + uintptr(name), + uintptr(unsafe.Pointer(val)), + uintptr(unsafe.Pointer(length)), + 0, + ) + return errno +} diff --git a/authbridge/authlib/listener/transparentproxy/origdst_other.go b/authbridge/authlib/listener/transparentproxy/origdst_other.go new file mode 100644 index 000000000..edf1efbd7 --- /dev/null +++ b/authbridge/authlib/listener/transparentproxy/origdst_other.go @@ -0,0 +1,15 @@ +//go:build !linux + +package transparentproxy + +import ( + "fmt" + "net" +) + +// originalDst is unsupported off Linux. SO_ORIGINAL_DST is a netfilter feature; +// the transparent listener only runs in-cluster (Linux). This stub keeps the +// proxy binary buildable on dev hosts (e.g. macOS). +func originalDst(_ *net.TCPConn) (string, error) { + return "", fmt.Errorf("transparentproxy: SO_ORIGINAL_DST is only supported on Linux") +} diff --git a/authbridge/authlib/listener/transparentproxy/server.go b/authbridge/authlib/listener/transparentproxy/server.go new file mode 100644 index 000000000..2d75dc44b --- /dev/null +++ b/authbridge/authlib/listener/transparentproxy/server.go @@ -0,0 +1,101 @@ +// Package transparentproxy implements an outbound transparent proxy listener +// for proxy-sidecar enforce-redirect mode. Unlike the forward proxy (which +// requires the agent to honor HTTP_PROXY and speak explicit CONNECT), this +// listener receives connections that iptables transparently REDIRECTed to it. +// The agent believes it is connecting directly to the destination, so the +// listener recovers the original destination from the kernel via +// SO_ORIGINAL_DST and hands the connection to a ConnHandler that gates and +// blind-tunnels it — emitting no proxy-protocol bytes back to the agent. +// +// This is the Go equivalent of Envoy's original_dst listener filter + +// ORIGINAL_DST cluster used by envoy-sidecar mode; the auth pipeline behind +// the ConnHandler is identical to the forward proxy's CONNECT path. +package transparentproxy + +import ( + "errors" + "log/slog" + "net" +) + +// ConnHandler processes one accepted outbound connection whose original +// destination has been recovered. dst is "host:port". The handler owns the +// connection's lifecycle, including closing it. +type ConnHandler func(conn net.Conn, dst string) + +// Server accepts iptables-REDIRECTed connections and dispatches them to a +// ConnHandler after recovering each connection's original destination. +type Server struct { + handle ConnHandler +} + +// NewServer returns a transparent proxy server that dispatches each accepted, +// destination-recovered connection to handle. In proxy-sidecar mode handle is +// forwardproxy.Server.HandleTransparentConn, so transparent and explicit-proxy +// egress share one auth pipeline. +func NewServer(handle ConnHandler) *Server { + if handle == nil { + // Defensive: a nil handler would panic at dispatch and take down the + // process. Fall back to closing the connection so a misconfiguration + // degrades to "no capture" rather than a crash. + handle = func(conn net.Conn, _ string) { + slog.Error("transparent-proxy: nil connection handler; closing connection", + "remote", conn.RemoteAddr().String()) + _ = conn.Close() + } + } + return &Server{handle: handle} +} + +// Serve accepts connections on ln until it is closed, recovering each +// connection's original destination and dispatching to the handler in its own +// goroutine. Returns nil when ln is closed (graceful shutdown), or the accept +// error otherwise. +func (s *Server) Serve(ln *net.TCPListener) error { + for { + conn, err := ln.AcceptTCP() + if err != nil { + if errors.Is(err, net.ErrClosed) { + return nil + } + return err + } + go s.dispatch(conn) + } +} + +func (s *Server) dispatch(conn *net.TCPConn) { + dst, err := originalDst(conn) + if err != nil { + // No recoverable original destination means this connection did not + // arrive via the REDIRECT (e.g. a direct dial to the listener port). + // Drop it rather than guess a destination — we will not blind-tunnel + // to an attacker-chosen target. + slog.Warn("transparent-proxy: dropping connection with no original destination", + "remote", conn.RemoteAddr().String(), "error", err) + _ = conn.Close() + return + } + // Defense-in-depth against a self-redirect loop: a genuinely REDIRECTed + // connection's original destination is always some external host — never + // this listener itself. Two ways a connection could point back at us: + // - a loopback dst (a direct dial to 127.0.0.1:); the enforce-redirect + // rules RETURN loopback before the REDIRECT, so a real capture never has one; + // - the listener's own address (e.g. a podIP: self-dial that slipped + // past the iptables CLUSTER_CIDRS RETURN under a misconfigured CIDR set). + // Tunnelling either would spiral into ever more connections/goroutines. The + // iptables layer is the primary control; this is belt-and-suspenders. Drop it. + selfLoop := dst == conn.LocalAddr().String() + if host, _, splitErr := net.SplitHostPort(dst); !selfLoop && splitErr == nil { + if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { + selfLoop = true + } + } + if selfLoop { + slog.Warn("transparent-proxy: dropping self-referential connection (would self-loop)", + "remote", conn.RemoteAddr().String(), "dst", dst, "local", conn.LocalAddr().String()) + _ = conn.Close() + return + } + s.handle(conn, dst) +} diff --git a/authbridge/authlib/listener/transparentproxy/sockaddr.go b/authbridge/authlib/listener/transparentproxy/sockaddr.go new file mode 100644 index 000000000..a0ff43312 --- /dev/null +++ b/authbridge/authlib/listener/transparentproxy/sockaddr.go @@ -0,0 +1,53 @@ +package transparentproxy + +import ( + "encoding/binary" + "fmt" + "net" + "strconv" +) + +// Linux address-family values. These are hardcoded rather than taken from +// golang.org/x/sys/unix because the sockaddr we parse always originates from +// a Linux kernel getsockopt (SO_ORIGINAL_DST), even when this code is compiled +// for a non-Linux dev host where AF_INET6 has a different numeric value +// (Darwin uses 30). Keeping them local makes parseSockaddr pure and lets its +// test run on any platform. +const ( + afInet = 2 // AF_INET + afInet6 = 10 // AF_INET6 on Linux +) + +// parseSockaddr decodes a Linux sockaddr_in / sockaddr_in6 (as returned by the +// SO_ORIGINAL_DST getsockopt) into a "host:port" string. The buffer layout is: +// +// sockaddr_in : family[0:2] port[2:4](BE) addr[4:8] +// sockaddr_in6 : family[0:2] port[2:4](BE) flowinfo[4:8] addr[8:24] scope[24:28] +// +// sa_family is in host byte order; the port is network byte order (big-endian). +// It is deliberately syscall-free so it is unit-testable on any OS. +func parseSockaddr(b []byte) (string, error) { + if len(b) < 4 { + return "", fmt.Errorf("transparentproxy: short sockaddr (%d bytes)", len(b)) + } + family := binary.NativeEndian.Uint16(b[0:2]) + port := int(binary.BigEndian.Uint16(b[2:4])) + + switch family { + case afInet: + if len(b) < 8 { + return "", fmt.Errorf("transparentproxy: short sockaddr_in (%d bytes)", len(b)) + } + ip := net.IPv4(b[4], b[5], b[6], b[7]) + return net.JoinHostPort(ip.String(), strconv.Itoa(port)), nil + case afInet6: + if len(b) < 24 { + return "", fmt.Errorf("transparentproxy: short sockaddr_in6 (%d bytes)", len(b)) + } + ip := make(net.IP, net.IPv6len) + copy(ip, b[8:24]) + return net.JoinHostPort(ip.String(), strconv.Itoa(port)), nil + default: + return "", fmt.Errorf("transparentproxy: unexpected sockaddr family %d", family) + } +} diff --git a/authbridge/authlib/listener/transparentproxy/sockaddr_test.go b/authbridge/authlib/listener/transparentproxy/sockaddr_test.go new file mode 100644 index 000000000..4e268ea16 --- /dev/null +++ b/authbridge/authlib/listener/transparentproxy/sockaddr_test.go @@ -0,0 +1,60 @@ +package transparentproxy + +import ( + "encoding/binary" + "testing" +) + +// makeSockaddrIn builds a Linux sockaddr_in (family=2) for ip a.b.c.d : port. +func makeSockaddrIn(a, b, c, d byte, port uint16) []byte { + buf := make([]byte, 16) // sizeof(sockaddr_in) + binary.NativeEndian.PutUint16(buf[0:2], afInet) + binary.BigEndian.PutUint16(buf[2:4], port) + buf[4], buf[5], buf[6], buf[7] = a, b, c, d + return buf +} + +// makeSockaddrIn6 builds a Linux sockaddr_in6 (family=10) for the given 16-byte +// address and port. +func makeSockaddrIn6(addr [16]byte, port uint16) []byte { + buf := make([]byte, 28) // sizeof(sockaddr_in6) + binary.NativeEndian.PutUint16(buf[0:2], afInet6) + binary.BigEndian.PutUint16(buf[2:4], port) + copy(buf[8:24], addr[:]) + return buf +} + +func TestParseSockaddr(t *testing.T) { + v6loop := [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} // ::1 + + tests := []struct { + name string + in []byte + want string + wantErr bool + }{ + {"ipv4 https", makeSockaddrIn(10, 96, 0, 10, 443), "10.96.0.10:443", false}, + {"ipv4 high port", makeSockaddrIn(192, 168, 1, 5, 65000), "192.168.1.5:65000", false}, + {"ipv6 loopback", makeSockaddrIn6(v6loop, 8080), "[::1]:8080", false}, + {"too short", []byte{2, 0}, "", true}, + {"short sockaddr_in", []byte{2, 0, 1, 187, 10, 96}, "", true}, + {"unknown family", []byte{0xFF, 0xFF, 1, 187, 10, 96, 0, 10}, "", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseSockaddr(tt.in) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got %q", got) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("parseSockaddr = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/authbridge/cmd/authbridge-lite/main.go b/authbridge/cmd/authbridge-lite/main.go index 83226f552..abde9d5ed 100644 --- a/authbridge/cmd/authbridge-lite/main.go +++ b/authbridge/cmd/authbridge-lite/main.go @@ -15,6 +15,7 @@ import ( "fmt" "log" "log/slog" + "net" "net/http" "os" "os/signal" @@ -38,6 +39,7 @@ import ( // (no gRPC, no envoy types). "github.com/kagenti/kagenti-extensions/authbridge/authlib/listener/forwardproxy" "github.com/kagenti/kagenti-extensions/authbridge/authlib/listener/reverseproxy" + "github.com/kagenti/kagenti-extensions/authbridge/authlib/listener/transparentproxy" // Auth gates only: drop the parsers and token-broker. _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/jwtvalidation" @@ -245,6 +247,11 @@ func main() { fpSrv.Shared = sharedStore httpServers = append(httpServers, startReverseProxyServer("reverse-proxy", rpSrv, cfg.Listener.ReverseProxyAddr)) httpServers = append(httpServers, startHTTPServer("forward-proxy", fpSrv.Handler(), cfg.Listener.ForwardProxyAddr)) + + // Outbound transparent listener (enforce-redirect mode); shares the forward + // proxy's outbound pipeline. Closed explicitly on shutdown. + transparentLn := startTransparentProxy(fpSrv, cfg.Listener.TransparentProxyAddr) + _ = mtlsMetrics // TODO Phase 2: surface metrics through /stats statsProvider := func() *auth.Stats { @@ -311,6 +318,9 @@ func main() { for _, srv := range httpServers { srv.Shutdown(shutdownCtx) } + if transparentLn != nil { + _ = transparentLn.Close() + } statSrv.Shutdown(shutdownCtx) if sessionAPISrv != nil { sessionAPISrv.Shutdown(shutdownCtx) @@ -339,6 +349,33 @@ func startHTTPServer(name string, handler http.Handler, addr string) *http.Serve return srv } +// startTransparentProxy binds the outbound transparent listener (enforce-redirect +// mode) and serves it in a goroutine, dispatching each REDIRECTed connection +// through the forward proxy's outbound pipeline. Returns the listener for +// shutdown, or nil when addr is empty. Bind failures are fatal — enforce-redirect +// iptables would otherwise REDIRECT to a dead port and break all egress silently. +func startTransparentProxy(fp *forwardproxy.Server, addr string) *net.TCPListener { + if addr == "" { + return nil + } + la, err := net.ResolveTCPAddr("tcp", addr) + if err != nil { + log.Fatalf("resolve transparent-proxy addr %q: %v", addr, err) + } + ln, err := net.ListenTCP("tcp", la) + if err != nil { + log.Fatalf("transparent-proxy listen on %q: %v", addr, err) + } + srv := transparentproxy.NewServer(fp.HandleTransparentConn) + go func() { + slog.Info("transparent proxy listening", "addr", addr) + if err := srv.Serve(ln); err != nil { + log.Fatalf("transparent-proxy serve: %v", err) + } + }() + return ln +} + // startReverseProxyServer mirrors startHTTPServer but routes through // reverseproxy.Server.Listen() so the byte-peek TLS-sniffing listener // is wired in when mTLS is enabled. Same shape as the equivalent diff --git a/authbridge/cmd/authbridge-proxy/main.go b/authbridge/cmd/authbridge-proxy/main.go index 2ad6d4811..418a75b5a 100644 --- a/authbridge/cmd/authbridge-proxy/main.go +++ b/authbridge/cmd/authbridge-proxy/main.go @@ -15,6 +15,7 @@ import ( "fmt" "log" "log/slog" + "net" "net/http" "os" "os/signal" @@ -38,6 +39,7 @@ import ( // (no gRPC, no envoy types). "github.com/kagenti/kagenti-extensions/authbridge/authlib/listener/forwardproxy" "github.com/kagenti/kagenti-extensions/authbridge/authlib/listener/reverseproxy" + "github.com/kagenti/kagenti-extensions/authbridge/authlib/listener/transparentproxy" // Plugins. Auth gates first, then the protocol parsers that // supply session-event context for abctl. @@ -260,6 +262,13 @@ func main() { fpSrv.Shared = sharedStore httpServers = append(httpServers, startReverseProxyServer("reverse-proxy", rpSrv, cfg.Listener.ReverseProxyAddr)) httpServers = append(httpServers, startHTTPServer("forward-proxy", fpSrv.Handler(), cfg.Listener.ForwardProxyAddr)) + + // Outbound transparent listener (enforce-redirect mode). It shares the + // forward proxy's outbound pipeline via HandleTransparentConn, so explicit + // HTTP_PROXY egress and iptables-REDIRECTed bypass egress are gated and + // tunnelled identically. Closed explicitly on shutdown (not an *http.Server). + transparentLn := startTransparentProxy(fpSrv, cfg.Listener.TransparentProxyAddr) + _ = mtlsMetrics // TODO Phase 2: surface metrics through /stats statsProvider := func() *auth.Stats { @@ -326,6 +335,9 @@ func main() { for _, srv := range httpServers { srv.Shutdown(shutdownCtx) } + if transparentLn != nil { + _ = transparentLn.Close() + } statSrv.Shutdown(shutdownCtx) if sessionAPISrv != nil { sessionAPISrv.Shutdown(shutdownCtx) @@ -381,6 +393,34 @@ func startReverseProxyServer(name string, rp *reverseproxy.Server, addr string) return srv } +// startTransparentProxy binds the outbound transparent listener and serves it +// in a goroutine, dispatching each REDIRECTed connection through the forward +// proxy's outbound pipeline. Returns the listener (for shutdown), or nil when +// addr is empty (transparent capture disabled). Bind failures are fatal — +// enforce-redirect iptables would otherwise REDIRECT to a dead port and break +// all egress silently. +func startTransparentProxy(fp *forwardproxy.Server, addr string) *net.TCPListener { + if addr == "" { + return nil + } + la, err := net.ResolveTCPAddr("tcp", addr) + if err != nil { + log.Fatalf("resolve transparent-proxy addr %q: %v", addr, err) + } + ln, err := net.ListenTCP("tcp", la) + if err != nil { + log.Fatalf("transparent-proxy listen on %q: %v", addr, err) + } + srv := transparentproxy.NewServer(fp.HandleTransparentConn) + go func() { + slog.Info("transparent proxy listening", "addr", addr) + if err := srv.Serve(ln); err != nil { + log.Fatalf("transparent-proxy serve: %v", err) + } + }() + return ln +} + func startStatServer(cfg *config.Config, cfgProvider observe.ConfigProvider, statsProvider observe.StatsProvider, reloadStatus http.Handler) *observe.StatServer { srv := observe.NewStatServer(cfg.Stats.StatsAddress, cfgProvider, statsProvider, observe.WithReloadStatus(reloadStatus)) diff --git a/authbridge/proxy-init/README.md b/authbridge/proxy-init/README.md index 0bb03d7c9..d4340d58f 100644 --- a/authbridge/proxy-init/README.md +++ b/authbridge/proxy-init/README.md @@ -8,7 +8,7 @@ env var: | `MODE` | Used by | What it does | |---|---|---| | `redirect` (default) | `envoy-sidecar` | Transparently **REDIRECT**s pod traffic to the Envoy listeners. | -| `enforce-drop` | `proxy-sidecar` | Fail-closed egress guard — **DROP**s any egress that bypasses the forward proxy. | +| `enforce-redirect` | `proxy-sidecar`, `lite` | Fail-closed egress guard that **captures**: REDIRECTs external TCP that bypasses the forward proxy to AuthBridge's transparent listener; DROPs non-TCP external egress. | ## `redirect` mode (envoy-sidecar) @@ -28,27 +28,42 @@ env var: `INBOUND_PORTS_EXCLUDE` env vars (commonly used to exclude Keycloak's port 8080 to avoid token-exchange loops). -## `enforce-drop` mode (proxy-sidecar) +## `enforce-redirect` mode (proxy-sidecar) In `proxy-sidecar` mode the workload is configured with `HTTP_PROXY` pointing at AuthBridge's forward proxy. On its own that is purely cooperative — an app that ignores `HTTP_PROXY` (or sets `NO_PROXY`) -egresses directly and bypasses AuthBridge. `enforce-drop` closes that -gap **without** transparently redirecting (you cannot REDIRECT raw -traffic into a CONNECT forward proxy): it installs a fail-closed guard -that DROPs any direct egress, forcing all external traffic through the -proxy regardless of whether the app honors `HTTP_PROXY`. - -`init-iptables.sh` builds a dedicated `AB_EGRESS` chain hooked from -**`mangle` OUTPUT at position 1**, with this order: - -1. `RETURN` ztunnel's own sockets (fwmark `0x539`) — keeps the mesh path working; a no-op when ambient is absent. -2. `RETURN` the proxy's own re-originated egress (`--uid-owner $PROXY_UID`, default 1337). -3. `RETURN` loopback (the app → proxy hop) and in-cluster CIDRs (`CLUSTER_CIDRS`, mesh/DNS). -4. `DROP` everything else — direct external egress, including UDP (QUIC/HTTP-3). - -An IPv6 mirror drops external v6 egress (allowing loopback, link-local, -the proxy UID, and `CLUSTER_CIDRS6`). +egresses directly and bypasses AuthBridge. `enforce-redirect` closes +that gap **by capturing** the bypass traffic instead of dropping it: +external TCP that did not go through the forward proxy is transparently +REDIRECTed to AuthBridge's **transparent listener** (`TRANSPARENT_PORT`, +default 8082), which recovers the original destination via +`SO_ORIGINAL_DST` and tunnels it through the same outbound pipeline. +Because nothing is dropped, agents that ignore `HTTP_PROXY` keep working +— which is what lets enforcement be always-on. + +`init-iptables.sh` installs **two** chains, because `REDIRECT` is a +nat-table target but the nat table forbids `DROP` (`iptables` errors with +"the use of DROP is therefore inhibited"): + +- **`nat` OUTPUT / `AB_REDIRECT`** (position 1): `RETURN` ztunnel mark + `0x539`, the proxy UID (`--uid-owner $PROXY_UID`, avoids the loop), + loopback, and `CLUSTER_CIDRS`; then `REDIRECT` external **TCP** to + `TRANSPARENT_PORT`. +- **`mangle` OUTPUT / `AB_NOTCP`** (position 1): the same exemptions + (plus `ESTABLISHED,RELATED` first, so UDP conntrack replies like DNS + pass), then `-p tcp -j RETURN` (TCP is handled by the nat REDIRECT) and + a terminal `DROP` for external **non-TCP** (UDP/QUIC), so HTTP/3 cannot + bypass — well-behaved clients fall back to TCP and get captured. + +Because the OUTPUT hook order is `raw → mangle → nat → filter`, the +mangle chain drops non-TCP on its original destination while TCP falls +through to the nat REDIRECT. Both chains are inserted at position 1, +ahead of Istio's appended (`-A`) chains, so they preempt ambient's nat +redirect for external destinations — exactly as `redirect` mode does for +the Envoy path. IPv6 mirrors apply the same rules. See +[`test-enforce-redirect.sh`](./test-enforce-redirect.sh), which proves +the capture, the preemption, and the non-TCP drop via packet counters. > **`CLUSTER_CIDRS` is Kind-shaped by default.** The `10.0.0.0/8` default > covers Kind (pods `10.244.0.0/16` + services `10.96.0.0/16`). Other @@ -57,26 +72,15 @@ the proxy UID, and `CLUSTER_CIDRS6`). > default would drop in-cluster service traffic. On OCP/EKS/etc. you > **must** override `CLUSTER_CIDRS` with the cluster's real pod+service > ranges. The script logs the resolved value at startup, and the -> operator wiring (follow-up PR) sets it from the cluster's CIDRs. +> operator sets it from the cluster's CIDRs. -> **`enforce-drop` intentionally ignores `OUTBOUND_PORTS_EXCLUDE`** (a +> **`enforce-redirect` intentionally ignores `OUTBOUND_PORTS_EXCLUDE`** (a > `redirect`-mode knob). Any destination previously bypassed that way — > e.g. a direct LLM endpoint at `host.docker.internal:11434` — is now -> dropped unless it goes through the forward proxy or falls within -> `CLUSTER_CIDRS`. That is the point: `enforce-drop` closes direct-egress -> holes. Operators relying on a bypass must route it through the proxy -> (or, for in-cluster targets, include it in `CLUSTER_CIDRS`). - -**Why `mangle` OUTPUT, not `filter`:** when Istio ambient is active it -installs an in-pod `nat OUTPUT` REDIRECT (`ISTIO_OUTPUT` → ztunnel -`:15001`). The netfilter OUTPUT hook order is `raw → mangle → nat → -filter`, so a DROP in `mangle` evaluates the original destination and -fires **before** ambient's nat redirect can rewrite it; a DROP in -`filter` would run after nat and be defeated. `-I 1` also places the -chain ahead of Istio's appended (`-A`) mangle chain. This makes the -guard robust with no ambient, in-pod ambient, or node-level ambient. -See [`test-enforce-drop.sh`](./test-enforce-drop.sh), which proves the -preemption via packet counters. +> captured (external TCP) or dropped (external non-TCP) unless it falls +> within `CLUSTER_CIDRS`. That is the point: `enforce-redirect` closes +> direct-egress holes. Operators relying on a bypass for an in-cluster +> target must include it in `CLUSTER_CIDRS`. ## iptables backend @@ -88,17 +92,18 @@ whichever the host kernel exposes. Override with `IPTABLES_CMD` (and | Variable | Default | Mode | Purpose | |---|---|---|---| -| `MODE` | `redirect` | both | `redirect` (envoy-sidecar) or `enforce-drop` (proxy-sidecar) | -| `PROXY_UID` | `1337` | both | UID of the AuthBridge sidecar process; exempted from redirect / drop | +| `MODE` | `redirect` | all | `redirect` (envoy-sidecar) or `enforce-redirect` (proxy-sidecar / lite) | +| `PROXY_UID` | `1337` | all | UID of the AuthBridge sidecar process; exempted from redirect | | `PROXY_PORT` | `15123` | redirect | AuthBridge outbound listener port | | `INBOUND_PROXY_PORT` | `15124` | redirect | AuthBridge inbound listener port | +| `TRANSPARENT_PORT` | `8082` | enforce-redirect | AuthBridge transparent listener port; REDIRECT target for captured external TCP egress | | `OUTBOUND_PORTS_EXCLUDE` | (empty) | redirect | Comma-separated outbound port list to skip (e.g. `8080`) | | `INBOUND_PORTS_EXCLUDE` | (empty) | redirect | Comma-separated inbound port list to skip | -| `POD_IP` | (required in `redirect`) | redirect | Set via Downward API; DNAT target for ambient-mesh inbound. Not used by `enforce-drop`. | -| `CLUSTER_CIDRS` | `10.0.0.0/8` | enforce-drop | Comma-separated in-cluster CIDRs allowed direct (pods/services/DNS) | -| `CLUSTER_CIDRS6` | (empty) | enforce-drop | IPv6 in-cluster CIDRs (dual-stack); empty drops all external v6 egress | -| `IPTABLES_CMD` | auto-detected | both | Override iptables binary (`iptables-legacy` / `iptables-nft`) | -| `IP6TABLES_CMD` | derived from `IPTABLES_CMD` | enforce-drop | Override ip6tables binary | +| `POD_IP` | (required in `redirect`) | redirect | Set via Downward API; DNAT target for ambient-mesh inbound. Not used by `enforce-redirect`. | +| `CLUSTER_CIDRS` | `10.0.0.0/8` | enforce-redirect | Comma-separated in-cluster CIDRs allowed direct (pods/services/DNS) | +| `CLUSTER_CIDRS6` | (empty) | enforce-redirect | IPv6 in-cluster CIDRs (dual-stack); empty drops all external v6 egress | +| `IPTABLES_CMD` | auto-detected | all | Override iptables binary (`iptables-legacy` / `iptables-nft`) | +| `IP6TABLES_CMD` | derived from `IPTABLES_CMD` | enforce-redirect | Override ip6tables binary | ## Required Kubernetes capabilities @@ -120,14 +125,16 @@ in [`.github/workflows/build.yaml`](../../.github/workflows/build.yaml)). ## Testing -[`test-enforce-drop.sh`](./test-enforce-drop.sh) validates `enforce-drop` -mode in a private network namespace (`unshare --net`): it asserts the -`AB_EGRESS` rule structure and proves the `mangle` DROP preempts a -simulated Istio ambient `nat OUTPUT` REDIRECT via packet counters. -Requires root + iptables-nft on Linux (runs on CI; not macOS): +[`test-enforce-redirect.sh`](./test-enforce-redirect.sh) validates +`enforce-redirect` mode in a private network namespace (`unshare --net`): +it asserts the `AB_REDIRECT` / `AB_NOTCP` rule structure, proves external +TCP is captured to `TRANSPARENT_PORT` while preempting a simulated Istio +ambient `nat OUTPUT` REDIRECT, and proves external UDP is dropped — all via +packet counters. Requires root + iptables-nft on Linux (runs on CI; not +macOS): ```sh -sudo ./test-enforce-drop.sh +sudo ./test-enforce-redirect.sh ``` ## Where it gets injected @@ -137,10 +144,10 @@ container automatically: - `redirect` mode (`MODE` unset) when the resolved AuthBridge mode is `envoy-sidecar`. -- `enforce-drop` mode (`MODE=enforce-drop`) when `proxy-sidecar` - egress enforcement is enabled (opt-in). _The operator wiring that - sets this lands in the follow-up kagenti-operator PR; this PR only - adds the mode to the image._ +- `enforce-redirect` mode (`MODE=enforce-redirect`) when the resolved + AuthBridge mode is `proxy-sidecar` / `lite` — the transparent listener + in those images receives the captured egress. This is always-on for + those modes (the operator injects it unconditionally). See [`authbridge/demos/weather-agent/demo-ui-advanced.md`](../demos/weather-agent/demo-ui-advanced.md) diff --git a/authbridge/proxy-init/init-iptables.sh b/authbridge/proxy-init/init-iptables.sh index 71fa67835..34856324c 100644 --- a/authbridge/proxy-init/init-iptables.sh +++ b/authbridge/proxy-init/init-iptables.sh @@ -128,18 +128,23 @@ set -e # --- Mode selection --- # MODE selects the interception strategy: -# redirect (default) — envoy-sidecar: transparently REDIRECT pod traffic -# to the Envoy listeners (the behavior documented above). -# enforce-drop — proxy-sidecar: a fail-closed egress guard. The app is -# configured with HTTP_PROXY pointing at AuthBridge's forward -# proxy; this mode DROPs any egress that bypasses the proxy, -# forcing all external traffic through AuthBridge regardless of -# whether the app honors HTTP_PROXY. It installs no REDIRECT and -# no PREROUTING/inbound rules. See setup_enforce_drop() below. +# redirect (default) — envoy-sidecar: transparently REDIRECT pod +# traffic to the Envoy listeners (the behavior documented +# above). +# enforce-redirect — proxy-sidecar: a fail-closed egress guard that CAPTURES +# rather than drops. External TCP egress that bypasses the +# forward proxy is transparently REDIRECTed to AuthBridge's +# transparent listener (TRANSPARENT_PORT), which recovers the +# original destination via SO_ORIGINAL_DST and tunnels it +# through the same outbound pipeline. Non-TCP external egress +# (UDP/QUIC) is DROPped so it cannot bypass via HTTP/3. +# In-cluster + loopback + proxy-UID traffic is left direct. +# Nothing breaks for agents that ignore HTTP_PROXY — their +# traffic is captured, not dropped. See setup_enforce_redirect(). MODE="${MODE:-redirect}" case "${MODE}" in - redirect|enforce-drop) ;; - *) echo "ERROR: unknown MODE='${MODE}' (expected: redirect | enforce-drop)" >&2; exit 1 ;; + redirect|enforce-redirect) ;; + *) echo "ERROR: unknown MODE='${MODE}' (expected: redirect | enforce-redirect)" >&2; exit 1 ;; esac # --- Auto-detect iptables backend --- @@ -164,15 +169,20 @@ echo "Using iptables command: ${IPT} ($(${IPT} --version 2>/dev/null || echo 'un PROXY_PORT="${PROXY_PORT:-15123}" INBOUND_PROXY_PORT="${INBOUND_PROXY_PORT:-15124}" +# enforce-redirect mode: the forward proxy's transparent listener port, the +# REDIRECT target for captured external TCP egress. Must match the authbridge +# proxy-sidecar listener.transparent_proxy_addr (default :8082). +TRANSPARENT_PORT="${TRANSPARENT_PORT:-8082}" PROXY_UID="${PROXY_UID:-1337}" SSH_PORT="${SSH_PORT:-22}" OUTBOUND_PORTS_EXCLUDE="${OUTBOUND_PORTS_EXCLUDE:-}" INBOUND_PORTS_EXCLUDE="${INBOUND_PORTS_EXCLUDE:-}" -# enforce-drop mode: in-cluster destinations the agent may reach directly -# (pods / services / DNS) — everything else egressing the pod is dropped. -# Defaults to the RFC1918 10/8 block which covers typical Kind pod (10.244/16) -# and service (10.96/16) CIDRs; override with the cluster's actual ranges. +# enforce-redirect mode: in-cluster destinations the agent may reach directly +# (pods / services / DNS) — external TCP is REDIRECTed to the transparent +# listener and external non-TCP is dropped. Defaults to the RFC1918 10/8 block +# which covers typical Kind pod (10.244/16) and service (10.96/16) CIDRs; +# override with the cluster's actual ranges. CLUSTER_CIDRS="${CLUSTER_CIDRS:-10.0.0.0/8}" CLUSTER_CIDRS6="${CLUSTER_CIDRS6:-}" # IPv6 in-cluster CIDRs (dual-stack); empty = none @@ -190,7 +200,7 @@ ISTIO_HEALTH_PROBE_SRC="${ISTIO_HEALTH_PROBE_SRC:-169.254.7.127}" # We use DNAT to the pod IP instead of REDIRECT to avoid needing route_localnet=1, # which would require a privileged init container (to write to read-only /proc/sys). # POD_IP is only needed by redirect mode (DNAT target for the ambient inbound -# rule). enforce-drop does no DNAT, so it does not require it. +# rule). enforce-redirect does no DNAT, so it does not require it. if [ "${MODE}" = "redirect" ] && [ -z "${POD_IP}" ]; then echo "ERROR: POD_IP environment variable is not set (required for redirect mode)." >&2 echo "Set it via the Kubernetes Downward API (status.podIP) or manually." >&2 @@ -198,103 +208,140 @@ if [ "${MODE}" = "redirect" ] && [ -z "${POD_IP}" ]; then fi # ============================================================================= -# enforce-drop mode (proxy-sidecar fail-closed egress guard) +# enforce-redirect mode (proxy-sidecar fail-closed egress guard, capture variant) # ============================================================================= # -# proxy-sidecar configures the app with HTTP_PROXY=127.0.0.1:. -# Unlike redirect mode we do NOT transparently REDIRECT — you cannot redirect -# raw traffic into a CONNECT forward proxy. Instead we DROP any egress that -# leaves the pod without going through the proxy, forcing all external traffic -# through AuthBridge regardless of whether the app honors HTTP_PROXY. -# -# Placement — a dedicated chain hooked from *mangle* OUTPUT at position 1: -# * Istio ambient, when active, installs an in-pod `nat OUTPUT` REDIRECT -# (ISTIO_OUTPUT -> ztunnel :15001). The netfilter OUTPUT hook order is -# raw -> mangle -> nat -> filter, so a DROP in mangle evaluates the -# ORIGINAL destination and fires BEFORE ambient's nat redirect can rewrite -# it. A DROP in `filter` would run after nat and be defeated (dst already -# rewritten to 127.0.0.1). -I 1 also places us ahead of Istio's appended -# (-A) mangle ISTIO_OUTPUT chain. -# * Works identically with no ambient, in-pod ambient, or node-level ambient -# (in the node-level case our pod-netns rule runs before the packet ever -# reaches the host netns). -# -# Rule order in the chain: RETURN ztunnel's own sockets (fwmark 0x539, a no-op -# when ambient is absent) -> RETURN the proxy's own egress (PROXY_UID) -> -# RETURN loopback (app -> proxy) -> RETURN in-cluster CIDRs (mesh/DNS) -> -# DROP everything else (direct external egress, incl. UDP/QUIC). -setup_enforce_drop() { - CHAIN="AB_EGRESS" - - echo "enforce-drop: installing fail-closed egress guard (mangle OUTPUT, chain ${CHAIN})" - echo "enforce-drop: exempt proxy UID=${PROXY_UID}; allowed in-cluster CIDRs=${CLUSTER_CIDRS}" - - # --- IPv4 --- - ${IPT} -t mangle -N "${CHAIN}" 2>/dev/null || true - ${IPT} -t mangle -F "${CHAIN}" - # Replies to inbound connections (and related flows) are locally generated and - # also traverse OUTPUT — a reply is never a "bypass". Let established/related - # traffic through FIRST, so e.g. kubelet health-probe responses to an - # off-cluster node IP (in Kind the node is 172.18.0.0/16, outside CLUSTER_CIDRS) - # are not caught by the terminal DROP. Only NEW app-initiated flows are gated. - ${IPT} -t mangle -A "${CHAIN}" -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN - # ztunnel's own sockets (ambient) carry fwmark 0x539 — let them through so the - # mesh/HBONE path keeps working. No-op when ambient is not installed. - ${IPT} -t mangle -A "${CHAIN}" -m mark --mark "${ZTUNNEL_MARK}" -j RETURN - # the AuthBridge proxy's own re-originated egress (must run as PROXY_UID). - ${IPT} -t mangle -A "${CHAIN}" -m owner --uid-owner "${PROXY_UID}" -j RETURN - # app -> proxy over loopback (HTTP_PROXY target), and any loopback traffic. - ${IPT} -t mangle -A "${CHAIN}" -o lo -j RETURN - ${IPT} -t mangle -A "${CHAIN}" -d 127.0.0.0/8 -j RETURN - # in-cluster traffic (pods / services / DNS) — carried by the mesh, not the proxy. +# Forces all external egress through AuthBridge regardless of whether the app +# honors HTTP_PROXY, by CAPTURING bypass traffic: external TCP is transparently +# REDIRECTed to the forward proxy's transparent listener (TRANSPARENT_PORT), +# which recovers the original destination via SO_ORIGINAL_DST and tunnels it +# through the same outbound pipeline. Because nothing is dropped, agents that +# ignore HTTP_PROXY keep working — this is what lets enforcement be always-on. +# +# Placement — a dedicated chain hooked from *nat* OUTPUT at position 1 (REDIRECT +# is a nat-table target). Inserted before Istio's appended ISTIO_OUTPUT so we +# preempt ambient's nat redirect for external destinations, exactly as +# redirect mode does for the Envoy path. +# +# Rule order: RETURN ztunnel's own sockets (fwmark 0x539, no-op without ambient) +# -> RETURN the proxy's own re-originated egress (PROXY_UID, avoids the loop) -> +# RETURN loopback (app -> forward proxy via HTTP_PROXY, and any loopback) -> +# RETURN in-cluster CIDRs (mesh/DNS, left direct) -> REDIRECT external TCP to +# TRANSPARENT_PORT -> DROP all other external egress (UDP/QUIC, so HTTP/3 can't +# bypass; well-behaved clients fall back to TCP and get captured). +# +# The nat REDIRECT chain has no conntrack ESTABLISHED rule: nat only evaluates +# the first packet of a flow, so replies and established connections are not +# re-translated. +# Two chains are needed because REDIRECT is a nat-table target but the nat table +# forbids DROP ("the use of DROP is therefore inhibited"): +# * nat OUTPUT / AB_REDIRECT — REDIRECT external TCP to TRANSPARENT_PORT. +# * mangle OUTPUT / AB_NOTCP — DROP external non-TCP (UDP/QUIC) so HTTP/3 +# cannot bypass; `-p tcp -j RETURN` lets TCP +# fall through to the nat REDIRECT. +# mangle runs before nat in the OUTPUT hook, so non-TCP is dropped on its +# original destination and TCP is passed to the nat REDIRECT. Both are inserted +# at position 1 to precede Istio's appended chains. +setup_enforce_redirect() { + REDIR_CHAIN="AB_REDIRECT" + NOTCP_CHAIN="AB_NOTCP" + + echo "enforce-redirect: installing fail-closed egress capture" + echo "enforce-redirect: external TCP -> 127.0.0.1:${TRANSPARENT_PORT} (nat REDIRECT); external non-TCP -> DROP (mangle)" + echo "enforce-redirect: exempt proxy UID=${PROXY_UID}; direct in-cluster CIDRs=${CLUSTER_CIDRS}" + + # --- IPv4: nat REDIRECT for TCP --- + ${IPT} -t nat -N "${REDIR_CHAIN}" 2>/dev/null || true + ${IPT} -t nat -F "${REDIR_CHAIN}" + # ztunnel's own sockets (ambient) carry fwmark 0x539 — let them through. + ${IPT} -t nat -A "${REDIR_CHAIN}" -m mark --mark "${ZTUNNEL_MARK}" -j RETURN + # the AuthBridge proxy's own re-originated egress (runs as PROXY_UID) — avoids + # redirecting the proxy's upstream dial back into itself. + ${IPT} -t nat -A "${REDIR_CHAIN}" -m owner --uid-owner "${PROXY_UID}" -j RETURN + # app -> forward proxy over loopback (HTTP_PROXY target), and any loopback. + ${IPT} -t nat -A "${REDIR_CHAIN}" -o lo -j RETURN + ${IPT} -t nat -A "${REDIR_CHAIN}" -d 127.0.0.0/8 -j RETURN + # in-cluster traffic (pods / services / DNS) — left direct, carried by the mesh. for cidr in $(echo "${CLUSTER_CIDRS}" | tr ',' ' '); do - [ -n "${cidr}" ] && ${IPT} -t mangle -A "${CHAIN}" -d "${cidr}" -j RETURN + [ -n "${cidr}" ] && ${IPT} -t nat -A "${REDIR_CHAIN}" -d "${cidr}" -j RETURN done - # everything else == direct external egress that bypassed the proxy. Drop it. - # No -p filter, so UDP (QUIC/HTTP-3) is dropped as well as TCP. - ${IPT} -t mangle -A "${CHAIN}" -j DROP - # Hook at position 1 so we run before any appended Istio mangle chain and - # before nat OUTPUT. - if ! ${IPT} -t mangle -C OUTPUT -j "${CHAIN}" 2>/dev/null; then - ${IPT} -t mangle -I OUTPUT 1 -j "${CHAIN}" + # external TCP that bypassed the forward proxy — capture it transparently. + ${IPT} -t nat -A "${REDIR_CHAIN}" -p tcp -j REDIRECT --to-port "${TRANSPARENT_PORT}" + if ! ${IPT} -t nat -C OUTPUT -j "${REDIR_CHAIN}" 2>/dev/null; then + ${IPT} -t nat -I OUTPUT 1 -j "${REDIR_CHAIN}" fi - echo "enforce-drop: IPv4 egress guard configured" + + # --- IPv4: mangle DROP for non-TCP --- + ${IPT} -t mangle -N "${NOTCP_CHAIN}" 2>/dev/null || true + ${IPT} -t mangle -F "${NOTCP_CHAIN}" + # established/related replies (incl. UDP conntrack, e.g. DNS replies) first. + ${IPT} -t mangle -A "${NOTCP_CHAIN}" -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN + ${IPT} -t mangle -A "${NOTCP_CHAIN}" -m mark --mark "${ZTUNNEL_MARK}" -j RETURN + ${IPT} -t mangle -A "${NOTCP_CHAIN}" -m owner --uid-owner "${PROXY_UID}" -j RETURN + ${IPT} -t mangle -A "${NOTCP_CHAIN}" -o lo -j RETURN + ${IPT} -t mangle -A "${NOTCP_CHAIN}" -d 127.0.0.0/8 -j RETURN + for cidr in $(echo "${CLUSTER_CIDRS}" | tr ',' ' '); do + [ -n "${cidr}" ] && ${IPT} -t mangle -A "${NOTCP_CHAIN}" -d "${cidr}" -j RETURN + done + # TCP is handled by the nat REDIRECT above — let it pass mangle untouched. + ${IPT} -t mangle -A "${NOTCP_CHAIN}" -p tcp -j RETURN + # everything else == external non-TCP egress (UDP/QUIC) — drop it. + ${IPT} -t mangle -A "${NOTCP_CHAIN}" -j DROP + if ! ${IPT} -t mangle -C OUTPUT -j "${NOTCP_CHAIN}" 2>/dev/null; then + ${IPT} -t mangle -I OUTPUT 1 -j "${NOTCP_CHAIN}" + fi + echo "enforce-redirect: IPv4 egress capture configured" # --- IPv6 --- - # Cluster is IPv4-only by default; until v6 cluster CIDRs are wired - # (CLUSTER_CIDRS6), drop external v6 egress while allowing: established/related - # replies, loopback, link-local unicast (fe80::/10) and link-local multicast - # (ff02::/16, which carries NDP neighbor/router solicitations and MLD), the - # proxy UID, and ztunnel's mark. - if command -v "${IP6T%% *}" >/dev/null 2>&1 && ${IP6T} -t mangle -L >/dev/null 2>&1; then - ${IP6T} -t mangle -N "${CHAIN}" 2>/dev/null || true - ${IP6T} -t mangle -F "${CHAIN}" - ${IP6T} -t mangle -A "${CHAIN}" -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN - ${IP6T} -t mangle -A "${CHAIN}" -m mark --mark "${ZTUNNEL_MARK}" -j RETURN - ${IP6T} -t mangle -A "${CHAIN}" -m owner --uid-owner "${PROXY_UID}" -j RETURN - ${IP6T} -t mangle -A "${CHAIN}" -o lo -j RETURN - ${IP6T} -t mangle -A "${CHAIN}" -d ::1/128 -j RETURN - ${IP6T} -t mangle -A "${CHAIN}" -d fe80::/10 -j RETURN - ${IP6T} -t mangle -A "${CHAIN}" -d ff02::/16 -j RETURN + # Mirror of IPv4. Until v6 cluster CIDRs are wired (CLUSTER_CIDRS6), allow + # loopback + link-local (fe80::/10 unicast, ff02::/16 NDP/MLD multicast) and + # the proxy UID / ztunnel mark; REDIRECT external v6 TCP; DROP other v6 egress. + if command -v "${IP6T%% *}" >/dev/null 2>&1 && ${IP6T} -t nat -L >/dev/null 2>&1; then + ${IP6T} -t nat -N "${REDIR_CHAIN}" 2>/dev/null || true + ${IP6T} -t nat -F "${REDIR_CHAIN}" + ${IP6T} -t nat -A "${REDIR_CHAIN}" -m mark --mark "${ZTUNNEL_MARK}" -j RETURN + ${IP6T} -t nat -A "${REDIR_CHAIN}" -m owner --uid-owner "${PROXY_UID}" -j RETURN + ${IP6T} -t nat -A "${REDIR_CHAIN}" -o lo -j RETURN + ${IP6T} -t nat -A "${REDIR_CHAIN}" -d ::1/128 -j RETURN + ${IP6T} -t nat -A "${REDIR_CHAIN}" -d fe80::/10 -j RETURN + ${IP6T} -t nat -A "${REDIR_CHAIN}" -d ff02::/16 -j RETURN + for cidr in $(echo "${CLUSTER_CIDRS6}" | tr ',' ' '); do + [ -n "${cidr}" ] && ${IP6T} -t nat -A "${REDIR_CHAIN}" -d "${cidr}" -j RETURN + done + ${IP6T} -t nat -A "${REDIR_CHAIN}" -p tcp -j REDIRECT --to-port "${TRANSPARENT_PORT}" + if ! ${IP6T} -t nat -C OUTPUT -j "${REDIR_CHAIN}" 2>/dev/null; then + ${IP6T} -t nat -I OUTPUT 1 -j "${REDIR_CHAIN}" + fi + + ${IP6T} -t mangle -N "${NOTCP_CHAIN}" 2>/dev/null || true + ${IP6T} -t mangle -F "${NOTCP_CHAIN}" + ${IP6T} -t mangle -A "${NOTCP_CHAIN}" -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN + ${IP6T} -t mangle -A "${NOTCP_CHAIN}" -m mark --mark "${ZTUNNEL_MARK}" -j RETURN + ${IP6T} -t mangle -A "${NOTCP_CHAIN}" -m owner --uid-owner "${PROXY_UID}" -j RETURN + ${IP6T} -t mangle -A "${NOTCP_CHAIN}" -o lo -j RETURN + ${IP6T} -t mangle -A "${NOTCP_CHAIN}" -d ::1/128 -j RETURN + ${IP6T} -t mangle -A "${NOTCP_CHAIN}" -d fe80::/10 -j RETURN + ${IP6T} -t mangle -A "${NOTCP_CHAIN}" -d ff02::/16 -j RETURN for cidr in $(echo "${CLUSTER_CIDRS6}" | tr ',' ' '); do - [ -n "${cidr}" ] && ${IP6T} -t mangle -A "${CHAIN}" -d "${cidr}" -j RETURN + [ -n "${cidr}" ] && ${IP6T} -t mangle -A "${NOTCP_CHAIN}" -d "${cidr}" -j RETURN done - ${IP6T} -t mangle -A "${CHAIN}" -j DROP - if ! ${IP6T} -t mangle -C OUTPUT -j "${CHAIN}" 2>/dev/null; then - ${IP6T} -t mangle -I OUTPUT 1 -j "${CHAIN}" + ${IP6T} -t mangle -A "${NOTCP_CHAIN}" -p tcp -j RETURN + ${IP6T} -t mangle -A "${NOTCP_CHAIN}" -j DROP + if ! ${IP6T} -t mangle -C OUTPUT -j "${NOTCP_CHAIN}" 2>/dev/null; then + ${IP6T} -t mangle -I OUTPUT 1 -j "${NOTCP_CHAIN}" fi - echo "enforce-drop: IPv6 egress guard configured" + echo "enforce-redirect: IPv6 egress capture configured" else - echo "enforce-drop: ip6tables unavailable — skipping IPv6 egress guard" + echo "enforce-redirect: ip6tables unavailable — skipping IPv6 egress capture" fi - echo "enforce-drop: fail-closed egress guard active" + echo "enforce-redirect: fail-closed egress capture active" } -# Dispatch enforce-drop here and exit; redirect mode falls through to the +# Dispatch enforce-redirect here and exit; redirect mode falls through to the # transparent-interception logic below. -if [ "${MODE}" = "enforce-drop" ]; then - setup_enforce_drop +if [ "${MODE}" = "enforce-redirect" ]; then + setup_enforce_redirect exit 0 fi diff --git a/authbridge/proxy-init/test-enforce-drop.sh b/authbridge/proxy-init/test-enforce-drop.sh deleted file mode 100755 index 0cf570fd2..000000000 --- a/authbridge/proxy-init/test-enforce-drop.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash -# -# Test harness for init-iptables.sh "enforce-drop" mode (proxy-sidecar -# fail-closed egress guard). -# -# It validates two things in a private network namespace: -# 1. Rule STRUCTURE — the AB_EGRESS chain is hooked from mangle OUTPUT at -# position 1 with the expected RETURN exemptions and a terminal DROP, and -# that no nat/filter rules are created. -# 2. AMBIENT ROBUSTNESS — a DROP in mangle OUTPUT preempts a simulated Istio -# ambient "nat OUTPUT REDIRECT" (ISTIO_OUTPUT). Proven via packet counters: -# after generating an external SYN, the mangle DROP increments and the nat -# REDIRECT does NOT. -# -# Requirements: root (for unshare --net + iptables), iproute2, iptables-nft, -# bash, the dummy kernel module. Runs on Linux / CI (e.g. ubuntu-latest); not -# on macOS. Uses `unshare --net` (not named `ip netns`) so it also works inside -# nested containers. Exit code 0 = all pass. -set -euo pipefail - -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -INIT="${INIT_SCRIPT:-${SCRIPT_DIR}/init-iptables.sh}" -IPT="${IPTABLES_CMD:-iptables-nft}" -EXTERNAL="198.51.100.7" # RFC5737 TEST-NET-2, guaranteed unused - -# Re-exec into a private network namespace. unshare avoids the /sys remount that -# named `ip netns exec` performs, so this works inside nested containers too. -if [ -z "${_AB_NETNS_REEXEC:-}" ]; then - exec unshare --net env _AB_NETNS_REEXEC=1 INIT_SCRIPT="${INIT}" \ - IPTABLES_CMD="${IPT}" bash "$0" "$@" -fi - -fail=0 - -# Fresh netns: bring up lo and a dummy default route so packets to an external -# destination are actually generated and traverse the OUTPUT chain. -ip link set lo up -if ip link add eth-test type dummy 2>/dev/null; then - ip addr add 10.255.255.2/24 dev eth-test - ip link set eth-test up - ip route add default via 10.255.255.1 -else - echo "WARN: dummy interface unavailable; preemption packet may not be generated" -fi - -echo "### Installing enforce-drop rules" -env MODE=enforce-drop PROXY_UID=1337 CLUSTER_CIDRS=10.0.0.0/8 \ - IPTABLES_CMD="${IPT}" IP6TABLES_CMD=ip6tables-nft \ - sh "${INIT}" || { echo "FAIL: init script exited non-zero"; exit 1; } - -dump=$("${IPT}" -t mangle -S) -echo "--- mangle ruleset ---"; echo "${dump}" - -assert() { if echo "${dump}" | grep -qE "$2"; then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi; } -assert "AB_EGRESS hooked from OUTPUT" '^-A OUTPUT -j AB_EGRESS' -assert "established/related RETURN" 'AB_EGRESS -m conntrack --ctstate (ESTABLISHED,RELATED|RELATED,ESTABLISHED) -j RETURN' -assert "ztunnel mark RETURN" 'AB_EGRESS .*mark.*0x539.*-j RETURN' -assert "proxy UID RETURN" 'AB_EGRESS .*--uid-owner 1337 -j RETURN' -assert "loopback iface RETURN" 'AB_EGRESS -o lo -j RETURN' -assert "loopback cidr RETURN" 'AB_EGRESS -d 127.0.0.0/8 -j RETURN' -assert "cluster cidr RETURN" 'AB_EGRESS -d 10.0.0.0/8 -j RETURN' -assert "terminal DROP" 'AB_EGRESS -j DROP' - -pos1=$("${IPT}" -t mangle -L OUTPUT --line-numbers -n | awk '$1=="1"{print $2}') -if [ "${pos1}" = "AB_EGRESS" ]; then echo "PASS: AB_EGRESS at OUTPUT position 1" -else echo "FAIL: AB_EGRESS not at OUTPUT position 1 (got '${pos1}')"; fail=1; fi - -# the established/related RETURN must be the first rule in the chain (replies -# must be let through before any owner/dest evaluation). -first_rule=$("${IPT}" -t mangle -S AB_EGRESS | grep '^-A AB_EGRESS' | head -1) -if echo "${first_rule}" | grep -q 'conntrack'; then echo "PASS: established/related RETURN is first in AB_EGRESS" -else echo "FAIL: first AB_EGRESS rule is not the conntrack RETURN (got: ${first_rule})"; fail=1; fi - -natcount=$("${IPT}" -t nat -S | grep -cE 'AB_EGRESS|REDIRECT|PROXY_' || true) -if [ "${natcount:-0}" -eq 0 ]; then echo "PASS: no nat-table rules created" -else echo "FAIL: enforce-drop created nat rules"; fail=1; fi - -echo "### Ambient-preemption test: append a simulated ISTIO_OUTPUT nat REDIRECT" -"${IPT}" -t nat -A OUTPUT -p tcp -d "${EXTERNAL}" -j REDIRECT --to-ports 19999 -# Generate an external SYN (uid 0, like an agent bypass attempt). -timeout 2 bash -c "exec 3<>/dev/tcp/${EXTERNAL}/80" 2>/dev/null || true - -dropc=$("${IPT}" -t mangle -L AB_EGRESS -n -v | awk '/DROP/{print $1; exit}') -redirc=$("${IPT}" -t nat -L OUTPUT -n -v | awk '/REDIRECT/{print $1; exit}') -echo "mangle AB_EGRESS DROP pkts=${dropc:-?} | nat REDIRECT pkts=${redirc:-?}" -if [ "${dropc:-0}" -gt 0 ] && [ "${redirc:-0}" -eq 0 ]; then - echo "PASS: mangle DROP preempted nat REDIRECT (ambient-robust)" -else - echo "FAIL: preemption not demonstrated (DROP=${dropc:-?}, REDIRECT=${redirc:-?})"; fail=1 -fi - -echo -[ "${fail}" -eq 0 ] && echo "ALL TESTS PASSED" || echo "SOME TESTS FAILED" -exit "${fail}" diff --git a/authbridge/proxy-init/test-enforce-redirect.sh b/authbridge/proxy-init/test-enforce-redirect.sh new file mode 100755 index 000000000..966f5fd66 --- /dev/null +++ b/authbridge/proxy-init/test-enforce-redirect.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# Test harness for init-iptables.sh "enforce-redirect" mode (proxy-sidecar +# fail-closed egress guard, capture variant). +# +# It validates, in a private network namespace: +# 1. Rule STRUCTURE — the AB_REDIRECT chain is hooked from nat OUTPUT at +# position 1 with the expected RETURN exemptions and a `-p tcp` REDIRECT to +# TRANSPARENT_PORT (no DROP — the nat table forbids it); and the AB_NOTCP +# chain is hooked from mangle OUTPUT with `-p tcp RETURN` then a terminal +# DROP for external non-TCP egress. +# 2. CAPTURE (not drop) + AMBIENT ROBUSTNESS — external TCP egress is +# REDIRECTed to TRANSPARENT_PORT, preempting a simulated Istio ambient +# "nat OUTPUT REDIRECT" appended after our chain. Proven via packet +# counters: our REDIRECT increments, the simulated ISTIO REDIRECT does not. +# 3. NON-TCP DROP — an external UDP datagram (QUIC/HTTP-3 bypass attempt) hits +# the mangle AB_NOTCP DROP, proving non-TCP external egress cannot bypass. +# +# Requirements: root (for unshare --net + iptables), iproute2, iptables-nft, +# bash, the dummy kernel module. Runs on Linux / CI (e.g. ubuntu-latest); not on +# macOS. Uses `unshare --net` so it also works inside nested containers. Exit +# code 0 = all pass. +set -euo pipefail + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +INIT="${INIT_SCRIPT:-${SCRIPT_DIR}/init-iptables.sh}" +IPT="${IPTABLES_CMD:-iptables-nft}" +EXTERNAL="198.51.100.7" # RFC5737 TEST-NET-2, guaranteed unused +TPORT="8082" + +# Re-exec into a private network namespace. +if [ -z "${_AB_NETNS_REEXEC:-}" ]; then + exec unshare --net env _AB_NETNS_REEXEC=1 INIT_SCRIPT="${INIT}" \ + IPTABLES_CMD="${IPT}" bash "$0" "$@" +fi + +fail=0 + +# Fresh netns: bring up lo and a dummy default route so packets to an external +# destination are actually generated and traverse the OUTPUT chain. +ip link set lo up +if ip link add eth-test type dummy 2>/dev/null; then + ip addr add 10.255.255.2/24 dev eth-test + ip link set eth-test up + ip route add default via 10.255.255.1 +else + echo "WARN: dummy interface unavailable; capture packet may not be generated" +fi + +echo "### Installing enforce-redirect rules" +env MODE=enforce-redirect PROXY_UID=1337 CLUSTER_CIDRS=10.0.0.0/8 \ + TRANSPARENT_PORT="${TPORT}" \ + IPTABLES_CMD="${IPT}" IP6TABLES_CMD=ip6tables-nft \ + sh "${INIT}" || { echo "FAIL: init script exited non-zero"; exit 1; } + +natdump=$("${IPT}" -t nat -S) +mangledump=$("${IPT}" -t mangle -S) +echo "--- nat ruleset ---"; echo "${natdump}" +echo "--- mangle ruleset ---"; echo "${mangledump}" + +assert() { if echo "$3" | grep -qE "$2"; then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi; } +# nat AB_REDIRECT — TCP capture (no DROP; nat forbids it). +assert "AB_REDIRECT hooked from nat OUTPUT" '^-A OUTPUT -j AB_REDIRECT' "${natdump}" +assert "nat ztunnel mark RETURN" 'AB_REDIRECT .*mark.*0x539.*-j RETURN' "${natdump}" +assert "nat proxy UID RETURN" 'AB_REDIRECT .*--uid-owner 1337 -j RETURN' "${natdump}" +assert "nat loopback iface RETURN" 'AB_REDIRECT -o lo -j RETURN' "${natdump}" +assert "nat loopback cidr RETURN" 'AB_REDIRECT -d 127.0.0.0/8 -j RETURN' "${natdump}" +assert "nat cluster cidr RETURN" 'AB_REDIRECT -d 10.0.0.0/8 -j RETURN' "${natdump}" +assert "nat tcp REDIRECT to transparent" "AB_REDIRECT -p tcp -j REDIRECT --to-ports ${TPORT}" "${natdump}" +if echo "${natdump}" | grep -qE 'AB_REDIRECT -j DROP'; then + echo "FAIL: nat AB_REDIRECT must not contain DROP (nat table forbids it)"; fail=1 +else echo "PASS: nat AB_REDIRECT has no DROP (correctly delegated to mangle)"; fi +# mangle AB_NOTCP — non-TCP drop, TCP passes through to the nat REDIRECT. +assert "AB_NOTCP hooked from mangle OUTPUT" '^-A OUTPUT -j AB_NOTCP' "${mangledump}" +assert "mangle established/related RETURN" 'AB_NOTCP -m conntrack --ctstate (ESTABLISHED,RELATED|RELATED,ESTABLISHED) -j RETURN' "${mangledump}" +assert "mangle proxy UID RETURN" 'AB_NOTCP .*--uid-owner 1337 -j RETURN' "${mangledump}" +assert "mangle cluster cidr RETURN" 'AB_NOTCP -d 10.0.0.0/8 -j RETURN' "${mangledump}" +assert "mangle tcp RETURN (defer to nat)" 'AB_NOTCP -p tcp -j RETURN' "${mangledump}" +assert "mangle terminal DROP (non-tcp)" 'AB_NOTCP -j DROP' "${mangledump}" + +pos1=$("${IPT}" -t nat -L OUTPUT --line-numbers -n | awk '$1=="1"{print $2}') +if [ "${pos1}" = "AB_REDIRECT" ]; then echo "PASS: AB_REDIRECT at nat OUTPUT position 1" +else echo "FAIL: AB_REDIRECT not at nat OUTPUT position 1 (got '${pos1}')"; fail=1; fi +mpos1=$("${IPT}" -t mangle -L OUTPUT --line-numbers -n | awk '$1=="1"{print $2}') +if [ "${mpos1}" = "AB_NOTCP" ]; then echo "PASS: AB_NOTCP at mangle OUTPUT position 1" +else echo "FAIL: AB_NOTCP not at mangle OUTPUT position 1 (got '${mpos1}')"; fail=1; fi + +echo "### Capture + preemption test: append a simulated ISTIO_OUTPUT nat REDIRECT" +"${IPT}" -t nat -A OUTPUT -p tcp -d "${EXTERNAL}" -j REDIRECT --to-ports 19999 +# Generate an external TCP SYN (uid 0, like an agent bypass attempt). With no +# listener on TPORT the redirected SYN gets an RST; the rule counter still ticks. +timeout 2 bash -c "exec 3<>/dev/tcp/${EXTERNAL}/80" 2>/dev/null || true + +capc=$("${IPT}" -t nat -L AB_REDIRECT -n -v | awk '/REDIRECT/{print $1; exit}') +istioc=$("${IPT}" -t nat -L OUTPUT -n -v | awk '/REDIRECT/{print $1; exit}') +echo "AB_REDIRECT REDIRECT pkts=${capc:-?} | simulated ISTIO REDIRECT pkts=${istioc:-?}" +if [ "${capc:-0}" -gt 0 ] && [ "${istioc:-0}" -eq 0 ]; then + echo "PASS: external TCP captured to transparent port, preempting nat REDIRECT (ambient-robust)" +else + echo "FAIL: capture/preemption not demonstrated (AB=${capc:-?}, ISTIO=${istioc:-?})"; fail=1 +fi + +echo "### Non-TCP drop test: send an external UDP datagram (QUIC bypass attempt)" +timeout 2 bash -c "echo -n x >/dev/udp/${EXTERNAL}/53" 2>/dev/null || true +dropc=$("${IPT}" -t mangle -L AB_NOTCP -n -v | awk '/DROP/{print $1; exit}') +echo "mangle AB_NOTCP DROP pkts=${dropc:-?}" +if [ "${dropc:-0}" -gt 0 ]; then + echo "PASS: external UDP dropped (HTTP/3 cannot bypass)" +else + echo "FAIL: external UDP not dropped (DROP=${dropc:-?})"; fail=1 +fi + +echo +[ "${fail}" -eq 0 ] && echo "ALL TESTS PASSED" || echo "SOME TESTS FAILED" +exit "${fail}"