diff --git a/configs/ip_allow.yaml.default b/configs/ip_allow.yaml.default index 16db1d2845a..c0fd0944829 100644 --- a/configs/ip_allow.yaml.default +++ b/configs/ip_allow.yaml.default @@ -21,8 +21,9 @@ # The top level tag 'ip_allow' identifies the rule items. Its value must be a rule item or a # sequence of rule items. # -# Rules are applied in the order listed starting from the top. -# That means you generally want to append your rules after the ones listed here. +# Rules are applied in the order listed starting from the top. The first +# matching IP range determines the rule used for that connection, so add +# explicit exceptions before broader deny rules. # # Allow anything on localhost, limit destructive and debug methods elsewhere. ip_allow: @@ -50,3 +51,26 @@ ip_allow: - PUSH - DELETE - TRACE + # Deny CONNECT tunnels to unspecified, loopback, private, and link-local + # destination addresses by default. If you intentionally proxy CONNECT to + # one of these ranges, add a more specific outbound allow rule before this + # one. + - apply: out + ip_addrs: + - 0.0.0.0/8 + - 127.0.0.0/8 + - "::" + - ::1 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 169.254.0.0/16 + - ::/96 + - fc00::/7 + - fe80::/10 + # IPv4-compatible IPv6 ranges use ::/96. IPv4-mapped IPv6 ranges use + # hexadecimal notation because the range parser does not accept + # embedded dotted-quad tails. + - ::ffff:0:0/96 + action: deny + methods: CONNECT diff --git a/doc/admin-guide/configuration/index.en.rst b/doc/admin-guide/configuration/index.en.rst index af3d1fe7c7b..520e0e95430 100644 --- a/doc/admin-guide/configuration/index.en.rst +++ b/doc/admin-guide/configuration/index.en.rst @@ -34,3 +34,4 @@ Proxy Cache Configuration hierarchical-caching.en proxy-protocol.en hrw4u.en + regex-best-practices.en diff --git a/doc/admin-guide/configuration/regex-best-practices.en.rst b/doc/admin-guide/configuration/regex-best-practices.en.rst new file mode 100644 index 00000000000..adae135db0d --- /dev/null +++ b/doc/admin-guide/configuration/regex-best-practices.en.rst @@ -0,0 +1,423 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _admin-regex-best-practices: + +Writing Secure Regex Rules +************************** + +Several |TS| configuration files and plugins accept regular expressions +for matching incoming requests. When the regex result drives a security +or routing decision (rule selection, ACL allow/deny, signature +exclusion, parent selection, SNI routing, and so on), the way the +operator writes the regex matters: an unanchored or partially-anchored +pattern can match more inputs than the operator intended, including +inputs crafted by clients to fire a rule that was meant to apply to +something else. + +This page documents: + +- the regex matching contract used at security-sensitive call sites, +- the input subject each call site matches against, +- common pitfalls that produce over-matching patterns, and +- recommended pattern shapes for each site. + +.. contents:: + :local: + :depth: 2 + +The matching contract +===================== + +|TS| uses PCRE2 for regex matching. Patterns are compiled once at +config load and reused per request. By default, a successful match +means the pattern matched **some substring** of the input subject — +possibly the whole subject, possibly a prefix, possibly a fragment in +the middle. The matcher does not, by default, require the pattern to +consume the entire subject. + +This is the standard PCRE behavior. It has security implications when +the subject is influenced by a client (a request URL, a host header, +an SNI value, a Referer header, etc.). An operator who writes the +pattern ``cdn\.example\.com`` as a host regex thinking *"match this +exact host"* will find the rule firing on client-supplied hosts like +``cdn.example.com.other.org``, because the substring +``cdn.example.com`` appears at position 0 of the longer subject. + +The fix at the operator level is to anchor patterns explicitly so the +pattern says exactly what it should match: + +- ``^pattern$`` — match exactly the full input +- ``^pattern`` — match anything starting with ``pattern`` +- ``pattern$`` — match anything ending with ``pattern`` +- ``^.*pattern.*$`` — match anything containing ``pattern`` (when + substring matching is the actual intent) + +Anchored patterns produce predictable behavior regardless of the +matcher's defaults at any given site. Unanchored or single-anchored +patterns may behave one way today and another way after future +changes; the only durable approach is to write patterns whose intent +is unambiguous from the pattern itself. + +Subjects by call site +===================== + +The "subject" — the string the regex is matched against — varies by +site. Knowing the subject is essential for writing correct patterns, +because the same regex against different subjects gives different +results. + +remap.config — ``regex_map`` +---------------------------- + +**Subject:** request host (no scheme, port, or path). + +For URL ``http://cdn.example.com:8080/path``, the regex sees the +subject ``cdn.example.com``. + +remap.config — ``map_with_referer`` +----------------------------------- + +**Subject:** the value of the ``Referer`` HTTP header (header value +only, not including the ``Referer:`` field name or trailing CRLF). + +For header ``Referer: https://www.partner.com/page``, the regex sees +the subject ``https://www.partner.com/page``. + +parent.config — ``url_regex`` +----------------------------- + +**Subject:** the full request URL including scheme, host, and path. + +For URL ``http://example.com/news/politics/today``, the regex sees the +subject ``http://example.com/news/politics/today``. + +cache.config — ``host_regex`` +----------------------------- + +**Subject:** request host (no scheme, port, or path) — same as +``regex_map``. + +cache.config — ``url_regex`` +---------------------------- + +**Subject:** the full request URL — same as ``parent.config``'s +``url_regex``. + +splitdns.config — ``url_regex`` +------------------------------- + +**Subject:** the full request URL — same as ``parent.config``'s +``url_regex``. + +url_sig — ``excl_regex`` +------------------------ + +**Subject:** the full request URL, sliced before the first ``?`` or +``#``. + +For URL ``http://host/path?query``, the regex sees the subject +``http://host/path``. + +maxmind_acl — country regex +--------------------------- + +**Subject:** ``host + "/" + path`` (no scheme, no query string). + +For URL ``http://example.com/file.txt``, the regex sees the subject +``example.com/file.txt``. + +geoip_acl — country regex +------------------------- + +**Subject:** the request URL path returned by ``TSUrlPathGet``, which +does **not** include the leading ``/``. + +For URL ``http://example.com/song.mp3``, the regex sees the subject +``song.mp3``. For URL ``http://example.com/foo/song.mp3``, the regex +sees the subject ``foo/song.mp3``. + +tls_bridge — SNI routing +------------------------ + +**Subject:** the SNI value from the inbound TLS ClientHello. + +Patterns at this site are start-anchored at compile time by the +plugin, so prefix injection is blocked. Operators should still add the +end-anchor ``$`` to block trailing-content matches. + +uri_signing — ``cdniuc`` regex match +------------------------------------ + +.. note:: + The "matching contract" section at the top of this document + describes |TS|'s general regex behavior, which is implemented + with PCRE2. This call site is the exception: ``uri_signing`` + uses the GNU ``regex.h`` library's ``re_match``, which is + start-anchored by definition (the pattern always matches + starting at offset 0 of the subject). A leading ``^`` in an + issuer pattern is therefore implicit, and only the trailing + anchor ``$`` controls whether trailing content is allowed — + the opposite of PCRE2's default substring match. + +**Subject:** the normalized request URI (scheme, authority, path, and +query — produced by the plugin's URI-normalization step). + +Unlike the other sites in this document, the regex pattern at this +site is **not operator-controlled**: it is the value of an inbound +JWT's ``cdniuc`` claim, in the form ``regex:``, and is +chosen by the token issuer. +The ATS operator is the verifier, not the author of the pattern. An +unanchored issuer pattern that names a directory prefix (for example +``https://media\.example\.com/preview/``) will accept any request URI +that *starts* with that prefix — including URIs the issuer did not +intend the token to authorize. Operators who deploy ``uri_signing`` +should require their token issuers to fully anchor every ``regex:`` +container with ``^...$`` and confirm the issuer rejects unanchored +patterns at token-mint time. + +The CDNI URI Signing draft itself does not explicitly require any +anchoring — it states only that the URI must match the regex, leaving +"match" open to interpretation. |TS| performs start-anchoring (the +GNU ``re_match`` API used internally is start-anchored by definition) +but does not enforce end-anchoring; other CDNI relying parties may +interpret the contract differently. Fully anchored issuer patterns +are the one shape every relying party agrees on, so they remove +cross-implementation ambiguity in addition to bounding the issuer's +intended scope. + +Common pitfalls +=============== + +Bare-token patterns without anchors +----------------------------------- + +A pattern like ``\.pdf`` without any anchor matches anywhere in the +input. Against the subject ``http://host/protected.pdf.alternate``, +the substring ``.pdf`` appears at the expected position, so the rule +fires — even though the URL does not actually end in ``.pdf``. + +**Recommended:** ``.*\.pdf$`` (suffix anchor) for "URLs ending in +``.pdf``", or ``^http://[^?#]*\.pdf$`` (full anchor) for full-URL +matching against a subject that includes the scheme. + +DNS-label boundaries +-------------------- + +A pattern like ``cdn\.example\.com`` matches anywhere in the host +subject. Against ``cdn.example.com.other.org``, the substring matches +at position 0 and the rule fires — but the operator probably meant +"the exact host ``cdn.example.com``." + +**Recommended:** ``^cdn\.example\.com$`` for an exact-host match, or +``^.*\.example\.com$`` for "any subdomain of ``example.com``." Note +that simply adding the start-anchor ``^`` is not enough: +``^cdn\.example\.com`` (start-anchored only) still matches +``cdn.example.com.other.org`` because the start matches and the end +is unanchored. + +Forgetting that the subject excludes the scheme +----------------------------------------------- + +For ``regex_map``, ``cache.config`` ``host_regex``, ``maxmind_acl``, +and ``geoip_acl``, the subject does **not** include the URL scheme +(``http://``). Patterns that try to match a leading ``http://`` will +never fire at these sites; check the *Subjects by call site* section +above before writing the pattern. + +Forgetting that ``geoip_acl`` strips the leading slash +------------------------------------------------------ + +For ``geoip_acl``, the subject is the URL path returned by +``TSUrlPathGet``, which strips the leading ``/``. A pattern like +``/songs/.*\.mp3`` will never fire against this subject; use +``^songs/.*\.mp3$`` instead. + +Suffix-only anchoring may not express the full operator intent +-------------------------------------------------------------- + +A pattern like ``\.pdf$`` does match subjects that end in ``.pdf`` — +PCRE scans forward through the subject and the suffix anchor is +satisfied at end-of-input. The pattern works. + +What the pattern does *not* do is constrain the rest of the subject: +it matches ``http://example.com/file.pdf`` and ``cdn.example.com/file.pdf`` +and ``a.pdf``, regardless of the rest of the input. At sites whose +subject is path-only or host-only (for example, ``geoip_acl`` or +``cache.config`` ``host_regex``), this can match more contexts than +the operator had in mind. + +For clarity, prefer patterns that document the full operator intent. +``^https?://.*\.pdf$`` is more verbose than ``\.pdf$`` but makes the +expected subject shape (an HTTP/HTTPS URL ending in ``.pdf``) +self-documenting and resistant to surprise if the same pattern is +copied to a different site whose subject shape differs. + +Recommended pattern cookbook +============================ + +The following table gives a recommended pattern shape for each +common operator intent at each site. Replace the example tokens +(``example.com``, ``politics``, ``.pdf``, etc.) with the values for +your deployment. + +remap.config — ``regex_map`` +---------------------------- + +Subject: host only. + +============================================================ ================================================================= +Operator intent Recommended pattern +============================================================ ================================================================= +Match the exact host ``cdn.example.com`` ``^cdn\.example\.com$`` +Match any subdomain of ``example.com`` ``^.*\.example\.com$`` +Match any host containing ``example`` ``^.*example.*$`` +============================================================ ================================================================= + +remap.config — ``map_with_referer`` +----------------------------------- + +Subject: full Referer header value. + +============================================================ ================================================================= +Operator intent Recommended pattern +============================================================ ================================================================= +Match Referer values from any subdomain of ``partner.com`` ``^https?://[^/]*\.partner\.com(/.*)?$`` +Match the exact Referer ``https://www.partner.com/`` ``^https://www\.partner\.com/$`` +Match any Referer containing ``partner`` ``^.*partner.*$`` +============================================================ ================================================================= + +parent.config — ``url_regex`` +----------------------------- + +Subject: full URL including scheme, host, and path. + +============================================================ ================================================================= +Operator intent Recommended pattern +============================================================ ================================================================= +Match URLs whose path starts with ``/news/politics/`` ``^http://[^/]+/news/politics/.*$`` +Match any URL containing ``politics`` ``^.*politics.*$`` +Match the exact URL ``http://example.com/index.html`` ``^http://example\.com/index\.html$`` +============================================================ ================================================================= + +cache.config — ``host_regex`` +----------------------------- + +Subject: host only. + +Use the same patterns as ``regex_map``. + +cache.config — ``url_regex`` +---------------------------- + +Subject: full URL. + +Use the same patterns as ``parent.config`` ``url_regex``. + +splitdns.config — ``url_regex`` +------------------------------- + +Subject: full URL. + +Use the same patterns as ``parent.config`` ``url_regex``. + +url_sig — ``excl_regex`` +------------------------ + +Subject: full URL sliced before ``?`` or ``#``. + +============================================================ ================================================================= +Operator intent Recommended pattern +============================================================ ================================================================= +Exclude a fixed set of paths from signature checks ``^https?://[^?#]*(/crossdomain\.xml|/clientaccesspolicy\.xml)$`` +Exclude any URL ending in ``.pdf`` ``^https?://[^?#]*\.pdf$`` +Exclude every URL under ``/public/`` ``^https?://[^?#]+/public/.*$`` +============================================================ ================================================================= + +maxmind_acl — country regex +--------------------------- + +Subject: ``host + "/" + path``. + +============================================================ ================================================================= +Operator intent Recommended pattern +============================================================ ================================================================= +Match any URL ending in ``.txt`` ``^.*\.txt$`` +Match any URL under ``example.com`` ``^example\.com/.*$`` +Match a specific path ``example.com/file.txt`` ``^example\.com/file\.txt$`` +============================================================ ================================================================= + +geoip_acl — country regex +------------------------- + +Subject: URL path with leading slash stripped. + +============================================================ ================================================================= +Operator intent Recommended pattern +============================================================ ================================================================= +Match any path ending in ``.mp3`` ``^.*\.mp3$`` +Match files under ``songs/`` ``^songs/.*$`` +Match a specific path ``songs/track.mp3`` ``^songs/track\.mp3$`` +============================================================ ================================================================= + +tls_bridge — SNI routing +------------------------ + +Subject: SNI value. + +============================================================ ================================================================= +Operator intent Recommended pattern +============================================================ ================================================================= +Match the exact SNI ``svc.example.com`` ``^svc\.example\.com$`` +Match any SNI under ``.example.com`` ``^.*\.example\.com$`` +============================================================ ================================================================= + +uri_signing — ``cdniuc`` regex match +------------------------------------ + +Subject: normalized request URI (full URI, including scheme, authority, +path, and query). + +Patterns are issuer-authored; the table below describes shapes the +issuer should use when minting tokens, not patterns the ATS operator +edits directly. + +============================================================ ================================================================= +Issuer intent Recommended pattern +============================================================ ================================================================= +Authorize a single fixed asset ``^https://media\.example\.com/clip\.m3u8$`` +Authorize a single fixed asset, allow any query string ``^https://media\.example\.com/clip\.m3u8(\?.*)?$`` +Authorize every asset under ``/preview/`` ``^https://media\.example\.com/preview/.*$`` +Authorize one named asset under ``/preview/`` ``^https://media\.example\.com/preview/clip\.m3u8$`` +============================================================ ================================================================= + +Why this matters +================ + +Operator-written regex rules can become security boundaries when they +gate behavior such as access control, signature verification, +upstream selection, or DNS routing. A regex that is permissive in +ways the operator did not intend can let traffic through that the +operator meant to block, route to an unintended upstream, or skip a +verification step that should have applied. + +Auditing existing regex rules to confirm they match exactly what the +operator intends — and no more — is a one-time cost that pays off +in predictable rule behavior across upgrades, configuration changes, +and unexpected client inputs. diff --git a/doc/admin-guide/files/cache.config.en.rst b/doc/admin-guide/files/cache.config.en.rst index c75fcaf72f9..14352962c65 100644 --- a/doc/admin-guide/files/cache.config.en.rst +++ b/doc/admin-guide/files/cache.config.en.rst @@ -82,13 +82,19 @@ which the caching rule will apply. .. _cache-config-format-dest-host-regex: ``host_regex`` - A regular expression to be tested against the destination host name in the - request. + A regular expression matched against the request host name (no scheme, + port, or path). .. _cache-config-format-url-regex: ``url_regex`` - A regular expression to be tested against the URL in the request. + A regular expression matched against the full request URL. + +.. seealso:: + Operator-written regex rules can match more inputs than intended when + the pattern is unanchored. For per-site subject definitions, common + pitfalls, and recommended pattern shapes, see + :ref:`admin-regex-best-practices`. Secondary Specifiers -------------------- diff --git a/doc/admin-guide/files/ip_allow.yaml.en.rst b/doc/admin-guide/files/ip_allow.yaml.en.rst index bfb840da17b..36e785b4123 100644 --- a/doc/admin-guide/files/ip_allow.yaml.en.rst +++ b/doc/admin-guide/files/ip_allow.yaml.en.rst @@ -69,6 +69,33 @@ Format - PUSH - DELETE - TRACE + - apply: out + ip_addrs: + - 0.0.0.0/8 + - 127.0.0.0/8 + - "::" + - ::1 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 169.254.0.0/16 + - ::/96 + - fc00::/7 + - fe80::/10 + - ::ffff:0:0/96 + action: deny + methods: CONNECT + +.. important:: + + Upgrade note: newly generated default :file:`ip_allow.yaml` files deny outbound + ``CONNECT`` tunnels to unspecified, loopback, private, link-local, + IPv4-compatible IPv6, and IPv4-mapped IPv6 destination ranges. Existing custom files are not + rewritten, but forward proxy ``CONNECT`` requests and SNI ``tunnel_route`` targets + in these ranges require an explicit ``apply: out`` allow rule before the default + deny rule. Outbound :file:`ip_allow.yaml` policy also applies to ``forward_route`` + and ``partial_blind_route`` upstream destinations, so configure explicit policy for + intentional local or private routes. Each rule is a mapping. The YAML data must have a top level key of "ip_allow" and its value must be a mapping or a sequence of mappings, each of those being one rule. @@ -144,17 +171,17 @@ loopback address because the latter is matched first. A major difference in application between ``in`` and ``out`` rules is that by default, inbound connections are denied and therefore if there is no rule that matches, the connection is -denied. Outbound rules allow by default, so the absence of rules in the default configuration -enables all methods for all outbound connections. +denied. Outbound rules allow by default if no rule matches. The default configuration includes +explicit outbound rules that deny ``CONNECT`` tunnels to unspecified, loopback, private, +link-local, and IPv4-mapped IPv6 destination addresses. .. note:: - Be aware that ip_allow rules will not, and indeed cannot, be applied to TLS - connections which are tunneled via ``tunnel_route`` to the upstream target. - Such connections are not decrypted and thus are not processed by |TS|. This - applies as well to TLS connections which are forwarded via ``forward_route`` - since, while those are decrypted, they are not processed by |TS|. For - details, see :ref:`sni-routing` and :file:`sni.yaml`. + A ``tunnel_route`` in :file:`sni.yaml` synthesizes a ``CONNECT`` transaction, so outbound + ``ip_allow`` rules are applied to its destination address before |TS| opens the upstream + connection. ``forward_route`` and ``partial_blind_route`` upstream destinations are also + subject to outbound ``ip_allow`` policy. The data flowing through these routes is still not + interpreted as HTTP by |TS|. For details, see :ref:`sni-routing` and :file:`sni.yaml`. Timing ====== @@ -219,6 +246,19 @@ Alternatively this can be done with:: Or also by having no rules at all, as outbound by default is allow. +The default configuration denies ``CONNECT`` tunnels to unspecified, loopback, private, +link-local, and IPv4-mapped IPv6 destinations. To intentionally allow a private tunnel +destination, add the more specific outbound rule before the default deny rule:: + + - apply: out + ip_addrs: 127.0.0.1 + action: allow + methods: CONNECT + +Because the first matching IP range selects the rule, an allow rule with a method list denies other +methods to the same destination. If the same destination also receives non-``CONNECT`` traffic, add +those methods to the exception or use ``methods: ALL``. + The following example denies to access all servers on a specific subnet:: apply: out @@ -278,7 +318,13 @@ For the purposes of illustration, here is the default configuration in compact f { apply: in, ip_addrs: 127.0.0.1, action: allow }, { apply: in, ip_addrs: "::1", action: allow }, { apply: in, ip_addrs: 0/0, action: deny, methods: [ PURGE, PUSH, DELETE, TRACE ] }, - { apply: in, ip_addrs: "::/0", action: deny, methods: [ PURGE, PUSH, DELETE, TRACE ] } + { apply: in, ip_addrs: "::/0", action: deny, methods: [ PURGE, PUSH, DELETE, TRACE ] }, + { apply: out, + ip_addrs: [ 0.0.0.0/8, 127.0.0.0/8, "::", "::1", 10.0.0.0/8, 172.16.0.0/12, + 192.168.0.0/16, 169.254.0.0/16, "::/96", "fc00::/7", + "fe80::/10", "::ffff:0:0/96" ], + action: deny, + methods: [ CONNECT ] } ] The following example demonstrates how to use ``ip_categories``. In this example, the diff --git a/doc/admin-guide/files/parent.config.en.rst b/doc/admin-guide/files/parent.config.en.rst index 64fa4d89749..7bace63bb13 100644 --- a/doc/admin-guide/files/parent.config.en.rst +++ b/doc/admin-guide/files/parent.config.en.rst @@ -65,7 +65,13 @@ allowed values. .. _parent-config-format-url-regex: ``url_regex`` - A regular expression (regex) to be found in a URL + A regular expression (regex) matched against the full request URL. + + .. seealso:: + Operator-written regex rules can match more inputs than + intended when the pattern is unanchored. For per-site subject + definitions, common pitfalls, and recommended pattern shapes, + see :ref:`admin-regex-best-practices`. The secondary specifiers are optional in the :file:`parent.config` file. The following list shows the possible secondary specifiers and their allowed diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index 378ae6a6760..96a2dcad765 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -792,6 +792,9 @@ HTTP Engine tr-pass Pass through enabled. mptcp Multipath TCP. allow-plain Allow failback to non-TLS for TLS ports + uds-perm Value Unix domain socket file permission mode. + uds-user Value Unix domain socket file owner name. + uds-group Value Unix domain socket file group name. ============ =============== ======================================== *port* @@ -894,6 +897,41 @@ allow-plain For TLS ports, will fall back to non-TLS processing if the TLS handshake fails. Incompatible with quic ports. +uds-perm + Set the file permission mode applied to a Unix domain socket listener after + ``bind()``. The value is parsed as octal (e.g. ``0660`` or ``660``) and must + be in the range ``0`` to ``0777``. The default is ``0666`` -- read/write for + any local user, matching the connect access of a TCP listener under the + default :file:`ip_allow.yaml`. Tighten it (e.g. ``0660``) together with + ``uds-user`` / ``uds-group`` to restrict which local users may connect. + + Only valid for Unix domain socket ports. + +uds-user + Set the owning user of a Unix domain socket listener file. The value is a + user name resolved via :manpage:`getpwnam(3)`. If ``uds-user`` and / or + ``uds-group`` is specified, |TS| performs :manpage:`chown(2)` on the socket + path; otherwise the ownership inherited from the running process is kept. + + Only valid for Unix domain socket ports. + +uds-group + Set the owning group of a Unix domain socket listener file. The value is a + group name resolved via :manpage:`getgrnam(3)`. See ``uds-user``. + + Only valid for Unix domain socket ports. + +.. important:: + + IP-based access controls -- :file:`ip_allow.yaml` and remap ``@src_ip`` / + ``@src_ip_category`` rules -- are **not** evaluated for connections that + arrive over a Unix domain socket, because the peer has no IP address. + Connections accepted on a UDS listener are subject only to the filesystem + permissions and ownership of the socket file. Use ``uds-perm``, + ``uds-user`` and ``uds-group`` (or external filesystem ACLs on the + containing directory) to restrict which local users may connect, unless an + IP source is supplied through :ts:cv:`proxy.config.acl.subjects`. + .. topic:: Example Listen on port 80 on any address for IPv4 and IPv6.:: @@ -907,6 +945,14 @@ allow-plain /var/run/trafficserver/proxy.sock:pp +.. topic:: Example + + Listen on unix domain socket at /var/run/trafficserver/proxy.sock owned by + user ``trafficserver`` and group ``ats-clients`` with mode ``0660`` so that + only members of ``ats-clients`` can connect:: + + /var/run/trafficserver/proxy.sock:uds-perm=0660:uds-user=trafficserver:uds-group=ats-clients + .. topic:: Example Listen transparently on any IPv4 address on port 8080, and @@ -949,6 +995,10 @@ allow-plain .. note:: These are the ports on the *origin server*, not |TS| :ts:cv:`proxy ports `. + This setting is not a destination host or address policy. Use outbound rules in + :file:`ip_allow.yaml` to control which destination IP addresses may be used for ``CONNECT`` + tunnels. The default :file:`ip_allow.yaml` denies ``CONNECT`` tunnels to unspecified, + loopback, private, link-local, and IPv4-mapped IPv6 destination ranges. .. ts:cv:: CONFIG proxy.config.http.forward_connect_method INT 0 @@ -1109,6 +1159,10 @@ allow-plain for details about chunked trailers. By default, this option is enabled and therefore |TS| will drop chunked trailers. + This option controls HTTP/1.1 chunked trailer handling. HTTP/2 origin + response trailers are not converted to HTTP/1.1 chunked trailers, and |TS| + drops them when the client connection is HTTP/1.x. + .. ts:cv:: CONFIG proxy.config.http.strict_chunk_parsing INT 1 :reloadable: :overridable: @@ -1424,6 +1478,13 @@ allow-plain in a request with the sum of their name and value that exceed this size will cause the entire request to be treated as invalid and rejected by the proxy. + A header field name and value are each stored with a 16-bit length, so each is + limited to 65535 bytes (the maximum a 16-bit length can hold) regardless of this + setting. A value greater than 65535 in records.yaml fails validation and the + default is used; a value set at runtime via :program:`traffic_ctl` is clamped to + 65535. Either way an individual field name or value that exceeds the limit is + rejected rather than truncated. + .. ts:cv:: CONFIG proxy.config.http.request_header_max_size INT 32768 :overridable: :reloadable: @@ -3309,7 +3370,8 @@ HostDB .. ts:cv:: CONFIG proxy.config.hostdb.round_robin_max_count INT 16 - The maximum count of DNS answers per round robin hostdb record. The default variable is 16. + The maximum count of DNS answers per round robin hostdb record. The + default value is ``16``. Valid range is ``1`` to ``1024``. .. ts:cv:: CONFIG proxy.config.hostdb.ttl_mode INT 0 :reloadable: @@ -5161,12 +5223,11 @@ HTTP/2 Configuration frames. Write operation will be triggered at least once every this configured number of millisecond regardless of pending data size. -.. ts:cv:: CONFIG proxy.config.http2.default_buffer_water_mark INT -1 +.. ts:cv:: CONFIG proxy.config.http2.default_buffer_water_mark INT 32768 :reloadable: :units: bytes Specifies the high water mark for all HTTP/2 frames on an outgoing connection. - Default is -1 to preserve existing water marking behavior. You can override this global setting on a per domain basis in the :file:`sni.yaml` file using the :ref:`http2_buffer_water_mark ` attribute. diff --git a/doc/admin-guide/files/remap.config.en.rst b/doc/admin-guide/files/remap.config.en.rst index 73012823734..71b076843d5 100644 --- a/doc/admin-guide/files/remap.config.en.rst +++ b/doc/admin-guide/files/remap.config.en.rst @@ -315,6 +315,13 @@ limitations below: ``regex_map`` you should make sure the reverse path is clear by setting (:ts:cv:`proxy.config.url_remap.pristine_host_hdr`) +.. seealso:: + The ``host`` regex is matched against the request host (no scheme, + port, or path). Operator-written patterns can match more inputs + than intended when unanchored. For per-site subject definitions, + common pitfalls, and recommended pattern shapes, see + :ref:`admin-regex-best-practices`. + Examples -------- @@ -369,6 +376,13 @@ verified. In case an actual request does not have "Referer" header or it does not match with referer regular expression, the HTTP request will be redirected to 'redirect-URL'. +.. seealso:: + Each referer regex is matched against the value of the ``Referer`` + HTTP header (the header value only, not the field name). + Operator-written patterns can match more inputs than intended when + unanchored. For per-site subject definitions, common pitfalls, and + recommended pattern shapes, see :ref:`admin-regex-best-practices`. + At least one regular expressions must be specified in order to activate 'deep linking protection'. There are limitations for the number of referer regular expression strings - 2048. In order to enable the 'deep linking diff --git a/doc/admin-guide/files/sni.yaml.en.rst b/doc/admin-guide/files/sni.yaml.en.rst index 404f7ea62b1..4a4f2c48080 100644 --- a/doc/admin-guide/files/sni.yaml.en.rst +++ b/doc/admin-guide/files/sni.yaml.en.rst @@ -316,11 +316,19 @@ tunnel_route Inbound Destination as an FQDN and po specified in the :ts:cv:`proxy.config.http.connect_ports` configuration in order for the tunnel to succeed. + The destination address is also subject to outbound :file:`ip_allow.yaml` policy. The + default :file:`ip_allow.yaml` denies ``CONNECT`` tunnels to unspecified, loopback, + private, link-local, and IPv4-mapped IPv6 destination addresses. To + intentionally tunnel to one of those ranges, add an explicit outbound allow rule + before the default deny rule. + forward_route Inbound Destination as an FQDN and port, separated by a colon ``:``. This is similar to tunnel_route, but it terminates the TLS connection and forwards the decrypted traffic. |TS| will not interpret the decrypted data, so the contents do not need to be HTTP. + The upstream destination is subject to outbound :file:`ip_allow.yaml` policy before + |TS| opens the connection. partial_blind_route Inbound Destination as an FQDN and port, separated by a colon ``:``. @@ -328,6 +336,8 @@ partial_blind_route Inbound Destination as an FQDN and po In addition partial_blind_route creates a new TLS connection to the specified origin. It does not interpret the decrypted data before passing it to the origin TLS connection, so the contents do not need to be HTTP. + The upstream destination is subject to outbound :file:`ip_allow.yaml` policy before + |TS| opens the connection. tunnel_alpn Inbound List of ALPN Protocol Ids for Partial Blind Tunnel. diff --git a/doc/admin-guide/files/splitdns.config.en.rst b/doc/admin-guide/files/splitdns.config.en.rst index 3f5154eebee..32ad07144c8 100644 --- a/doc/admin-guide/files/splitdns.config.en.rst +++ b/doc/admin-guide/files/splitdns.config.en.rst @@ -71,7 +71,14 @@ The following list describes each field. ``url_regex`` A valid URL regular expression. This specifies that DNS server - selection will be based on a regular expression. + selection will be based on a regular expression matched against + the full request URL. + + .. seealso:: + Operator-written regex rules can match more inputs than + intended when the pattern is unanchored. For per-site subject + definitions, common pitfalls, and recommended pattern shapes, + see :ref:`admin-regex-best-practices`. .. _splitdns-config-format-named: diff --git a/doc/admin-guide/layer-4-routing.en.rst b/doc/admin-guide/layer-4-routing.en.rst index e0996b27e5f..aa777a8110b 100644 --- a/doc/admin-guide/layer-4-routing.en.rst +++ b/doc/admin-guide/layer-4-routing.en.rst @@ -114,6 +114,15 @@ In addition to this, in the :file:`records.yaml` file, edit ``connect_ports`` li - :ts:cv:`proxy.config.http.connect_ports`: ``443 4443`` to allow |TS| to connect to the destination port +If the route target resolves to an unspecified, loopback, private, link-local, or IPv4-mapped IPv6 +address, the default :file:`ip_allow.yaml` outbound policy will deny the synthesized ``CONNECT``. +Add an explicit ``apply: out`` allow rule for the intended destination before the default deny rule. + +``forward_route`` and ``partial_blind_route`` do not blind tunnel the original TLS session, but +their upstream destinations are also subject to outbound :file:`ip_allow.yaml` policy before |TS| +opens the connection. Configure explicit outbound policy for intentional local or private layer-4 +routes. + The sequence of network activity for a Client connecting to ``service-2`` is .. uml:: ../uml/l4-sni-routing-seq.uml diff --git a/doc/admin-guide/plugins/esi.en.rst b/doc/admin-guide/plugins/esi.en.rst index 0c0994ae51d..5ddc0753662 100644 --- a/doc/admin-guide/plugins/esi.en.rst +++ b/doc/admin-guide/plugins/esi.en.rst @@ -95,6 +95,41 @@ Enabling ESI - ``--allowed-response-codes `` specifies a comma-separated list of HTTP response codes that should be processed for ESI transformation. Only responses with these status codes will be examined for ESI content. Default is ``200,304``. +- ``--include-host-allow `` restricts which hostnames may appear in ```` after variable + expansion. The post-expansion hostname (everything between ``://`` and the next ``/``, ``:``, ``?``, or ``#``, with any + ``user@`` prefix and IPv6 brackets stripped) must fully match the PCRE2-syntax regex (case-insensitive). If unset, + scheme/private-host checks still apply but any non-private host is permitted. One exception applies regardless of this + setting (and regardless of ``--allow-private-include-hosts``): a host containing a ``%`` character is always rejected + as ``private-host``. This blocks IPv6 zone IDs (e.g. ``[fe80::1%25eth0]``, which select a network interface and are + only meaningful for link-local addresses) and percent-encoding tricks in the host. Rejected includes increment + ``esi.n_include_errs`` and the offending URL is logged with any ``user:password@`` portion redacted to ``***@``. + Because this is a security control, a regex that fails to compile causes plugin initialization to fail (the plugin + refuses to load with no allowlist rather than silently fail open). + + .. important:: + + The private-host denylist does **not** perform DNS resolution. It only classifies hosts that are IP literals + (e.g. ``10.0.0.1``, ``[fe80::1]``) or localhost-style names (``localhost``, ``*.localhost``). An ordinary DNS + hostname such as ``internal.example.com`` is treated as a non-private host and is permitted when no allowlist is + set, *even if it resolves to a private or link-local address*. This means the built-in denylist alone does not + protect against SSRF via DNS — including DNS rebinding, where a name resolves to a public address at validation + time and a private one when the include is fetched. To constrain ``esi:include`` targets to hosts you trust, you + must configure ``--include-host-allow`` with an explicit allowlist; do not rely on the private-host denylist for + hostname-based SSRF protection. +- ``--allow-private-include-hosts`` disables the default denylist that rejects ``esi:include`` URLs whose host parses to + a non-globally-routable or otherwise reserved IP address. The intent is to fail closed: anything that is not ordinary + public address space is treated as private. For IPv4 this covers the unspecified/"this network" block + (``0.0.0.0/8``), loopback (``127.0.0.0/8``), link-local (``169.254.0.0/16``, including the cloud metadata address + ``169.254.169.254``), RFC 1918 (``10.0.0.0/8``, ``172.16.0.0/12``, ``192.168.0.0/16``), CGNAT (``100.64.0.0/10``), + IETF protocol assignments (``192.0.0.0/24``), the TEST-NET ranges (``192.0.2.0/24``, ``198.51.100.0/24``, + ``203.0.113.0/24``), benchmarking (``198.18.0.0/15``), multicast (``224.0.0.0/4``), reserved (``240.0.0.0/4``), and the + broadcast address (``255.255.255.255``). For IPv6 it covers the unspecified address (``::``), loopback (``::1``), + link-local (``fe80::/10``), unique-local (``fc00::/7``), and multicast (``ff00::/8``); IPv4-mapped (``::ffff:0:0/96``) + and NAT64-encoded (``64:ff9b::/96``) addresses are unwrapped and re-checked against the IPv4 rules above, and any + address carrying a zone id is treated as private. The hostname ``localhost`` (and ``*.localhost``) is also rejected. + Non-canonical numeric IPv4 forms (decimal, octal, hex, or shortcut notations such as ``2130706433`` or ``0x7f000001``) + are rejected outright. Enable this flag only if you intentionally use ESI to assemble responses from internal-IP + backends. Schemes other than ``http`` and ``https`` are always rejected regardless of this flag. 3. ``HTTP_COOKIE`` variable support is turned off by default. It can be turned on with ``-f `` or ``-handler ``. For example: diff --git a/doc/admin-guide/plugins/geoip_acl.en.rst b/doc/admin-guide/plugins/geoip_acl.en.rst index 95d1e8e22b3..8c64e516102 100644 --- a/doc/admin-guide/plugins/geoip_acl.en.rst +++ b/doc/admin-guide/plugins/geoip_acl.en.rst @@ -72,6 +72,13 @@ Note that the default in the case of no matches on the regular expressions is to "allow" the request. This can be overridden, see next use case. +.. seealso:: + The regex is matched against the URL path returned by + ``TSUrlPathGet`` (which excludes the leading ``/``). Operator-written + patterns can match more inputs than intended when unanchored. For + per-site subject definitions, common pitfalls, and recommended + pattern shapes, see :ref:`admin-regex-best-practices`. + 3. You can also combine 1) and 2), and provide defaults in the remap.config configuration, which then applies for the cases where no regular expressions matches at all. This would be useful to override diff --git a/doc/admin-guide/plugins/maxmind_acl.en.rst b/doc/admin-guide/plugins/maxmind_acl.en.rst index 32fc35d199e..7943778c59c 100644 --- a/doc/admin-guide/plugins/maxmind_acl.en.rst +++ b/doc/admin-guide/plugins/maxmind_acl.en.rst @@ -63,9 +63,15 @@ You can mix and match the allow rules and deny rules, however deny rules will al The IP rules can take either single IPs or cidr formatted rules. It will also accept IPv6 IP and ranges. The regex portion can be added to both the allow and deny sections for creating allowable or deniable regexes. Each regex takes a country code first and a regex second. The regex -operates on the entire original request URL, the pre-remapped fqdn and path. +operates on the pre-remap host and path joined as ``host + "/" + path`` (no scheme, no query string). In the above example all requests from the US would be allowed except for those on ``txt`` and ``mp3`` files. More rules should be added as pairs, not as additions to existing lists. +.. seealso:: + Operator-written regex rules can match more inputs than intended + when the pattern is unanchored. For the exact subject this site + matches against, common pitfalls, and recommended pattern shapes, + see :ref:`admin-regex-best-practices`. + Currently the only rules available are ``country``, ``ip``, and ``regex``, though more can easily be added if needed. Each config file does require a top level ``maxmind`` entry as well as a ``database`` entry for the IP lookups. You can supply a separate database for each remap used in case you use custom ones and have specific needs per remap. diff --git a/doc/admin-guide/plugins/uri_signing.en.rst b/doc/admin-guide/plugins/uri_signing.en.rst index ec144e59ead..12086a37b6d 100644 --- a/doc/admin-guide/plugins/uri_signing.en.rst +++ b/doc/admin-guide/plugins/uri_signing.en.rst @@ -86,14 +86,15 @@ like this: "Kabletown URI Authority": { "renewal_kid": "Second Key", "auth_directives": [ - { auth: "allow", uri: "uri-regex:.*crossdomain.xml" }, - { auth: "deny", uri: "uri-regex:https?://[^/]*/public/secret.xml.*" }, - { auth: "allow", uri: "uri-regex:https?://[^/]*/public/.*" }, - { auth: "allow", uri: "uri-regex:.*favicon.ico" } - ] + { "auth": "allow", "uri": "regex:.*crossdomain.xml" }, + { "auth": "deny", "uri": "regex:https?://[^/]*/public/secret.xml.*" }, + { "auth": "allow", "uri": "regex:https?://[^/]*/public/.*" }, + { "auth": "allow", "uri": "regex:.*favicon.ico" } + ], "keys": [ ⋮ ] + } } Each of the ``auth_directives`` will be evaluated in order for each url @@ -130,10 +131,11 @@ ID "id" : "mycdn", "auth_directives": [ ⋮ - ] + ], "keys": [ ⋮ ] + } } Usage @@ -166,6 +168,15 @@ The following claims are understood: * ``cdnistt``: If present, must be 1. * ``cdnistd``: Renewal token cookies will have cdnistd path segments of the request in their path attribute. +.. seealso:: + The ``cdniuc`` claim with kind ``regex`` is matched against the + full normalized request URI. Unanchored issuer patterns can + authorize request URIs the issuer did not intend, because a + pattern that names only a prefix will still satisfy the claim for + any URI that starts with that prefix. For the per-site subject + definition, common pitfalls, and recommended issuer pattern + shapes, see :ref:`admin-regex-best-practices`. + Unsupported Claims ------------------ diff --git a/doc/admin-guide/plugins/url_sig.en.rst b/doc/admin-guide/plugins/url_sig.en.rst index 5faa2930256..973aca257bb 100644 --- a/doc/admin-guide/plugins/url_sig.en.rst +++ b/doc/admin-guide/plugins/url_sig.en.rst @@ -261,6 +261,14 @@ in the configuration file:: pcre regex for urls that aren't signed. default: no regex + .. seealso:: + The exclusion regex is matched against the full request URL + (sliced before any ``?`` or ``#``). Operator-written patterns + can match more inputs than intended when unanchored. For + per-site subject definitions, common pitfalls, and + recommended pattern shapes, see + :ref:`admin-regex-best-practices`. + url_type which url to match against pristine or remap diff --git a/doc/admin-guide/plugins/webp_transform.en.rst b/doc/admin-guide/plugins/webp_transform.en.rst index f729a59df12..2150c54401d 100644 --- a/doc/admin-guide/plugins/webp_transform.en.rst +++ b/doc/admin-guide/plugins/webp_transform.en.rst @@ -32,7 +32,44 @@ Installation Add the following line to :file:`plugin.config`:: - webp_transform.so [convert_to_jpeg,convert_to_webp] + webp_transform.so [convert_to_jpeg] [convert_to_webp] [max_buffer_size=] + + +Plugin Arguments +================ + +The plugin is configured with space-separated arguments. All are optional. + +``convert_to_webp`` + Convert ``image/jpeg`` and ``image/png`` responses to ``image/webp`` for + clients that advertise ``image/webp`` in their ``Accept`` header. + +``convert_to_jpeg`` + Convert ``image/webp`` responses to ``image/jpeg`` for clients that do not + advertise ``image/webp``. + + If neither ``convert_to_webp`` nor ``convert_to_jpeg`` is given, both + conversions are enabled. + +``max_buffer_size=`` + The maximum size of a single response body the plugin will buffer in memory + before handing it to ImageMagick, which bounds the memory a large image + response can consume. ```` is a byte count with an optional ``K``, + ``M``, or ``G`` suffix (1024-based), for example ``max_buffer_size=32M``. + The default is 16 MiB. + + There are two distinct over-limit behaviors, depending on whether the size + is known before the body is read: + + * If the origin advertises a ``Content-Length`` greater than the limit, the + transform is declined up front and the original response is passed through + to the client unchanged. + + * If the size is not known in advance (for example a chunked response with + no usable ``Content-Length``), the body is buffered until it exceeds the + limit. At that point the transform cannot complete and produces no body, + so the client receives a ``502 Bad Gateway`` rather than a pass-through of + the original bytes. Note @@ -40,3 +77,9 @@ Note This plugin only supports jpeg and png and requires Magick++ from ImageMagick. Other image formats can easily be supported. + +In addition to ``max_buffer_size``, the plugin sets fixed ImageMagick decode +resource limits (image dimensions, pixel-cache memory, and disk) so that a +small image declaring very large dimensions cannot decode into an oversized +pixel buffer. An image that exceeds those limits is passed through in its +original format rather than converted. diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldCreate.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldCreate.en.rst index 9319bc55444..d263fdb78dd 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldCreate.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldCreate.en.rst @@ -48,3 +48,9 @@ For both functions a reference to the new field is returned via :arg:`out`. The field created is not in a header even though it is in the same buffer. It can be added to a header with :func:`TSMimeHdrFieldAppend`. The field also has no value, only a name. If a value is needed it must be added explicitly with a function such as :func:`TSMimeHdrFieldValueIntSet`. + +A header field name is stored with a 16-bit length, so :func:`TSMimeHdrFieldCreateNamed` limits +:arg:`name` to ``65535`` (``UINT16_MAX``) bytes. A longer :arg:`name` is rejected: no field is +created, :arg:`out` is set to ``nullptr``, and the function returns :enumerator:`TS_ERROR`. + +These functions return :enumerator:`TS_SUCCESS` on success and :enumerator:`TS_ERROR` on failure. diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.rst index c4739277bc8..e54f880db8c 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldNameSet.en.rst @@ -32,3 +32,14 @@ Synopsis Description =========== + +:func:`TSMimeHdrFieldNameSet` sets the name of the MIME field identified by :arg:`bufp`, +:arg:`hdr`, and :arg:`field` to :arg:`name`. The :arg:`name` is copied into the header +represented by :arg:`bufp` and does not have to be null terminated. :arg:`length` is the length +of :arg:`name`, or ``-1`` if :arg:`name` is null terminated. + +A header field name is stored with a 16-bit length, so it is limited to ``65535`` (``UINT16_MAX``) +bytes. A :arg:`name` longer than that is rejected: the field's existing name is left unchanged and +the function returns :enumerator:`TS_ERROR`. + +This function returns :enumerator:`TS_SUCCESS` if the name was set, :enumerator:`TS_ERROR` if not. diff --git a/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.rst b/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.rst index aef43bb78da..71b0fda163e 100644 --- a/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.rst +++ b/doc/developer-guide/api/functions/TSMimeHdrFieldValueStringSet.en.rst @@ -47,4 +47,8 @@ non-negative it must be the index of an existing element or exactly one past the call will fail. In the example case :arg:`idx` must be between ``0`` and ``3`` inclusive. :func:`TSMimeHdrFieldValuesCount` can be used to get the current number of elements. +A header field value is stored with a 16-bit length, so it is limited to ``65535`` (``UINT16_MAX``) +bytes. A :arg:`value` longer than that is rejected: the field's existing value is left unchanged and +the function returns :enumerator:`TS_ERROR`. + This function returns :enumerator:`TS_SUCCESS` if the value was set, :enumerator:`TS_ERROR` if not. diff --git a/doc/developer-guide/cripts/cripts-connections.en.rst b/doc/developer-guide/cripts/cripts-connections.en.rst index 22549bb66b9..26c656ed682 100644 --- a/doc/developer-guide/cripts/cripts-connections.en.rst +++ b/doc/developer-guide/cripts/cripts-connections.en.rst @@ -101,7 +101,7 @@ beyond string conversion: Method Description ======================= ========================================================================= ``string()`` Convert IP to string with optional CIDR masking. -``Socket()`` Convert IP to a ``sockaddr`` structure for low-level socket operations. +``Socket()`` Convert IP to a ``swoc::IPEndpoint`` (holds IPv4 or IPv6) for low-level socket operations. ``Hasher()`` Generate a hash value for the IP address. ``Sample()`` Determine if IP should be sampled based on rate and seed. ``ASN()`` Get ASN number (if Geo-IP support is available). diff --git a/doc/developer-guide/cripts/cripts-headers.en.rst b/doc/developer-guide/cripts/cripts-headers.en.rst index 66ff13df094..20e1c9bc6f8 100644 --- a/doc/developer-guide/cripts/cripts-headers.en.rst +++ b/doc/developer-guide/cripts/cripts-headers.en.rst @@ -71,6 +71,13 @@ A header can also be removed by using the ``Erase`` method, which is a little mo req.Erase("X-Foo"); +Header names beginning with ``@`` are reserved for internal Traffic Server +metadata. They remain in ATS's in-memory header objects, but they are not +sent on the wire. This makes them useful for trusted in-process +coordination and logging annotations such as ``@TCPInfo``. Treat them as +an internal namespace, not as externally supplied protocol fields. For +the general plugin-facing rules, see :ref:`developer-plugins-http-headers-mime-headers`. + .. note:: There is also a Cripts Bundle for headers, see :ref:`Bundles `. .. _cripts-headers-iterators: diff --git a/doc/developer-guide/plugins/http-headers/mime-headers.en.rst b/doc/developer-guide/plugins/http-headers/mime-headers.en.rst index 817a2c059c7..9b73ba447bf 100644 --- a/doc/developer-guide/plugins/http-headers/mime-headers.en.rst +++ b/doc/developer-guide/plugins/http-headers/mime-headers.en.rst @@ -87,6 +87,26 @@ does not coalesce duplicate fields. Correctly-behaving plugins should check for the presence of duplicate fields and iterate over the duplicate fields by using ``TSMimeHdrFieldNextDup``. +Internal ``@`` Headers +====================== + +Traffic Server reserves header names that begin with ``@`` for internal +use. These headers are stored in the in-memory MIME / HTTP header +structures, but they are not serialized on the wire when Traffic Server +prints a request or response. Core code and plugins use this namespace +for internal metadata, control flags, and logging or debug annotations +such as ``@Ats-Internal``, ``@Content-Type``, ``@ICAP-Status``, and +``@TCPInfo``. + +Plugins may create and read these headers when they need transaction +local state inside Traffic Server, but they should treat them as an +internal-only namespace rather than part of the external HTTP protocol. +In particular, client supplied request headers and origin supplied +response headers whose names begin with ``@`` are stripped before +``TS_HTTP_READ_REQUEST_HDR_HOOK`` and ``TS_HTTP_READ_RESPONSE_HDR_HOOK`` +run, so plugins should not rely on untrusted peers to provide ``@`` +headers. + To facilitate fast comparisons and reduce storage size, Traffic Server defines several pre-allocated field names. These field names correspond to the field names in HTTP and NNTP headers. diff --git a/include/cripts/Bundles/Headers.hpp b/include/cripts/Bundles/Headers.hpp index 431ec53d915..5ef4b387355 100644 --- a/include/cripts/Bundles/Headers.hpp +++ b/include/cripts/Bundles/Headers.hpp @@ -36,18 +36,18 @@ class HRWBridge HRWBridge(const self_type &) = delete; void operator=(const self_type &) = delete; - HRWBridge(const cripts::string_view &str) : _value(str) {} + HRWBridge(const cripts::string_view &str) : _raw(str) {} virtual ~HRWBridge() = default; virtual cripts::string_view - value(cripts::Context * /* context ATS_UNUSED */) + value(cripts::Context * /* context ATS_UNUSED */, cripts::string & /* scratch ATS_UNUSED */) { - return _value; + return _raw; } -protected: - cripts::string _value; +private: + const cripts::string _raw; }; // class HRWBridge diff --git a/include/cripts/Connections.hpp b/include/cripts/Connections.hpp index 88a76e896e2..5d7eccbd08b 100644 --- a/include/cripts/Connections.hpp +++ b/include/cripts/Connections.hpp @@ -22,6 +22,8 @@ #include "ts/apidefs.h" #include "ts/ts.h" +#include "swoc/IPEndpoint.h" + #include "cripts/Lulu.hpp" #include "cripts/Matcher.hpp" @@ -77,8 +79,8 @@ class IP : public swoc::IPAddr uint64_t Hasher(unsigned ipv4_cidr = 32, unsigned ipv6_cidr = 128); bool Sample(double rate, uint32_t seed = 0, unsigned ipv4_cidr = 32, unsigned ipv6_cidr = 128); - // Convert IP to sockaddr structure - [[nodiscard]] sockaddr Socket() const; + // Convert IP to a socket address (sized for both IPv4 and IPv6) + [[nodiscard]] swoc::IPEndpoint Socket() const; // Geo-IP functionality - can be used with any IP address [[nodiscard]] cripts::string ASN() const; diff --git a/include/cripts/Epilogue.hpp b/include/cripts/Epilogue.hpp index ce83f07476c..c7aa5f25b16 100644 --- a/include/cripts/Epilogue.hpp +++ b/include/cripts/Epilogue.hpp @@ -948,18 +948,20 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) // levels of failure here? Non-fatal vs fatal? context->state.error.Execute(context); + bool url_modified = context->_urls.request.Modified(); + // For now, we always allocate the context, but a possible future optimization // could be to use stack allocation when there is only a do_remap() callback. if (!keep_context) { context->Release(); } - // See if the Client URL was modified, which dicates the return code here. - if (context->_urls.request.Modified()) { - context->p_instance.debug("Client::URL was modified, returning TSREMAP_DID_REMAP"); + // Log via the live instance, not the (possibly freed) context. + if (url_modified) { + inst->debug("Client::URL was modified, returning TSREMAP_DID_REMAP"); return TSREMAP_DID_REMAP; } else { - context->p_instance.debug("Client::URL was NOT modified, returning TSREMAP_NO_REMAP"); + inst->debug("Client::URL was NOT modified, returning TSREMAP_NO_REMAP"); return TSREMAP_NO_REMAP; } } diff --git a/include/cripts/Urls.hpp b/include/cripts/Urls.hpp index 3ed143bae2f..8d3e861a718 100644 --- a/include/cripts/Urls.hpp +++ b/include/cripts/Urls.hpp @@ -723,15 +723,21 @@ namespace Client URL(const self_type &) = delete; void operator=(const self_type &) = delete; - // We must not release the bufp etc. since it comes from the RRI structure - // However, we still need to clear cached data in query and path components + // Release/re-init only when we own _urlp: a borrowed remap handle isn't ours, and re-init would deref a dangling RRI. void Reset() override { - query.Reset(); + if (_owns_urlp) { + TSHandleMLocRelease(_bufp, _hdr_loc, _urlp); + _urlp = nullptr; + _bufp = nullptr; + _hdr_loc = nullptr; + _owns_urlp = false; + _initialized = false; + } path.Reset(); - _initialized = false; - _modified = false; + query.Reset(); + _modified = false; } static self_type &_get(cripts::Context *context); @@ -740,6 +746,9 @@ namespace Client protected: void _initialize() override; + private: + bool _owns_urlp = false; + }; // End class Client::URL } // namespace Client diff --git a/include/iocore/eventsystem/IOBuffer.h b/include/iocore/eventsystem/IOBuffer.h index d15fdd81ac6..e3a8bcd0d6b 100644 --- a/include/iocore/eventsystem/IOBuffer.h +++ b/include/iocore/eventsystem/IOBuffer.h @@ -1036,19 +1036,22 @@ class MIOBuffer char * buf_end() { - return first_write_block()->buf_end(); + IOBufferBlock *b = first_write_block(); + return b ? b->buf_end() : nullptr; } char * start() { - return first_write_block()->start(); + IOBufferBlock *b = first_write_block(); + return b ? b->start() : nullptr; } char * end() { - return first_write_block()->end(); + IOBufferBlock *b = first_write_block(); + return b ? b->end() : nullptr; } /** diff --git a/include/iocore/net/AcceptOptions.h b/include/iocore/net/AcceptOptions.h index 53000273412..8f5cc84821a 100644 --- a/include/iocore/net/AcceptOptions.h +++ b/include/iocore/net/AcceptOptions.h @@ -26,6 +26,9 @@ #include "tscore/ink_inet.h" +#include +#include + struct AcceptOptions { using self = AcceptOptions; ///< Self reference type. @@ -36,6 +39,9 @@ struct AcceptOptions { /// If not set -> any address. IpAddr local_ip; UnAddr local_path; + mode_t unix_perm = 0666; + uid_t unix_uid = static_cast(-1); + gid_t unix_gid = static_cast(-1); /// IP address family. /// @note Ignored if an explicit incoming address is set in the /// the configuration (@c local_ip). If neither is set IPv4 is used. diff --git a/include/iocore/net/NetVConnection.h b/include/iocore/net/NetVConnection.h index 4da60535390..4194d468111 100644 --- a/include/iocore/net/NetVConnection.h +++ b/include/iocore/net/NetVConnection.h @@ -487,7 +487,9 @@ class NetVConnection : public VConnection, public PluginUserArgsget_proxy_protocol_addr(ProxyProtocolData::SRC)); + sockaddr const *addr = this->get_proxy_protocol_addr(ProxyProtocolData::SRC); + + return addr == nullptr ? 0 : ats_ip_port_host_order(addr); } sockaddr const * @@ -499,7 +501,9 @@ class NetVConnection : public VConnection, public PluginUserArgsget_proxy_protocol_addr(ProxyProtocolData::DST)); + sockaddr const *addr = this->get_proxy_protocol_addr(ProxyProtocolData::DST); + + return addr == nullptr ? 0 : ats_ip_port_host_order(addr); }; void set_proxy_protocol_info(const ProxyProtocol &src); @@ -712,20 +716,22 @@ inline sockaddr const * NetVConnection::get_effective_remote_addr() { if (pp_info.version != ProxyProtocolVersion::UNDEFINED && is_proxy_protocol_cp_src) { - return get_proxy_protocol_src_addr(); - } else { - return get_remote_addr(); + if (sockaddr const *addr = get_proxy_protocol_src_addr(); addr != nullptr) { + return addr; + } } + + return get_remote_addr(); } inline IpEndpoint const & NetVConnection::get_client_endpoint() { - if (pp_info.version != ProxyProtocolVersion::UNDEFINED && is_proxy_protocol_cp_src) { + if (pp_info.version != ProxyProtocolVersion::UNDEFINED && is_proxy_protocol_cp_src && get_proxy_protocol_src_addr() != nullptr) { return pp_info.src_addr; - } else { - return remote_addr; } + + return get_remote_endpoint(); } /// @return The remote port in host order. diff --git a/include/iocore/net/TLSSNISupport.h b/include/iocore/net/TLSSNISupport.h index 2ce9556f431..3e30f7aaf27 100644 --- a/include/iocore/net/TLSSNISupport.h +++ b/include/iocore/net/TLSSNISupport.h @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -132,17 +133,17 @@ class TLSSNISupport bool would_have_actions_for(const char *servername, IpEndpoint remote, int &enforcement_policy); struct HintsFromSNI { - std::optional http2_buffer_water_mark; - std::optional server_max_early_data; - std::optional http2_initial_window_size_in; - std::optional http2_max_settings_frames_per_minute; - std::optional http2_max_ping_frames_per_minute; - std::optional http2_max_priority_frames_per_minute; - std::optional http2_max_rst_stream_frames_per_minute; - std::optional http2_max_continuation_frames_per_minute; - std::optional ssl_ticket_enabled; - std::optional ssl_ticket_number; - std::optional outbound_sni_policy; + std::optional http2_buffer_water_mark; + std::optional server_max_early_data; + std::optional http2_initial_window_size_in; + std::optional http2_max_settings_frames_per_minute; + std::optional http2_max_ping_frames_per_minute; + std::optional http2_max_priority_frames_per_minute; + std::optional http2_max_rst_stream_frames_per_minute; + std::optional http2_max_continuation_frames_per_minute; + std::optional ssl_ticket_enabled; + std::optional ssl_ticket_number; + std::optional outbound_sni_policy; } hints_from_sni; protected: diff --git a/include/proxy/ControlBase.h b/include/proxy/ControlBase.h index 77cf2d65bd0..128430cf807 100644 --- a/include/proxy/ControlBase.h +++ b/include/proxy/ControlBase.h @@ -31,6 +31,7 @@ #pragma once +#include #include #include "tscore/ink_platform.h" diff --git a/include/proxy/ParentSelection.h b/include/proxy/ParentSelection.h index 7102844d38e..df8003e7484 100644 --- a/include/proxy/ParentSelection.h +++ b/include/proxy/ParentSelection.h @@ -193,20 +193,38 @@ struct ParentResult { const char *url; int port; bool retry; - bool chash_init[MAX_GROUP_RINGS] = {false}; - bool use_pristine = false; - TSHostStatus first_choice_status = TSHostStatus::TS_HOST_STATUS_INIT; - bool do_not_cache_response = false; + bool chash_init[MAX_GROUP_RINGS]; + bool use_pristine; + TSHostStatus first_choice_status; + bool do_not_cache_response; void reset() { - ink_zero(*this); - line_number = -1; + // Public members result = ParentResultType::UNDEFINED; - mapWrapped[0] = false; - mapWrapped[1] = false; + hostname = nullptr; + url = nullptr; + port = 0; + retry = false; + use_pristine = false; + first_choice_status = TSHostStatus::TS_HOST_STATUS_INIT; do_not_cache_response = false; + + // Private members + line_number = -1; + rec = nullptr; + last_parent = 0; + start_parent = 0; + last_group = 0; + wrap_around = false; + last_lookup = 0; + + for (uint32_t i = 0; i < MAX_GROUP_RINGS; ++i) { + chash_init[i] = false; + mapWrapped[i] = false; + chashIter[i] = ATSConsistentHashIter{}; + } } bool @@ -332,7 +350,7 @@ struct ParentResult { uint32_t start_parent; uint32_t last_group; bool wrap_around; - bool mapWrapped[2]; + bool mapWrapped[MAX_GROUP_RINGS]; // state for consistent hash. int last_lookup; ATSConsistentHashIter chashIter[MAX_GROUP_RINGS]; diff --git a/include/proxy/ProxySession.h b/include/proxy/ProxySession.h index 398130bdee4..5eaa6ce0605 100644 --- a/include/proxy/ProxySession.h +++ b/include/proxy/ProxySession.h @@ -152,6 +152,7 @@ class ProxySession : public VConnection, public PluginUserArgs virtual bool support_sni() const; APIHook *hook_get(TSHttpHookID id) const; + bool has_session_hook(TSHttpHookID id) const; HttpAPIHooks const *feature_hooks() const; // Returns null pointer if session does not use a TLS connection. @@ -293,6 +294,12 @@ ProxySession::has_hooks() const return this->api_hooks.has_hooks() || http_global_hooks->has_hooks(); } +inline bool +ProxySession::has_session_hook(TSHttpHookID id) const +{ + return this->hook_get(id) != nullptr || http_global_hooks->get(id) != nullptr; +} + inline SSLProxySession const * ProxySession::ssl() const { diff --git a/include/proxy/ProxyTransaction.h b/include/proxy/ProxyTransaction.h index 32b24c6b5bc..7665392ec50 100644 --- a/include/proxy/ProxyTransaction.h +++ b/include/proxy/ProxyTransaction.h @@ -53,6 +53,10 @@ class ProxyTransaction : public VConnection virtual void cancel_active_timeout(); virtual bool is_read_closed() const; virtual bool expect_send_trailer() const; + /// @return @c true if this transaction can send an HTTP/2 trailer through + /// the delayed trailer tunnel. HTTP/1 chunked trailer pass-through is + /// handled by HttpTunnel and does not use this state query. + virtual bool can_send_h2_trailer() const; virtual void set_expect_send_trailer(); virtual bool expect_receive_trailer() const; virtual void set_expect_receive_trailer(); diff --git a/include/proxy/ReverseProxy.h b/include/proxy/ReverseProxy.h index 201327d8aa2..de18033c8d5 100644 --- a/include/proxy/ReverseProxy.h +++ b/include/proxy/ReverseProxy.h @@ -43,12 +43,14 @@ #include "proxy/http/remap/UrlMapping.h" #include "mgmt/config/ConfigContext.h" +#include "tsutil/AtomicSharedPtr.h" + #define EMPTY_PORT_MAPPING (int32_t) ~0 class url_mapping; struct host_hdr_info; -extern std::atomic rewrite_table; +extern AtomicSharedPtr rewrite_table; // API Functions int init_reverse_proxy(); @@ -61,4 +63,10 @@ bool reloadUrlRewrite(ConfigContext ctx); bool urlRewriteVerify(); void init_remap_volume_host_records(); -int url_rewrite_CB(const char *name, RecDataT data_type, RecData data, void *cookie); + +// Synchronously drops rewrite_table. Call from a Continuation context +// before TSSystemState::shut_down_event_system() so plugin doneInstance() +// has this_ethread() for TSMutexLock. +void shutdown_url_rewrite(); + +int url_rewrite_CB(const char *name, RecDataT data_type, RecData data, void *cookie); diff --git a/include/proxy/hdrs/HTTP.h b/include/proxy/hdrs/HTTP.h index eb73b68eb31..c23f8dc353c 100644 --- a/include/proxy/hdrs/HTTP.h +++ b/include/proxy/hdrs/HTTP.h @@ -486,7 +486,10 @@ class HTTPHdr : public MIMEHdr int valid() const; + // destroy() and clear() name-hide HdrHeapSDKHandle's non-virtual versions + // Dispatch is static; "override" doesn't apply void create(HTTPType polarity, HTTPVersion version = HTTP_INVALID, HdrHeap *heap = nullptr); + void destroy(); void clear(); void reset(); void copy(const HTTPHdr *hdr); @@ -722,6 +725,16 @@ class HTTPHdr : public MIMEHdr @ _fill_target_cache @b always does a cache fill. */ void _test_and_fill_target_cache() const; + /** Null cached pointers/flags without touching the URL or heap. + Shared between @c _reset_local_state() and @c reset(); URL and + heap handling differ between those paths. + */ + void _reset_local_fields(); + /** Null out members that reference the heap. + Shared prologue for @c clear() and @c destroy(); ensures a reused + HTTPHdr can't dereference a stale @c m_host_mime via @c host_get(). + */ + void _reset_local_state(); static Arena *const USE_HDR_HEAP_MAGIC; @@ -759,23 +772,47 @@ HTTPHdr::create(HTTPType polarity, HTTPVersion version, HdrHeap *heap) } inline void -HTTPHdr::clear() +HTTPHdr::_reset_local_fields() +{ + m_http = nullptr; + m_mime = nullptr; + m_host_mime = nullptr; + m_host_length = 0; + m_port = 0; + m_target_cached = false; + m_target_in_url = false; + m_port_in_header = false; +} + +inline void +HTTPHdr::_reset_local_state() { if (m_http && m_http->m_polarity == HTTPType::REQUEST) { m_url_cached.clear(); } + _reset_local_fields(); +} + +inline void +HTTPHdr::clear() +{ + _reset_local_state(); this->HdrHeapSDKHandle::clear(); - m_http = nullptr; - m_mime = nullptr; +} + +inline void +HTTPHdr::destroy() +{ + _reset_local_state(); + this->HdrHeapSDKHandle::destroy(); } inline void HTTPHdr::reset() { m_heap = nullptr; - m_http = nullptr; - m_mime = nullptr; m_url_cached.reset(); + _reset_local_fields(); } /*------------------------------------------------------------------------- diff --git a/include/proxy/hdrs/MIME.h b/include/proxy/hdrs/MIME.h index 7605fc0640b..932217a5e5c 100644 --- a/include/proxy/hdrs/MIME.h +++ b/include/proxy/hdrs/MIME.h @@ -169,10 +169,10 @@ struct MIMEField { time_t value_get_date() const; int value_get_comma_list(StrList *list) const; - void name_set(HdrHeap *heap, MIMEHdrImpl *mh, std::string_view name); + bool name_set(HdrHeap *heap, MIMEHdrImpl *mh, std::string_view name); bool name_is_valid(uint32_t invalid_char_bits = is_control_BIT) const; - void value_set(HdrHeap *heap, MIMEHdrImpl *mh, std::string_view value); + bool value_set(HdrHeap *heap, MIMEHdrImpl *mh, std::string_view value); void value_set_int(HdrHeap *heap, MIMEHdrImpl *mh, int32_t value); void value_set_uint(HdrHeap *heap, MIMEHdrImpl *mh, uint32_t value); void value_set_int64(HdrHeap *heap, MIMEHdrImpl *mh, int64_t value); @@ -763,7 +763,7 @@ MIMEField *mime_hdr_prepare_for_value_set(HdrHeap *heap, MIMEHdrImpl *mh, std::s void mime_field_destroy(MIMEHdrImpl *mh, MIMEField *field); -void mime_field_name_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int16_t name_wks_idx_or_neg1, std::string_view name, +bool mime_field_name_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int16_t name_wks_idx_or_neg1, std::string_view name, bool must_copy_string); int32_t mime_field_value_get_int(const MIMEField *field); @@ -779,12 +779,12 @@ void mime_field_value_delete_comma_val(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField void mime_field_value_extend_comma_val(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int idx, std::string_view new_piece); void mime_field_value_insert_comma_val(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int idx, std::string_view new_piece); -void mime_field_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, std::string_view value, bool must_copy_string); +bool mime_field_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, std::string_view value, bool must_copy_string); void mime_field_value_set_int(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int32_t value); void mime_field_value_set_uint(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, uint32_t value); void mime_field_value_set_int64(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int64_t value); void mime_field_value_set_date(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, time_t value); -void mime_field_name_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int16_t name_wks_idx_or_neg1, +bool mime_field_name_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int16_t name_wks_idx_or_neg1, std::string_view name, std::string_view value, int n_v_raw_printable, int n_v_raw_length, bool must_copy_strings); @@ -837,18 +837,18 @@ bool mime_parse_integer(const char *&buf, const char *end, int *integer); /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ -inline void +inline bool MIMEField::name_set(HdrHeap *heap, MIMEHdrImpl *mh, std::string_view name) { const char *name_wks; if (hdrtoken_is_wks(name.data())) { int16_t name_wks_idx = hdrtoken_wks_to_index(name.data()); - mime_field_name_set(heap, mh, this, name_wks_idx, name, true); + return mime_field_name_set(heap, mh, this, name_wks_idx, name, true); } else { int field_name_wks_idx = hdrtoken_tokenize(name.data(), static_cast(name.length()), &name_wks); - mime_field_name_set(heap, mh, this, field_name_wks_idx, - field_name_wks_idx == -1 ? name : std::string_view{name_wks, name.length()}, true); + return mime_field_name_set(heap, mh, this, field_name_wks_idx, + field_name_wks_idx == -1 ? name : std::string_view{name_wks, name.length()}, true); } } @@ -903,10 +903,10 @@ MIMEField::value_get_comma_list(StrList *list) const /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ -inline void +inline bool MIMEField::value_set(HdrHeap *heap, MIMEHdrImpl *mh, std::string_view value) { - mime_field_value_set(heap, mh, this, value, true); + return mime_field_value_set(heap, mh, this, value, true); } inline void @@ -1533,11 +1533,15 @@ MIMEHdr::get_age() const { int64_t age = value_get_int64(static_cast(MIME_FIELD_AGE)); - if (age < 0) // We should ignore negative Age: values + if (age < 0) { return 0; + } - if ((4 == sizeof(time_t)) && (age > INT_MAX)) // Overflow - return -1; + // RFC 9111 §1.2.2: "the greatest positive integer it can conveniently + // represent" — any Age >= ~68 years is effectively infinity for caching. + if (age >= INT32_MAX) { + return INT32_MAX; + } return age; } @@ -1545,6 +1549,7 @@ MIMEHdr::get_age() const /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ +// Overflow rejected upstream in validate_hdr_content_length (RFC 9112 §6.3). inline int64_t MIMEHdr::get_content_length() const { @@ -1608,6 +1613,7 @@ MIMEHdr::get_if_range_date() const /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ +// RFC 9110 §7.6.2: clamp to implementation max. Saturating mime_parse_int is sufficient. inline int32_t MIMEHdr::get_max_forwards() const { diff --git a/include/proxy/hdrs/XPACK.h b/include/proxy/hdrs/XPACK.h index 6990f866d79..5829a4ac953 100644 --- a/include/proxy/hdrs/XPACK.h +++ b/include/proxy/hdrs/XPACK.h @@ -35,7 +35,7 @@ int64_t xpack_encode_integer(uint8_t *buf_start, const uint8_t *buf_end, uint64_ int64_t xpack_decode_integer(uint64_t &dst, const uint8_t *buf_start, const uint8_t *buf_end, uint8_t n); int64_t xpack_encode_string(uint8_t *buf_start, const uint8_t *buf_end, const char *value, uint64_t value_len, uint8_t n = 7); int64_t xpack_decode_string(Arena &arena, char **str, uint64_t &str_length, const uint8_t *buf_start, const uint8_t *buf_end, - uint8_t n = 7); + uint64_t max_string_len, uint8_t n = 7); struct XpackLookupResult { uint32_t index = 0; diff --git a/include/proxy/http/HttpConfig.h b/include/proxy/http/HttpConfig.h index a0e7ed7d9b7..c09bdddba80 100644 --- a/include/proxy/http/HttpConfig.h +++ b/include/proxy/http/HttpConfig.h @@ -193,6 +193,8 @@ struct HttpStatsBlock { Metrics::Counter::AtomicType *https_total_client_connections; Metrics::Counter::AtomicType *incoming_requests; Metrics::Counter::AtomicType *incoming_responses; + Metrics::Counter::AtomicType *client_request_at_headers_stripped; + Metrics::Counter::AtomicType *origin_response_at_headers_stripped; Metrics::Counter::AtomicType *invalid_client_requests; Metrics::Counter::AtomicType *misc_count; Metrics::Counter::AtomicType *misc_origin_server_bytes; @@ -343,6 +345,7 @@ struct HttpStatsBlock { Metrics::Counter::AtomicType *total_transactions_time; Metrics::Counter::AtomicType *total_x_redirect; Metrics::Counter::AtomicType *trace_requests; + Metrics::Counter::AtomicType *tunnel_chunked_throttle; Metrics::Gauge::AtomicType *tunnel_current_active_connections; Metrics::Counter::AtomicType *tunnels; Metrics::Counter::AtomicType *ua_begin_time; diff --git a/include/proxy/http/HttpSM.h b/include/proxy/http/HttpSM.h index fc3e1252452..c2128eeca20 100644 --- a/include/proxy/http/HttpSM.h +++ b/include/proxy/http/HttpSM.h @@ -33,6 +33,7 @@ #include #include +#include #include "tscore/ink_platform.h" #include "iocore/eventsystem/EventSystem.h" @@ -194,6 +195,11 @@ class HttpSM : public Continuation, public PluginUserArgs HttpCacheSM &get_cache_sm(); // Added to get the object of CacheSM YTS Team, yamsat std::string_view get_outbound_sni() const; std::string_view get_outbound_cert() const; + /// Return the name used by outbound TLS certificate name verification. This + /// usually matches @c get_outbound_sni() above, but adds fallbacks for reuse + /// checks that run after the original NetVCOptions are no longer being + /// built for those reuse situations. + std::string_view get_outbound_sni_for_cert_verification() const; void init(bool from_early_data = false); @@ -306,7 +312,7 @@ class HttpSM : public Continuation, public PluginUserArgs // This unfortunately can't go into the t_state, because of circular dependencies. We could perhaps refactor // this, with a lot of work, but this is easier for now. - UrlRewrite *m_remap = nullptr; + std::shared_ptr m_remap; History history; NetVConnection * @@ -414,7 +420,7 @@ class HttpSM : public Continuation, public PluginUserArgs void do_cache_prepare_write_transform(); void do_cache_prepare_update(); void do_cache_prepare_action(HttpCacheSM *c_sm, CacheHTTPInfo *object_read_info, bool retry, bool allow_multiple = false); - void do_cache_delete_all_alts(Continuation *cont); + void do_cache_delete_all_alts(); void do_auth_callout(); int do_api_callout(); int do_api_callout_internal(); diff --git a/include/proxy/http/HttpSessionManager.h b/include/proxy/http/HttpSessionManager.h index cfb365d5382..6c6523da407 100644 --- a/include/proxy/http/HttpSessionManager.h +++ b/include/proxy/http/HttpSessionManager.h @@ -95,8 +95,11 @@ class ServerSessionPool : public Continuation HSMresult_t acquireSession(sockaddr const *addr, CryptoHash const &host_hash, TSServerSessionSharingMatchMask match_style, HttpSM *sm, PoolableSession *&server_session); /** Release a session to the pool. + + @return @c true if the session was pooled; @c false if the session could not be pooled + (the caller is responsible for closing it in that case). */ - void releaseSession(PoolableSession *ss); + bool releaseSession(PoolableSession *ss); /// Close all sessions and then clear the table. void purge(); diff --git a/include/proxy/http/HttpTransact.h b/include/proxy/http/HttpTransact.h index b918fa0c27e..8ce6ec13bb1 100644 --- a/include/proxy/http/HttpTransact.h +++ b/include/proxy/http/HttpTransact.h @@ -673,29 +673,30 @@ class HttpTransact Arena arena; - bool force_dns = false; - bool is_upgrade_request = false; - bool is_websocket = false; - bool did_upgrade_succeed = false; - bool client_connection_allowed = true; - bool acl_filtering_performed = false; - bool api_cleanup_cache_read = false; - bool api_server_response_no_store = false; - bool api_server_response_ignore = false; - bool api_http_sm_shutdown = false; - bool api_modifiable_cached_resp = false; - bool api_server_request_body_set = false; - bool api_req_cacheable = false; - bool api_resp_cacheable = false; - bool api_server_addr_set_retried = false; - bool reverse_proxy = false; - bool url_remap_success = false; - bool api_skip_all_remapping = false; - bool already_downgraded = false; - bool transparent_passthrough = false; - bool range_in_cache = false; - bool is_method_stats_incremented = false; - bool skip_ip_allow_yaml = false; + bool force_dns = false; + bool is_upgrade_request = false; + bool is_websocket = false; + bool did_upgrade_succeed = false; + bool client_connection_allowed = true; + bool acl_filtering_performed = false; + bool api_cleanup_cache_read = false; + bool api_server_response_no_store = false; + bool api_server_response_ignore = false; + bool api_http_sm_shutdown = false; + bool api_modifiable_cached_resp = false; + bool api_server_request_body_set = false; + bool api_req_cacheable = false; + bool api_resp_cacheable = false; + bool api_server_addr_set_retried = false; + bool host_down_cache_fallback_attempted = false; + bool reverse_proxy = false; + bool url_remap_success = false; + bool api_skip_all_remapping = false; + bool already_downgraded = false; + bool transparent_passthrough = false; + bool range_in_cache = false; + bool is_method_stats_incremented = false; + bool skip_ip_allow_yaml = false; /// True if the response is cacheable because of negative caching configuration. /// @@ -988,6 +989,12 @@ class HttpTransact }; // End of State struct. + enum class AtHeaderSource { + CLIENT_REQUEST, + ORIGIN_RESPONSE, + }; + + static void strip_at_headers(HTTPHdr &header, AtHeaderSource source, std::int64_t sm_id); static void HandleBlindTunnel(State *s); static void StartRemapRequest(State *s); static void EndRemapRequest(State *s); @@ -1094,7 +1101,7 @@ class HttpTransact static bool is_response_valid(State *s, HTTPHdr *incoming_response); static void process_quick_http_filter(State *s, int method); - static bool will_this_request_self_loop(State *s); + static bool will_this_request_self_loop(State *s, bool is_outbound_transparent = false); static bool is_request_likely_cacheable(State *s, HTTPHdr *request); static bool is_cache_hit(CacheLookupResult_t r); static bool is_fresh_cache_hit(CacheLookupResult_t r); @@ -1147,9 +1154,9 @@ using TransactEntryFunc_t = void (*)(HttpTransact::State *); /* The spec says about message body the following: * - * All responses to the HEAD and CONNECT request method - * MUST NOT include a message-body, even though the presence - * of entity-header fields might lead one to believe they do. + * All responses to the HEAD request method MUST NOT include a + * message-body. Successful (2xx) responses to CONNECT MUST NOT + * include a message-body; error responses may. * * All 1xx (informational), 204 (no content), and 304 (not modified) * responses MUST NOT include a message-body. @@ -1171,7 +1178,9 @@ is_response_body_precluded(HTTPStatus status_code) inline bool is_response_body_precluded(HTTPStatus status_code, int method) { - if ((method == HTTP_WKSIDX_HEAD) || (method == HTTP_WKSIDX_CONNECT) || is_response_body_precluded(status_code)) { + if ((method == HTTP_WKSIDX_HEAD) || + (method == HTTP_WKSIDX_CONNECT && status_code >= HTTPStatus::OK && status_code < HTTPStatus::MULTIPLE_CHOICES) || + is_response_body_precluded(status_code)) { return true; } else { return false; diff --git a/include/proxy/http/HttpTunnel.h b/include/proxy/http/HttpTunnel.h index f245a8452f8..6a2867071a2 100644 --- a/include/proxy/http/HttpTunnel.h +++ b/include/proxy/http/HttpTunnel.h @@ -70,19 +70,23 @@ enum class HttpTunnelType_t { HTTP_SERVER, HTTP_CLIENT, CACHE_READ, CACHE_WRITE, enum class TunnelChunkingAction_t { CHUNK_CONTENT, DECHUNK_CONTENT, PASSTHRU_CHUNKED_CONTENT, PASSTHRU_DECHUNKED_CONTENT }; struct ChunkedHandler { + // Grants the unit test fixture access to the private read_size() parser. + friend class TestableChunkedHandler; + enum class ChunkedState { - READ_CHUNK = 0, - READ_SIZE_START, - READ_SIZE, - READ_SIZE_CRLF, - READ_TRAILER_BLANK, - READ_TRAILER_CR, - READ_TRAILER_LINE, - READ_ERROR, - READ_DONE, - WRITE_CHUNK, - WRITE_DONE, - FLOW_CONTROL + READ_CHUNK = 0, ///< Reading the chunk data bytes. + READ_SIZE_START, ///< Expecting the start of the next chunk size line. + READ_SIZE, ///< Reading the hex chunk size digits. + READ_EXTENSION, ///< Reading a chunk extension (;name=value or ;name="quoted-string"). + READ_SIZE_CRLF, ///< Expecting the CRLF that terminates the chunk size line. + READ_TRAILER_BLANK, ///< Expecting the blank line or first trailer field after the last chunk. + READ_TRAILER_CR, ///< Expecting the CR that ends the trailer section. + READ_TRAILER_LINE, ///< Reading a trailer field line. + READ_ERROR, ///< A protocol error was encountered; parsing stops. + READ_DONE, ///< The full chunked body has been read. + WRITE_CHUNK, ///< Writing a chunk while re-chunking content. + WRITE_DONE, ///< Finished writing chunked content. + FLOW_CONTROL ///< Paused for flow control. }; static int const DEFAULT_MAX_CHUNK_SIZE = 4096; @@ -128,6 +132,13 @@ struct ChunkedHandler { int num_cr = 0; bool prev_is_cr = false; + // Chunk extension parsing state. The parser tracks whether it is inside a + // quoted-string extension value (RFC 9110 Section 5.6.4) and whether the + // previous octet began a quoted-pair escape, so it can find the closing DQUOTE + // and reject a CR or LF appearing inside the quoted-string. + bool in_quoted_string = false; + bool in_escape = false; + /// @name Output data. //@{ /// The maximum chunk size. @@ -169,6 +180,11 @@ struct ChunkedHandler { */ std::pair generate_chunked_content(); + /** + * Check if chunked_reader or dechunked_reader has more data to read + */ + bool is_read_avail(); + private: /** Read a chunk header containing the size of the chunk. * @@ -416,7 +432,6 @@ class HttpTunnel : public Continuation void tunnel_run(HttpTunnelProducer *p = nullptr); int main_handler(int event, void *data); - void consumer_reenable(HttpTunnelConsumer *c); bool consumer_handler(int event, HttpTunnelConsumer *c); bool producer_handler(int event, HttpTunnelProducer *p); int producer_handler_dechunked(int event, HttpTunnelProducer *p); @@ -451,6 +466,11 @@ class HttpTunnel : public Continuation void _schedule_tls_tunnel_activity_check_event(); bool _is_tls_tunnel_active() const; + bool _should_reenable_for_chunk_handler_fc(HttpTunnelConsumer *c); + bool _should_reenable_for_tunnel_chain_fc(HttpTunnelConsumer *c); + + HttpTunnelConsumer *_throttle_chunked_producer(HttpTunnelProducer *p); + HttpTunnelProducer *get_producer(VIO *vio); HttpTunnelConsumer *get_consumer(VIO *vio); @@ -622,9 +642,11 @@ HttpTunnel::get_consumer(VConnection *vc) inline HttpTunnelProducer * HttpTunnel::get_producer(VIO *vio) { - for (int i = 0; i < MAX_PRODUCERS; i++) { - if (producers[i].read_vio == vio) { - return producers + i; + if (vio) { + for (int i = 0; i < MAX_PRODUCERS; i++) { + if (producers[i].alive && producers[i].read_vio == vio) { + return producers + i; + } } } return nullptr; diff --git a/include/proxy/http/remap/UrlRewrite.h b/include/proxy/http/remap/UrlRewrite.h index cfe46817d02..e2c3604b554 100644 --- a/include/proxy/http/remap/UrlRewrite.h +++ b/include/proxy/http/remap/UrlRewrite.h @@ -24,7 +24,6 @@ #pragma once -#include "iocore/eventsystem/Freer.h" #include "mgmt/config/ConfigContext.h" #include "proxy/http/remap/UrlMapping.h" #include "proxy/http/remap/UrlMappingPathIndex.h" @@ -57,12 +56,12 @@ enum class mapping_type { /** * **/ -class UrlRewrite : public RefCountObjInHeap +class UrlRewrite { public: using URLTable = std::unordered_map; UrlRewrite() = default; - ~UrlRewrite() override; + ~UrlRewrite(); /** Retrieve the configured ACL matching policy. * @@ -93,26 +92,6 @@ class UrlRewrite : public RefCountObjInHeap void SetReverseFlag(int flag); void Print() const; - // The UrlRewrite object is-a RefCountObj, but this is a convenience to make it clear that we - // don't delete() these objects directly, but via the release() method only. - UrlRewrite * - acquire() - { - this->refcount_inc(); - return this; - } - - void - release() - { - if (0 == this->refcount_dec()) { - // Delete this on an ET_TASK thread, which avoids doing potentially slow things on an ET_NET thread. - static DbgCtl dc{"url_rewrite"}; - Dbg(dc, "Deleting old configuration immediately"); - new_Deleter(this, 0); - } - } - bool is_valid() const { diff --git a/include/proxy/http2/HPACK.h b/include/proxy/http2/HPACK.h index b8edd3e01fa..f9fcc0a9dc6 100644 --- a/include/proxy/http2/HPACK.h +++ b/include/proxy/http2/HPACK.h @@ -71,16 +71,16 @@ class MIMEFieldWrapper { public: MIMEFieldWrapper(MIMEField *f, HdrHeap *hh, MIMEHdrImpl *impl) : _field(f), _heap(hh), _mh(impl) {} - void + bool name_set(const char *name, int name_len) { - _field->name_set(_heap, _mh, std::string_view{name, static_cast(name_len)}); + return _field->name_set(_heap, _mh, std::string_view{name, static_cast(name_len)}); } - void + bool value_set(const char *value, int value_len) { - _field->value_set(_heap, _mh, std::string_view{value, static_cast(value_len)}); + return _field->value_set(_heap, _mh, std::string_view{value, static_cast(value_len)}); } std::string_view @@ -143,14 +143,14 @@ int64_t encode_literal_header_field_with_new_name(uint8_t *buf_start, const uint int64_t decode_indexed_header_field(MIMEFieldWrapper &header, const uint8_t *buf_start, const uint8_t *buf_end, HpackIndexingTable &indexing_table); int64_t decode_literal_header_field(MIMEFieldWrapper &header, const uint8_t *buf_start, const uint8_t *buf_end, - HpackIndexingTable &indexing_table); + HpackIndexingTable &indexing_table, uint32_t header_field_max_size); int64_t update_dynamic_table_size(const uint8_t *buf_start, const uint8_t *buf_end, HpackIndexingTable &indexing_table, uint32_t maximum_table_size); // High level interfaces using HpackHandle = HpackIndexingTable; int64_t hpack_decode_header_block(HpackHandle &handle, HTTPHdr *hdr, const uint8_t *in_buf, const size_t in_buf_len, - uint32_t max_header_size, uint32_t maximum_table_size); + uint32_t max_header_size, uint32_t maximum_table_size, uint32_t header_field_max_size); int64_t hpack_encode_header_block(HpackHandle &handle, uint8_t *out_buf, const size_t out_buf_len, HTTPHdr *hdr, int32_t maximum_table_size = -1); int32_t hpack_get_maximum_table_size(HpackHandle &handle); diff --git a/include/proxy/http2/HTTP2.h b/include/proxy/http2/HTTP2.h index a8923924b6c..370bbbef929 100644 --- a/include/proxy/http2/HTTP2.h +++ b/include/proxy/http2/HTTP2.h @@ -373,8 +373,20 @@ bool http2_parse_goaway(IOVec, Http2Goaway &); bool http2_parse_window_update(IOVec, uint32_t &); +// Returns true if appending `payload_length` more bytes to an +// existing CONTINUATION header-block accumulator of `current_length` would +// overflow a uint32_t. Used by Http2ConnectionState::rcv_continuation_frame() +// to reject crafted CONTINUATION chains whose total payload would wrap the +// 32-bit accumulator and pair an undersized ats_realloc() with a memcpy at +// the pre-wrap offset. +static inline bool +http2_continuation_length_would_overflow(uint32_t current_length, uint32_t payload_length) +{ + return current_length > UINT32_MAX - payload_length; +} + Http2ErrorCode http2_decode_header_blocks(HTTPHdr *, const uint8_t *, const uint32_t, uint32_t *, HpackHandle &, bool, uint32_t, - bool is_outbound = false); + uint32_t, bool is_outbound = false); Http2ErrorCode http2_encode_header_blocks(HTTPHdr *, uint8_t *, uint32_t, uint32_t *, HpackHandle &, int32_t); diff --git a/include/proxy/http2/Http2ConnectionState.h b/include/proxy/http2/Http2ConnectionState.h index 53054416d9e..849451df6fe 100644 --- a/include/proxy/http2/Http2ConnectionState.h +++ b/include/proxy/http2/Http2ConnectionState.h @@ -166,14 +166,19 @@ class Http2ConnectionState : public Continuation bool send_push_promise_frame(Http2Stream *stream, URL &url, const MIMEField *accept_encoding); void send_rst_stream_frame(Http2StreamId id, Http2ErrorCode ec); + static constexpr bool SEND_EMPTY = true; + /** Send a SETTINGS frame to the peer. * * local_settings is updated to the value of @a new_settings as a byproduct * of this call. * * @param[in] new_settings The settings to send to the peer. + * @param[in] send_empty Whether to send a SETTINGS frame if @a new_settings + * has no changes from the current local settings. + * @return The result of sending the SETTINGS frame. */ - void send_settings_frame(const Http2ConnectionSettings &new_settings); + Http2Error send_settings_frame(const Http2ConnectionSettings &new_settings, bool send_empty); void send_ping_frame(Http2StreamId id, uint8_t flag, const uint8_t *opaque_data); void send_goaway_frame(Http2StreamId id, Http2ErrorCode ec); @@ -242,12 +247,59 @@ class Http2ConnectionState : public Continuation unsigned _adjust_concurrent_stream(); - /** Receive and process a SETTINGS frame with the ACK flag set. + /** Process an incoming SETTINGS ACK frame. + * + * Applies the settings from the front of the outstanding settings queue, + * updating acknowledged_local_settings and adjusting stream receive + * windows if INITIAL_WINDOW_SIZE changed. + * + * @return The result of processing the SETTINGS ACK. + */ + Http2Error _process_incoming_settings_ack_frame(); + + /** Check whether new settings differ from current local settings. + * + * This determines whether @a new_settings would require sending a non-empty + * SETTINGS frame, giving frame-suppression and outstanding-frame limit + * checks a shared definition of a settings change. + * + * @param[in] new_settings The candidate settings to compare. + * @return @c true if any emitted setting differs from current local + * settings. + */ + bool _settings_have_changes(const Http2ConnectionSettings &new_settings) const; + + /** Check whether a SETTINGS frame can be sent without exceeding local debt. + * + * A SETTINGS frame that would carry no settings and is not required to be + * sent is treated as sendable because it will be skipped. + * + * @param[in] new_settings The settings to send to the peer. + * @param[in] send_empty Whether to send an empty SETTINGS frame. + * @return The result of checking whether the SETTINGS frame can be sent. + */ + Http2Error _check_outgoing_settings_frame(const Http2ConnectionSettings &new_settings, bool send_empty) const; + + /** Calculate the maximum SETTINGS frames that can await peer ACKs. + * + * The limit bounds the memory used to retain local settings snapshots while + * still allowing the connection preface and one full concurrent-stream wave + * of dynamic stream window updates. + * + * @return The number of outstanding SETTINGS frames allowed on this + * connection. + */ + size_t _get_outstanding_settings_frame_limit() const; + + /** Signal a connection-level HTTP/2 error and schedule session shutdown. * - * This function will process any settings updates that have now been - * acknowledged by the peer. + * This sends GOAWAY once, marks the session half-closed locally so no new + * streams are accepted, and schedules finalization if it is not already + * pending. + * + * @param[in] error_code The HTTP/2 error code to send in GOAWAY. */ - void _process_incoming_settings_ack_frame(); + void _close_connection(Http2ErrorCode error_code); // Getters for stream control configurations that retrieve the inbound or // outbound values per the configured session. @@ -419,6 +471,8 @@ class Http2ConnectionState : public Continuation int32_t configured_max_rst_stream_frames_per_minute = 0; int32_t configured_max_continuation_frames_per_minute = 0; int32_t configured_max_empty_frames_per_minute = 0; + + uint32_t _header_field_max_size = 32768; }; /////////////////////////////////////////////// diff --git a/include/proxy/http2/Http2Stream.h b/include/proxy/http2/Http2Stream.h index bc4b0743c51..dbf64edb4b9 100644 --- a/include/proxy/http2/Http2Stream.h +++ b/include/proxy/http2/Http2Stream.h @@ -77,11 +77,12 @@ class Http2Stream : public ProxyTransaction void do_io_close(int lerrno = -1) override; bool expect_send_trailer() const override; + bool can_send_h2_trailer() const override; void set_expect_send_trailer() override; bool expect_receive_trailer() const override; void set_expect_receive_trailer() override; - Http2ErrorCode decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size); + Http2ErrorCode decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size, uint32_t header_field_max_size); void send_headers(Http2ConnectionState &cstate); void initiating_close(); bool is_outbound_connection() const; @@ -221,8 +222,9 @@ class Http2Stream : public ProxyTransaction #else MIOBuffer _receive_buffer{BUFFER_SIZE_INDEX_4K}; #endif - VIO read_vio; - VIO write_vio; + VIO read_vio; + VIO write_vio; + bool _read_event_paused = false; ///< The read VIO is intentionally gated by a zero-byte read. History _history; Milestones(Http2StreamMilestone::LAST_ENTRY)> _milestones; @@ -446,7 +448,7 @@ Http2Stream::read_vio_writer() const inline bool Http2Stream::is_read_enabled() const { - return !this->read_vio.is_disabled(); + return !this->_read_event_paused && this->read_vio.nbytes != 0 && !this->read_vio.is_disabled(); } inline void @@ -465,5 +467,7 @@ Http2Stream::update_read_length(int count) inline void Http2Stream::set_read_done() { - read_vio.nbytes = read_vio.ndone; + if (!this->_read_event_paused) { + read_vio.nbytes = read_vio.ndone; + } } diff --git a/include/proxy/http3/QPACK.h b/include/proxy/http3/QPACK.h index 4adfb33add2..40cd4a4160c 100644 --- a/include/proxy/http3/QPACK.h +++ b/include/proxy/http3/QPACK.h @@ -48,7 +48,8 @@ enum { class QPACK : public QUICApplication { public: - QPACK(QUICConnection *qc, uint32_t max_field_section_size, uint16_t max_table_size, uint16_t max_blocking_streams); + QPACK(QUICConnection *qc, uint32_t max_field_section_size, uint16_t max_table_size, uint16_t max_blocking_streams, + uint32_t header_field_max_size); virtual ~QPACK(); void on_stream_open(QUICStream &stream) override; @@ -195,6 +196,7 @@ class QPACK : public QUICApplication XpackDynamicTable _dynamic_table; std::map _references; uint32_t _max_field_section_size = 0; + uint32_t _header_field_max_size = 0; uint16_t _max_table_size = 0; uint16_t _max_blocking_streams = 0; diff --git a/include/records/RecHttp.h b/include/records/RecHttp.h index 6f6b24c3283..7fdf2fb6703 100644 --- a/include/records/RecHttp.h +++ b/include/records/RecHttp.h @@ -34,6 +34,8 @@ #include "tscore/ink_assert.h" #include #include +#include +#include /** Load IP addresses from a configuration value. * @@ -281,6 +283,9 @@ struct HttpProxyPort { IpAddr m_inbound_ip; /// Path for listening on unix domain socket UnAddr m_unix_path; + mode_t m_unix_perm = 0666; + uid_t m_unix_uid = static_cast(-1); + gid_t m_unix_gid = static_cast(-1); /// Local address for outbound connections (to origin server). ts::IPAddrPair m_outbound; /// Ordered preference for DNS resolution family ( @c FamilyPrefence ) @@ -437,6 +442,9 @@ struct HttpProxyPort { static const char *const OPT_PROTO_PREFIX; ///< Transport layer protocols. static const char *const OPT_MPTCP; ///< MPTCP. static const char *const OPT_PROXY_PROTO_CLIENT_SRC_IP; ///< The Proxy protocol SRC IP address is used as the client's IP address + static const char *const OPT_UDS_PERM_PREFIX; ///< Prefix for unix domain socket file permission mode. + static const char *const OPT_UDS_USER_PREFIX; ///< Prefix for unix domain socket file owner name. + static const char *const OPT_UDS_GROUP_PREFIX; ///< Prefix for unix domain socket file group name. static std::vector &m_global; ///< Global ("default") data. diff --git a/include/tscore/PendingAction.h b/include/tscore/PendingAction.h index 4650b0629e6..6b3a412602d 100644 --- a/include/tscore/PendingAction.h +++ b/include/tscore/PendingAction.h @@ -138,11 +138,9 @@ inline bool PendingAction::clear_if_action_is(Action *action) { if (action != nullptr) { - while (action == pending_action) { - if (pending_action.compare_exchange_strong(action, nullptr)) { - // do NOT cancel - this is called when the event is handled. - return true; - } + if (pending_action.compare_exchange_strong(action, nullptr)) { + // do NOT cancel - this is called when the event is handled. + return true; } } return false; diff --git a/include/tscore/X509HostnameValidator.h b/include/tscore/X509HostnameValidator.h index 67e2dd29a9c..ba5d97036a0 100644 --- a/include/tscore/X509HostnameValidator.h +++ b/include/tscore/X509HostnameValidator.h @@ -24,14 +24,15 @@ #pragma once #include +#include /* * Validate that the certificate is for the specified hostname/IP address * @param cert The X509 certificate we match against - * @param hostname Null terminated string that we want to match + * @param hostname The hostname/IP address that we want to match * @param is_ip Is the specified hostname an IP string * @param peername If not NULL, the matching value from the certificate will allocated and the ptr adjusted. * In this case caller must free afterwards with ats_free */ -bool validate_hostname(X509 *cert, const unsigned char *hostname, bool is_ip, char **peername); +bool validate_hostname(X509 *cert, std::string_view hostname, bool is_ip, char **peername); diff --git a/include/tsutil/AtomicSharedPtr.h b/include/tsutil/AtomicSharedPtr.h new file mode 100644 index 00000000000..c7138a8e575 --- /dev/null +++ b/include/tsutil/AtomicSharedPtr.h @@ -0,0 +1,84 @@ +/** @file + + Atomic wrapper around std::shared_ptr with the C++20 + std::atomic> API. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include + +// Use the C++20 std::atomic> specialization when the +// standard library provides it, otherwise fall back to the pre-C++20 +// std::atomic_*_explicit free-function overloads on shared_ptr. The +// fallback exists for libstdc++ < 12 and libc++ < 14, which predate the +// specialization. When those toolchains are no longer supported, delete +// the #else branch and the surrounding #if; call sites do not change. +#if defined(__cpp_lib_atomic_shared_ptr) && __cpp_lib_atomic_shared_ptr >= 201711L + +template using AtomicSharedPtr = std::atomic>; + +#else + +// Belt-and-suspenders: on the toolchains that take this branch (libstdc++ +// < 12, libc++ < 16) the free-function overloads are not yet marked +// [[deprecated]], so the suppression below is usually a no-op. It +// matters only if someone forces the fallback on a modern library (e.g. +// -D__cpp_lib_atomic_shared_ptr=0) or compiles against a library that +// ships the deprecation markers ahead of the specialization. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + +template class AtomicSharedPtr +{ +public: + AtomicSharedPtr() noexcept = default; + AtomicSharedPtr(std::shared_ptr desired) noexcept : _p(std::move(desired)) {} + + AtomicSharedPtr(const AtomicSharedPtr &) = delete; + AtomicSharedPtr &operator=(const AtomicSharedPtr &) = delete; + + std::shared_ptr + load(std::memory_order order = std::memory_order_seq_cst) const noexcept + { + return std::atomic_load_explicit(&_p, order); + } + + void + store(std::shared_ptr desired, std::memory_order order = std::memory_order_seq_cst) noexcept + { + std::atomic_store_explicit(&_p, std::move(desired), order); + } + + std::shared_ptr + exchange(std::shared_ptr desired, std::memory_order order = std::memory_order_seq_cst) noexcept + { + return std::atomic_exchange_explicit(&_p, std::move(desired), order); + } + +private: + std::shared_ptr _p; +}; + +#pragma GCC diagnostic pop + +#endif diff --git a/include/tsutil/Convert.h b/include/tsutil/Convert.h index b95fc3675a6..2613c9ecc97 100644 --- a/include/tsutil/Convert.h +++ b/include/tsutil/Convert.h @@ -25,6 +25,8 @@ #include "swoc/MemSpan.h" +#include +#include #include namespace ts @@ -40,7 +42,8 @@ transform_lower(std::string_view src, swoc::MemSpan dst) if (src.size() > dst.size() - 1) { // clip @a src, reserving space for the terminal nul. src = std::string_view{src.data(), dst.size() - 1}; } - auto final = std::transform(src.begin(), src.end(), dst.data(), [](char c) -> char { return std::tolower(c); }); - *final++ = '\0'; + auto final = + std::transform(src.begin(), src.end(), dst.data(), [](unsigned char c) -> char { return static_cast(std::tolower(c)); }); + *final++ = '\0'; } } // namespace ts diff --git a/include/tsutil/Regex.h b/include/tsutil/Regex.h index 3295712bdad..03ce7a32e33 100644 --- a/include/tsutil/Regex.h +++ b/include/tsutil/Regex.h @@ -37,7 +37,12 @@ enum REFlags { RE_CASE_INSENSITIVE = 0x00000008u, ///< Ignore case (by default, matches are case sensitive). RE_UNANCHORED = 0x00000400u, ///< Unanchored (@a DFA defaults to anchored). RE_ANCHORED = 0x80000000u, ///< Anchored (@a Regex defaults to unanchored). - RE_NOTEMPTY = 0x00000004u ///< Not empty (by default, matches may match empty string). + RE_ENDANCHORED = 0x20000000u, ///< Anchored at the subject end; with RE_ANCHORED, requires a whole-subject match. + RE_NOTEMPTY = 0x00000004u, ///< Not empty (by default, matches may match empty string). + /// Require the match to consume the entire subject. When set, a successful pcre2 match + /// that does not span [0, subject.size()) is reported as @c RE_ERROR_NOMATCH. Implemented + /// as a post-match length check so JIT remains eligible on all supported PCRE2 versions. + RE_FULL_MATCH = 0x10000000u, }; /// @brief Error codes returned by regular expression operations. @@ -296,4 +301,5 @@ class DFA bool build(std::string_view pattern, unsigned flags = 0); std::vector _patterns; + bool _full_match = false; ///< Apply RE_FULL_MATCH on every match() call. }; diff --git a/lib/swoc/src/swoc_ip.cc b/lib/swoc/src/swoc_ip.cc index 6f6ae13b101..5c3b688662f 100644 --- a/lib/swoc/src/swoc_ip.cc +++ b/lib/swoc/src/swoc_ip.cc @@ -9,7 +9,6 @@ #include "swoc/swoc_meta.h" using swoc::TextView; -using swoc::svtoi; using swoc::svtou; using namespace swoc::literals; @@ -465,8 +464,8 @@ IP6Addr::load(std::string_view const &str) { empty_idx = n; } else { TextView r; - auto x = svtoi(token, &r, 16); - if (r.size() == token.size()) { + auto x = svtou(token, &r, 16); + if (r.size() == token.size() && x <= std::numeric_limits::max()) { quad[QUAD_IDX[n++]] = x; } else { break; diff --git a/lib/swoc/unit_tests/test_ip.cc b/lib/swoc/unit_tests/test_ip.cc index 1a24b6a46aa..19d146d4d43 100644 --- a/lib/swoc/unit_tests/test_ip.cc +++ b/lib/swoc/unit_tests/test_ip.cc @@ -125,6 +125,11 @@ TEST_CASE("Basic IP", "[libswoc][ip]") { REQUIRE(alpha[3] == 135); CHECK(IP6Addr().load("ffee:1f2d:c587:24c3:9128:3349:3cee:143")); + CHECK(IP6Addr().load("ffff::1")); + CHECK_FALSE(IP6Addr().load("10000::1")); + CHECK_FALSE(IP6Addr().load("[10000::1]")); + CHECK_FALSE(IP6Addr().load("-1::1")); + CHECK_FALSE(IP6Addr().load("+1::1")); IP4Addr lo{"127.0.0.1"}; CHECK(lo.is_loopback()); diff --git a/plugins/authproxy/authproxy.cc b/plugins/authproxy/authproxy.cc index 3d697241a48..eb31e7ad804 100644 --- a/plugins/authproxy/authproxy.cc +++ b/plugins/authproxy/authproxy.cc @@ -666,7 +666,6 @@ StateAuthorized(AuthRequestContext *auth, void *) TSReleaseAssert(TSHttpTxnClientReqGet(auth->txn, &request_bufp, &request_hdr) == TS_SUCCESS); field_loc = TSMimeHdrFieldGet(auth->rheader.buffer, auth->rheader.header, 0); - TSReleaseAssert(field_loc != TS_NULL_MLOC); while (field_loc) { int key_len = 0; diff --git a/plugins/background_fetch/configs.cc b/plugins/background_fetch/configs.cc index 5a6b5099b9f..4f8cc546876 100644 --- a/plugins/background_fetch/configs.cc +++ b/plugins/background_fetch/configs.cc @@ -151,13 +151,17 @@ BgFetchConfig::readConfig(const char *config_file) if (cfg_value[0] == '<') { op = BgFetchRule::size_cmp_type::LESS_THAN_OR_EQUAL; } else if (cfg_value[0] == '>') { - op = BgFetchRule::size_cmp_type::LESS_THAN_OR_EQUAL; + op = BgFetchRule::size_cmp_type::GREATER_THAN_OR_EQUAL; } else { TSError("[%s] invalid Content-Length condition %.*s, skipping config value", PLUGIN_NAME, int(cfg_value.size()), cfg_value.data()); continue; } ++cfg_value; // Drop leading character. + if (cfg_value.empty()) { + TSError("[%s] missing Content-Length size value, skipping config value", PLUGIN_NAME); + continue; + } swoc::TextView parsed; auto n = swoc::svtou(cfg_value, &parsed); if (parsed.size() != cfg_value.size()) { diff --git a/plugins/background_fetch/rules.cc b/plugins/background_fetch/rules.cc index 1411033bea7..95473719aae 100644 --- a/plugins/background_fetch/rules.cc +++ b/plugins/background_fetch/rules.cc @@ -78,11 +78,13 @@ check_value(TSHttpTxn txnp, BgFetchRule::size_cmp_type const &cmp) TSMLoc loc = TSMimeHdrFieldFind(hdr_bufp, hdr_loc, TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH); if (TS_NULL_MLOC == loc) { Dbg(Bg_dbg_ctl, "No content-length field in resp"); + TSHandleMLocRelease(hdr_bufp, TS_NULL_MLOC, hdr_loc); return false; // Field not found. } auto content_len = TSMimeHdrFieldValueUintGet(hdr_bufp, hdr_loc, loc, 0 /* index */); TSHandleMLocRelease(hdr_bufp, hdr_loc, loc); + TSHandleMLocRelease(hdr_bufp, TS_NULL_MLOC, hdr_loc); if (cmp._op == BgFetchRule::size_cmp_type::OP::GREATER_THAN_OR_EQUAL) { return content_len >= cmp._size; @@ -108,11 +110,14 @@ check_value(TSHttpTxn txnp, BgFetchRule::field_cmp_type const &cmp) if (TS_NULL_MLOC == loc) { Dbg(Bg_dbg_ctl, "no field %s in request header", cmp._name.c_str()); + TSHandleMLocRelease(hdr_bufp, TS_NULL_MLOC, hdr_loc); return false; } - if (cmp._name.size() == 1 && cmp._name.front() == '*') { + if (cmp._value.size() == 1 && cmp._value.front() == '*') { Dbg(Bg_dbg_ctl, "Found %s wild card", cmp._name.c_str()); + TSHandleMLocRelease(hdr_bufp, hdr_loc, loc); + TSHandleMLocRelease(hdr_bufp, TS_NULL_MLOC, hdr_loc); return true; } @@ -127,6 +132,7 @@ check_value(TSHttpTxn txnp, BgFetchRule::field_cmp_type const &cmp) zret = std::string_view::npos != std::string_view(val_str, val_len).find(cmp._value); } TSHandleMLocRelease(hdr_bufp, hdr_loc, loc); + TSHandleMLocRelease(hdr_bufp, TS_NULL_MLOC, hdr_loc); return zret; } diff --git a/plugins/cache_range_requests/cache_range_requests.cc b/plugins/cache_range_requests/cache_range_requests.cc index 750c735b8e9..c70fe40f8f1 100644 --- a/plugins/cache_range_requests/cache_range_requests.cc +++ b/plugins/cache_range_requests/cache_range_requests.cc @@ -37,6 +37,8 @@ #include #include +#include "swoc/bwf_base.h" + #define PLUGIN_NAME "cache_range_requests" #define DEBUG_LOG(fmt, ...) Dbg(dbg_ctl, fmt, ##__VA_ARGS__) #define ERROR_LOG(fmt, ...) TSError("[%s:%d] %s(): " fmt, __FILE__, __LINE__, __func__, ##__VA_ARGS__) @@ -283,27 +285,55 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc) std::string const &rv = txn_state->range_value; DEBUG_LOG("txn_state->range_value: '%s'", rv.c_str()); + // for performance provide a stack buffer or a spill buffer + swoc::LocalBufferWriter<16384> cache_key_bw; + std::string cache_key_spill; + std::string_view cache_key{}; + // Consider config options if (nullptr != pc) { - txn_state->config = pc; - char cache_key_url[16384] = {0}; - int cache_key_url_len = 0; + txn_state->config = pc; if (pc->modify_cache_key || PS_CACHEKEY_URL == pc->ps_mode) { int url_len = 0; char *const req_url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len); - cache_key_url_len = snprintf(cache_key_url, sizeof(cache_key_url), "%s-%s", req_url, rv.c_str()); - DEBUG_LOG("Forming new cache URL for '%s': '%.*s'", req_url, cache_key_url_len, cache_key_url); - if (req_url != nullptr) { + if (nullptr != req_url && 0 < url_len) { + std::string_view const url_sv{req_url, static_cast(url_len)}; + + // Format once into the stack buffer; only fall back to heap when it overflows. + cache_key_bw.print("{}-{}", url_sv, rv); + if (!cache_key_bw.error()) { + cache_key = cache_key_bw.view(); + } else { + cache_key_spill.resize(cache_key_bw.extent()); + swoc::FixedBufferWriter spill_bw{cache_key_spill.data(), cache_key_spill.size()}; + spill_bw.print("{}-{}", url_sv, rv); + cache_key = std::string_view{cache_key_spill.data(), spill_bw.extent()}; + } TSfree(req_url); + DEBUG_LOG("Forming new cache URL: '%.*s'", static_cast(cache_key.size()), cache_key.data()); + } else { + if (nullptr != req_url) { + TSfree(req_url); + } + ERROR_LOG("TSHttpTxnEffectiveUrlStringGet returned nullptr or zero-length URL, skipping cache key modification."); } } // Modify the cache_key if (pc->modify_cache_key) { - DEBUG_LOG("Setting cache key to '%.*s'", cache_key_url_len, cache_key_url); - if (TS_SUCCESS != TSCacheUrlSet(txnp, cache_key_url, cache_key_url_len)) { - ERROR_LOG("Failed to change the cache url, disabling cache for this transaction to avoid cache poisoning."); + bool disable_cache = false; + if (!cache_key.empty()) { + DEBUG_LOG("Setting cache key to '%.*s'", static_cast(cache_key.size()), cache_key.data()); + if (TS_SUCCESS != TSCacheUrlSet(txnp, cache_key.data(), static_cast(cache_key.size()))) { + ERROR_LOG("Failed to change the cache url, disabling cache for this transaction to avoid cache poisoning."); + disable_cache = true; + } + } else { + ERROR_LOG("Failed to build override cache key, disabling cache for this transaction to avoid cache poisoning."); + disable_cache = true; + } + if (disable_cache) { TSHttpTxnCntlSet(txnp, TS_HTTP_CNTL_SERVER_NO_STORE, true); TSHttpTxnCntlSet(txnp, TS_HTTP_CNTL_RESPONSE_CACHEABLE, false); TSHttpTxnCntlSet(txnp, TS_HTTP_CNTL_REQUEST_CACHEABLE, false); @@ -311,15 +341,14 @@ range_header_check(TSHttpTxn txnp, pluginconfig *const pc) } // Set the parent_selection_url to the modified cache_key. - if (PS_CACHEKEY_URL == pc->ps_mode) { + if (PS_CACHEKEY_URL == pc->ps_mode && !cache_key.empty()) { TSMLoc ps_loc = TS_NULL_MLOC; - const char *start = cache_key_url; - const char *end = cache_key_url + cache_key_url_len; + const char *start = cache_key.data(); + const char *end = start + cache_key.size(); if (TS_SUCCESS == TSUrlCreate(hdr_buf, &ps_loc)) { - // This should always succeed. if (TS_PARSE_DONE == TSUrlParse(hdr_buf, ps_loc, &start, end) && TS_SUCCESS == TSHttpTxnParentSelectionUrlSet(txnp, hdr_buf, ps_loc)) { - DEBUG_LOG("Setting Parent Selection URL to '%.*s'", cache_key_url_len, cache_key_url); + DEBUG_LOG("Setting Parent Selection URL to '%.*s'", static_cast(cache_key.size()), cache_key.data()); } TSHandleMLocRelease(hdr_buf, TS_NULL_MLOC, ps_loc); } diff --git a/plugins/cachekey/cachekey.cc b/plugins/cachekey/cachekey.cc index 6bc838aec49..81748076899 100644 --- a/plugins/cachekey/cachekey.cc +++ b/plugins/cachekey/cachekey.cc @@ -21,10 +21,12 @@ * @brief Cache key manipulation. */ +#include /* INT_MAX */ #include /* strlen() */ #include /* istringstream */ #include #include "cachekey.h" +#include "tsutil/LocalBuffer.h" static void append(String &target, unsigned n) @@ -37,12 +39,12 @@ append(String &target, unsigned n) static void appendEncoded(String &target, const char *s, size_t len) { - if (0 == len) { + if (0 == len || len > static_cast(INT_MAX)) { return; } - char tmp[len * 3 + 1]; - size_t written; + ts::LocalBuffer tmp(len * 3 + 1); + size_t written; /* The default table does not encode the comma, so we need to use our own table here. */ static const unsigned char map[32] = { @@ -67,8 +69,8 @@ appendEncoded(String &target, const char *s, size_t len) 0x00 // . }; - if (TSStringPercentEncode(s, len, tmp, sizeof(tmp), &written, map) == TS_SUCCESS) { - target.append(tmp, written); + if (TSStringPercentEncode(s, len, tmp.data(), tmp.size(), &written, map) == TS_SUCCESS) { + target.append(tmp.data(), written); } else { /* If the encoding fails (pretty unlikely), then just append what we have. * This is just a best-effort encoding anyway. */ diff --git a/plugins/certifier/certifier.cc b/plugins/certifier/certifier.cc index 2274e1f141c..563a39bd40f 100644 --- a/plugins/certifier/certifier.cc +++ b/plugins/certifier/certifier.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include @@ -39,7 +40,8 @@ #include // cnDataMap #include // vconnQ #include // std::string -#include // ofstream +#include +#include // ofstream #include #include @@ -316,6 +318,38 @@ static TSMutex serial_mutex; ///< serial number mutex static std::unique_ptr ssl_list = nullptr; static std::string store_path; +/** Maximum textual DNS name length without a root dot. */ +static constexpr std::string_view::size_type MAX_DNS_NAME_LEN = 253; +/** Extension appended when a generated certificate is cached on disk. */ +static constexpr std::string_view CERT_FILENAME_EXTENSION = ".crt"; +/** Maximum SNI length that keeps the generated cache filename within @c NAME_MAX. */ +static constexpr std::string_view::size_type MAX_CERT_STORE_SERVERNAME_LEN = + std::min(MAX_DNS_NAME_LEN, NAME_MAX - CERT_FILENAME_EXTENSION.size()); + +/** + * Determine whether @a servername can safely key the certificate store. + * + * The certifier plugin uses the SNI both as a certificate subject name and as + * the base name for a generated @c .crt file under the configured store. This + * helper accepts the hostname-like names the plugin can store without path + * interpretation while rejecting empty, oversized, or separator-bearing inputs + * before they reach file APIs. + * + * @param[in] servername The SNI value from the TLS handshake. + * @return @c true if @a servername is suitable for certificate storage. + */ +static bool +is_servername_storage_safe(std::string_view servername) +{ + if (servername.empty() || servername.size() > MAX_CERT_STORE_SERVERNAME_LEN) { + return false; + } + + return std::all_of(servername.begin(), servername.end(), [](unsigned char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '.' || c == '-' || c == '_'; + }); +} + /** * Local helper function that adds a Subject Alternative Name field into a * certificate. @@ -527,8 +561,15 @@ cert_retriever(TSCont /* contp ATS_UNUSED */, TSEvent /* event ATS_UNUSED */, vo SSL_CTX *ref_ctx = nullptr; if (servername == nullptr) { - TSError("[%s] %s: no SNI available", __func__, PLUGIN_NAME); - return TS_ERROR; + Dbg(dbg_ctl, "%s: no SNI available; using default certificate", __func__); + TSVConnReenable(ssl_vc); + return TS_SUCCESS; + } + + if (!is_servername_storage_safe(servername)) { + Dbg(dbg_ctl, "%s: rejecting unsafe SNI for certificate storage", __func__); + TSVConnReenable(ssl_vc); + return TS_SUCCESS; } bool wontdo = false; diff --git a/plugins/compress/configuration.cc b/plugins/compress/configuration.cc index 793a04f4c22..407716d8ebb 100644 --- a/plugins/compress/configuration.cc +++ b/plugins/compress/configuration.cc @@ -40,6 +40,16 @@ namespace Compress { +// isspace and friends invoke undefined behavior when the argument is a +// signed char with the high bit set (negative when sign-extended to int). +// HTTP headers can carry obs-text bytes (>= 0x80), so route every ctype +// call through this wrapper that casts to unsigned char first. +inline int +safe_isspace(int ch) +{ + return ::isspace(static_cast(ch)); +} + swoc::TextView extractFirstToken(swoc::TextView &view, int (*fp)(int)) { @@ -154,7 +164,7 @@ swoc::TextView strip_params(swoc::TextView v) { v = v.take_prefix_at(';'); - v.rtrim_if(&::isspace); + v.rtrim_if(&safe_isspace); return v; } @@ -190,7 +200,7 @@ HostConfiguration::is_content_type_compressible(const char *content_type, int co constexpr int isCommaOrSpace(int ch) { - return (ch == ',') or isspace(ch); + return (ch == ',') or safe_isspace(ch); } void @@ -332,12 +342,12 @@ Configuration::Parse(const char *path) ++lineno; // Trim whitespace - line_view.trim_if(&::isspace); + line_view.trim_if(&safe_isspace); if (line_view.empty()) { continue; } for (;;) { - auto token = extractFirstToken(line_view, isspace); + auto token = extractFirstToken(line_view, safe_isspace); if (token.empty()) { break; diff --git a/plugins/compress/gzip_compress.cc b/plugins/compress/gzip_compress.cc index 04111a47489..9c812ea70a8 100644 --- a/plugins/compress/gzip_compress.cc +++ b/plugins/compress/gzip_compress.cc @@ -112,7 +112,8 @@ transform_one(Data *data, const char *upstream_buffer, int64_t upstream_length) } if (err != Z_OK) { - warning("deflate() call failed: %d", err); + error("deflate() call failed: %d", err); + return; } if (downstream_length > data->zstrm.avail_out) { diff --git a/plugins/esi/CMakeLists.txt b/plugins/esi/CMakeLists.txt index 7d1df2d3a34..73d01d6fde0 100644 --- a/plugins/esi/CMakeLists.txt +++ b/plugins/esi/CMakeLists.txt @@ -25,7 +25,7 @@ target_include_directories(esi PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}") verify_global_plugin(esi) verify_remap_plugin(esi) -add_atsplugin(combo_handler combo_handler.cc http_utils.cc) +add_atsplugin(combo_handler combo_handler.cc combo_handler_utils.cc http_utils.cc) target_link_libraries(combo_handler PRIVATE esicore fetcher) target_include_directories(combo_handler PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/plugins/esi/combo_handler.cc b/plugins/esi/combo_handler.cc index 9ff34ccecf2..92e07b2140f 100644 --- a/plugins/esi/combo_handler.cc +++ b/plugins/esi/combo_handler.cc @@ -41,6 +41,7 @@ #include "HttpDataFetcherImpl.h" #include "gzip.h" #include "Utils.h" +#include "combo_handler_utils.h" using namespace std; using namespace EsiLib; @@ -277,27 +278,23 @@ CacheControlHeader::update(TSMBuffer bufp, TSMLoc hdr_loc) int _val_len = 0; const char *val = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, i, &_val_len); - // Update max-age if necessary - if (strncasecmp(val, TS_HTTP_VALUE_MAX_AGE, TS_HTTP_LEN_MAX_AGE) == 0) { - unsigned int max_age = 0; - char *ptr = const_cast(val); - ptr += TS_HTTP_LEN_MAX_AGE; - while ((*ptr == ' ') || (*ptr == '\t')) { - ptr++; - } - if (*ptr == '=') { - ptr++; - max_age = atoi(ptr); - } - if (max_age > 0 && max_age < _max_age) { - _max_age = max_age; + if (val != nullptr && _val_len > 0) { + combo_handler::CacheControlValue const parsed = + combo_handler::parse_cache_control_value({val, static_cast(_val_len)}); + + // Update max-age if necessary. max-age=0 is a valid directive + // ("must revalidate") and must be honored as the minimum. + if (parsed.has_max_age) { + if (parsed.max_age < _max_age) { + _max_age = parsed.max_age; + } + // If we find even a single occurrence of private, the whole response must be private + } else if (parsed.is_private) { + found_private = true; + // Every requested document must have immutable for the final response to be immutable + } else if (parsed.is_immutable) { + found_immutable = true; } - // If we find even a single occurrence of private, the whole response must be private - } else if (strncasecmp(val, TS_HTTP_VALUE_PRIVATE, TS_HTTP_LEN_PRIVATE) == 0) { - found_private = true; - // Every requested document must have immutable for the final response to be immutable - } else if (strncasecmp(val, HTTP_IMMUTABLE, strlen(HTTP_IMMUTABLE)) == 0) { - found_immutable = true; } } } @@ -1092,6 +1089,11 @@ ContentTypeHandler::nextObjectHeader(TSMBuffer bufp, TSMLoc hdr_loc) const char *value; int value_len; int n_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc); + if (n_values <= 0 && !_content_type_allowlist.empty()) { + // An empty Content-Type field with a non-empty allowlist must not pass. + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + return false; + } for (int i = 0; i < n_values; ++i) { value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, i, &value_len); swoc::TextView tv{value, size_t(value_len)}; @@ -1101,9 +1103,11 @@ ContentTypeHandler::nextObjectHeader(TSMBuffer bufp, TSMLoc hdr_loc) } else if (std::find_if(_content_type_allowlist.begin(), _content_type_allowlist.end(), [tv](swoc::TextView tv2) -> bool { return strcasecmp(tv, tv2) == 0; }) == _content_type_allowlist.end()) { + TSHandleMLocRelease(bufp, hdr_loc, field_loc); return false; } else if (tv.empty()) { // allowlist is bad, contains an empty string. + TSHandleMLocRelease(bufp, hdr_loc, field_loc); return false; } if (!_added_content_type) { diff --git a/plugins/esi/combo_handler_utils.cc b/plugins/esi/combo_handler_utils.cc new file mode 100644 index 00000000000..ce579ac561f --- /dev/null +++ b/plugins/esi/combo_handler_utils.cc @@ -0,0 +1,141 @@ +/** @file + + Util functions for combo handler. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the + License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "combo_handler_utils.h" + +#include + +namespace +{ +constexpr std::string_view MaxAge{"max-age"}; +constexpr std::string_view Private{"private"}; +constexpr std::string_view Immutable{"immutable"}; + +// ASCII-only tolower. HTTP tokens are ASCII; std::tolower from +// is locale-dependent (notably tr_TR maps 'I' away from 'i') and would +// otherwise let the C locale at plugin init silently change how +// directives like "Immutable" and "PRIVATE" match. +constexpr char +ascii_tolower(char c) +{ + return (c >= 'A' && c <= 'Z') ? static_cast(c + ('a' - 'A')) : c; +} + +// ASCII-only digit test. std::isdigit from is locale-dependent and +// can accept non-ASCII digits when the process locale is not C; HTTP directive +// syntax (e.g. max-age) is ASCII, so match '0'..'9' explicitly. +constexpr bool +ascii_isdigit(char c) +{ + return c >= '0' && c <= '9'; +} + +bool +starts_with_ignore_case(std::string_view value, std::string_view token) +{ + if (value.size() < token.size()) { + return false; + } + + for (size_t i = 0; i < token.size(); ++i) { + if (ascii_tolower(value[i]) != ascii_tolower(token[i])) { + return false; + } + } + return true; +} + +bool +is_lws(char c) +{ + return c == ' ' || c == '\t'; +} + +// True if `value` begins with `token` (case-insensitive) AND the token is +// terminated by a directive boundary, so a longer word that merely starts +// with the token is not misclassified. The boundary is end-of-string or +// linear whitespace; when `allow_equals` is set (e.g. private="field-name"), +// an '=' introducing a value also terminates the token. This keeps values +// like "privatee" or "immutableX" from matching "private"/"immutable". +bool +matches_directive(std::string_view value, std::string_view token, bool allow_equals) +{ + if (!starts_with_ignore_case(value, token)) { + return false; + } + if (value.size() == token.size()) { + return true; + } + char const next = value[token.size()]; + return is_lws(next) || (allow_equals && next == '='); +} +} // namespace + +namespace combo_handler +{ +CacheControlValue +parse_cache_control_value(std::string_view value) +{ + CacheControlValue parsed; + + if (starts_with_ignore_case(value, MaxAge)) { + value.remove_prefix(MaxAge.size()); + while (!value.empty() && is_lws(value.front())) { + value.remove_prefix(1); + } + if (!value.empty() && value.front() == '=') { + value.remove_prefix(1); + while (!value.empty() && is_lws(value.front())) { + value.remove_prefix(1); + } + unsigned max_age = 0; + bool overflow = false; + bool any_digit = false; + while (!value.empty() && ascii_isdigit(value.front())) { + unsigned const digit = value.front() - '0'; + if (overflow || max_age > (std::numeric_limits::max() - digit) / 10) { + overflow = true; + } else { + max_age = (max_age * 10) + digit; + } + any_digit = true; + value.remove_prefix(1); + } + // Require at least one digit so that "max-age=" / "max-age=foo" + // don't masquerade as max-age=0. Clamp overflow to UINT_MAX so a + // huge upstream value can't be misread as the smallest possible + // max-age when callers honor zero. + if (any_digit) { + parsed.has_max_age = true; + parsed.max_age = overflow ? std::numeric_limits::max() : max_age; + } + } + } else if (matches_directive(value, Private, /* allow_equals */ true)) { + parsed.is_private = true; + } else if (matches_directive(value, Immutable, /* allow_equals */ false)) { + parsed.is_immutable = true; + } + + return parsed; +} +} // namespace combo_handler diff --git a/plugins/esi/combo_handler_utils.h b/plugins/esi/combo_handler_utils.h new file mode 100644 index 00000000000..f4d03921e09 --- /dev/null +++ b/plugins/esi/combo_handler_utils.h @@ -0,0 +1,38 @@ +/** @file + + Util functions for combo handler. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the + License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include + +namespace combo_handler +{ +struct CacheControlValue { + bool has_max_age{false}; + unsigned max_age{0}; + bool is_private{false}; + bool is_immutable{false}; +}; + +CacheControlValue parse_cache_control_value(std::string_view value); +} // namespace combo_handler diff --git a/plugins/esi/esi.cc b/plugins/esi/esi.cc index 04c49431f8c..516ea681642 100644 --- a/plugins/esi/esi.cc +++ b/plugins/esi/esi.cc @@ -47,6 +47,7 @@ #include "EsiProcessor.h" #include "HttpDataFetcher.h" #include "HandlerManager.h" +#include "IncludeUrlValidator.h" #include "serverIntercept.h" #include "Stats.h" #include "HttpDataFetcherImpl.h" @@ -60,15 +61,30 @@ using response_codes_t = std::unordered_set; #include "http_utils.h" struct OptionInfo { - bool packed_node_support{false}; - bool private_response{false}; - bool disable_gzip_output{false}; - bool first_byte_flush{false}; - unsigned max_doc_size{1024 * 1024}; - unsigned max_inclusion_depth{3}; - response_codes_t allowed_response_codes{200, 304}; + bool packed_node_support{false}; + bool private_response{false}; + bool disable_gzip_output{false}; + bool first_byte_flush{false}; + bool allow_private_include_hosts{false}; + unsigned max_doc_size{1024 * 1024}; + unsigned max_inclusion_depth{3}; + response_codes_t allowed_response_codes{200, 304}; + EsiLib::IncludeUrlValidator url_validator; }; +// OptionInfo is allocated with TSRalloc + placement-new, so TSfree alone will +// not run its destructor. It owns non-trivially destructible members (e.g. the +// PCRE2-backed ts::Regex state inside url_validator), so every free path must +// destroy first. +static void +freeOptionInfo(OptionInfo *pOptionInfo) +{ + if (pOptionInfo != nullptr) { + pOptionInfo->~OptionInfo(); + TSfree(pOptionInfo); + } +} + static HandlerManager *gHandlerManager = nullptr; static Utils::HeaderValueList gAllowlistCookies; @@ -254,7 +270,8 @@ ContData::init() esi_vars = new Variables(contp, gAllowlistCookies); } - esi_proc = new EsiProcessor(contp, *data_fetcher, *esi_vars, *gHandlerManager, option_info->max_doc_size, request_url); + esi_proc = new EsiProcessor(contp, *data_fetcher, *esi_vars, *gHandlerManager, option_info->max_doc_size, request_url, + &option_info->url_validator); esi_gzip = new EsiGzip(); esi_gunzip = new EsiGunzip(); @@ -1660,19 +1677,37 @@ esiPluginInit(int argc, const char *argv[], OptionInfo *pOptionInfo) if (argc > 1) { int c; static const struct option longopts[] = { - {const_cast("packed-node-support"), no_argument, nullptr, 'n'}, - {const_cast("private-response"), no_argument, nullptr, 'p'}, - {const_cast("disable-gzip-output"), no_argument, nullptr, 'z'}, - {const_cast("first-byte-flush"), no_argument, nullptr, 'b'}, - {const_cast("handler-filename"), required_argument, nullptr, 'f'}, - {const_cast("max-doc-size"), required_argument, nullptr, 'd'}, - {const_cast("max-inclusion-depth"), required_argument, nullptr, 'i'}, - {const_cast("allowed-response-codes"), required_argument, nullptr, 'r'}, - {nullptr, 0, nullptr, 0 }, + {const_cast("packed-node-support"), no_argument, nullptr, 'n'}, + {const_cast("private-response"), no_argument, nullptr, 'p'}, + {const_cast("disable-gzip-output"), no_argument, nullptr, 'z'}, + {const_cast("first-byte-flush"), no_argument, nullptr, 'b'}, + {const_cast("handler-filename"), required_argument, nullptr, 'f'}, + {const_cast("max-doc-size"), required_argument, nullptr, 'd'}, + {const_cast("max-inclusion-depth"), required_argument, nullptr, 'i'}, + {const_cast("allowed-response-codes"), required_argument, nullptr, 'r'}, + {const_cast("include-host-allow"), required_argument, nullptr, 'H'}, + {const_cast("allow-private-include-hosts"), no_argument, nullptr, 'P'}, + {nullptr, 0, nullptr, 0 }, }; + // Reset getopt's global parsing state. esiPluginInit() can run more than + // once per process (e.g. one TSRemapNewInstance per remap rule); without + // this, leftover state from a prior call makes getopt_long() skip or + // mis-parse options. Mirrors the reset ATS performs before plugin init + // in src/proxy/Plugin.cc. +#if (!defined(kfreebsd) && defined(freebsd)) || defined(darwin) + optreset = 1; +#endif +#if defined(__GLIBC__) + optind = 0; +#else + optind = 1; +#endif + opterr = 0; + optarg = nullptr; + int longindex = 0; - while ((c = getopt_long(argc, const_cast(argv), "npzbf:d:i:r:", longopts, &longindex)) != -1) { + while ((c = getopt_long(argc, const_cast(argv), "npzbf:d:i:r:H:P", longopts, &longindex)) != -1) { switch (c) { case 'n': pOptionInfo->packed_node_support = true; @@ -1754,6 +1789,25 @@ esiPluginInit(int argc, const char *argv[], OptionInfo *pOptionInfo) } break; } + case 'H': { + // The host allowlist is a security control; if the operator + // intended one but typo'd the regex, do not silently fall back + // to "no allowlist". TSEmergency exits the process, but return + // -1 makes the fail-closed contract explicit and survives any + // future change to that helper. The optarg-null guard is + // required for the analyzer; getopt_long is documented to + // supply optarg for required_argument options. + if (optarg == nullptr || !pOptionInfo->url_validator.setHostAllowRegex(optarg)) { + TSEmergency("[esi][%s] include-host-allow regex (%s) failed to compile", __FUNCTION__, optarg ? optarg : "(null)"); + return -1; + } + break; + } + case 'P': { + pOptionInfo->allow_private_include_hosts = true; + pOptionInfo->url_validator.setAllowPrivateHosts(true); + break; + } default: TSEmergency("[esi][%s] bad option", __FUNCTION__); return -1; @@ -1773,9 +1827,10 @@ esiPluginInit(int argc, const char *argv[], OptionInfo *pOptionInfo) Dbg(dbg_ctl_local, "[%s] Plugin started, " "packed-node-support: %d, private-response: %d, disable-gzip-output: %d, first-byte-flush: %d, max-doc-size %u, " - "max-inclusion-depth %u, allowed-response-codes: [%s]", + "max-inclusion-depth %u, allowed-response-codes: [%s], allow-private-include-hosts: %d", __FUNCTION__, pOptionInfo->packed_node_support, pOptionInfo->private_response, pOptionInfo->disable_gzip_output, - pOptionInfo->first_byte_flush, pOptionInfo->max_doc_size, pOptionInfo->max_inclusion_depth, response_codes_str.c_str()); + pOptionInfo->first_byte_flush, pOptionInfo->max_doc_size, pOptionInfo->max_inclusion_depth, response_codes_str.c_str(), + pOptionInfo->allow_private_include_hosts); return 0; } @@ -1799,14 +1854,14 @@ TSPluginInit(int argc, const char *argv[]) return; } if (esiPluginInit(argc, argv, pOptionInfo) != 0) { - TSfree(pOptionInfo); + freeOptionInfo(pOptionInfo); return; } TSCont global_contp = TSContCreate(globalHookHandler, nullptr); if (!global_contp) { TSError("[esi][%s] Could not create global continuation", __FUNCTION__); - TSfree(pOptionInfo); + freeOptionInfo(pOptionInfo); return; } TSContDataSet(global_contp, pOptionInfo); @@ -1867,7 +1922,7 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_s } if (esiPluginInit(index, new_argv, pOptionInfo) != 0) { snprintf(errbuf, errbuf_size, "esiPluginInit fail!"); - TSfree(pOptionInfo); + freeOptionInfo(pOptionInfo); return TS_ERROR; } TSCont contp = TSContCreate(globalHookHandler, nullptr); @@ -1882,6 +1937,8 @@ TSRemapDeleteInstance(void *ih) { TSCont contp = static_cast(ih); if (contp != nullptr) { + auto *pOptionInfo = static_cast(TSContDataGet(contp)); + freeOptionInfo(pOptionInfo); TSContDestroy(contp); } } diff --git a/plugins/esi/fetcher/HttpDataFetcherImpl.cc b/plugins/esi/fetcher/HttpDataFetcherImpl.cc index 4ee060007f8..a4a9911d55e 100644 --- a/plugins/esi/fetcher/HttpDataFetcherImpl.cc +++ b/plugins/esi/fetcher/HttpDataFetcherImpl.cc @@ -86,6 +86,22 @@ HttpDataFetcherImpl::~HttpDataFetcherImpl() bool HttpDataFetcherImpl::addFetchRequest(const string &url, FetchedDataProcessor *callback_obj /* = 0 */) { + static constexpr size_t MAX_REQ_LEN = 32 * 1024; + + size_t total_len = 0; + auto add_to_total_len = [&total_len](size_t part_len) -> bool { + if (part_len > (MAX_REQ_LEN - total_len)) { + return false; + } + total_len += part_len; + return true; + }; + if (!add_to_total_len(sizeof("GET ") - 1) || !add_to_total_len(url.length()) || !add_to_total_len(sizeof(" HTTP/1.0\r\n") - 1) || + !add_to_total_len(_headers_str.length()) || !add_to_total_len(sizeof("\r\n") - 1)) { + TSError("[HttpDataFetcherImpl][%s] HTTP request size exceeds maximum %zu", __FUNCTION__, MAX_REQ_LEN); + return false; + } + // do we already have a request for this? std::pair insert_result = _pages.insert(UrlToContentMap::value_type(url, RequestData())); if (callback_obj) { @@ -101,7 +117,7 @@ HttpDataFetcherImpl::addFetchRequest(const string &url, FetchedDataProcessor *ca int length; size_t req_buf_size = 0; - length = sizeof("GET ") - 1 + url.length() + sizeof(" HTTP/1.0\r\n") - 1 + _headers_str.length() + sizeof("\r\n") - 1; + length = static_cast(total_len); if (length < static_cast(sizeof(buff))) { http_req = buff; req_buf_size = sizeof(buff); diff --git a/plugins/esi/lib/CMakeLists.txt b/plugins/esi/lib/CMakeLists.txt index 7de411ce867..c57fad653d1 100644 --- a/plugins/esi/lib/CMakeLists.txt +++ b/plugins/esi/lib/CMakeLists.txt @@ -23,9 +23,18 @@ add_library( EsiProcessor.cc Expression.cc HandlerManager.cc + IncludeUrlValidator.cc Stats.cc Variables.cc ) +# esicore is a static library dlopen()ed as part of plugins (esi.so, combo_handler.so) alongside +# traffic_server. We do NOT link ts::tsutil here even though IncludeUrlValidator.cc references +# ts::Regex: if libtsutil.a were on the link line for esicore or its consumers, each plugin would +# statically embed tsutil globals (e.g., DbgCtl::_config_mode) and violate ODR against the same +# symbols in traffic_server at dlopen time (ASan detects_odr_violation). Instead, unresolved +# tsutil symbols are left in the plugin and resolved by the runtime linker against traffic_server +# at load time. The tsutil headers are visible via the global include_directories() in the +# top-level CMakeLists so compilation still succeeds. target_link_libraries(esicore PUBLIC esi-common fetcher libswoc::libswoc) target_include_directories(esicore PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}") set_target_properties(esicore PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/plugins/esi/lib/EsiProcessor.cc b/plugins/esi/lib/EsiProcessor.cc index b4687fd655b..70f4a364fc3 100644 --- a/plugins/esi/lib/EsiProcessor.cc +++ b/plugins/esi/lib/EsiProcessor.cc @@ -35,15 +35,61 @@ const char *EsiProcessor::INCLUDE_DATA_ID_ATTR = reinterpret_cast( namespace { -DbgCtl dbg_ctl{"plugin_esi_procesor"}; -} +DbgCtl dbg_ctl{"plugin_esi_processor"}; + +// Wraps an inner HttpDataFetcher with the include-URL validator so +// special-include handlers — which talk to the fetcher directly via +// the reference they were constructed with — can't reach TSFetchUrl +// with an unvalidated URL. Other virtuals delegate straight through. +class ValidatingFetcher : public HttpDataFetcher +{ +public: + ValidatingFetcher(HttpDataFetcher &inner, const IncludeUrlValidator &validator) : _inner(inner), _validator(validator) {} + + bool + addFetchRequest(const std::string &url, FetchedDataProcessor *callback_obj = nullptr) override + { + auto reason = _validator.validate(url); + if (reason != IncludeUrlValidator::OK) { + std::string const safe = IncludeUrlValidator::redactUserInfo(url); + TSError("[%s] Rejecting include URL [%s]: %s (via special-include handler)", __FUNCTION__, safe.c_str(), + IncludeUrlValidator::reasonString(reason)); + Stats::increment(Stats::N_SPCL_INCLUDE_ERRS); + return false; + } + return _inner.addFetchRequest(url, callback_obj); + } + + DataStatus + getRequestStatus(const std::string &url) const override + { + return _inner.getRequestStatus(url); + } + + int + getNumPendingRequests() const override + { + return _inner.getNumPendingRequests(); + } + + bool + getContent(const std::string &url, const char *&content, int &content_len) const override + { + return _inner.getContent(url, content, content_len); + } + +private: + HttpDataFetcher &_inner; + const IncludeUrlValidator &_validator; +}; +} // namespace // This can only be used in member functions of EsiProcessor. // #define DBG(FMT, ...) Dbg(dbg_ctl, FMT " contp=%p", ##__VA_ARGS__, _cont_addr) EsiProcessor::EsiProcessor(void *cont_addr, HttpDataFetcher &fetcher, Variables &variables, const HandlerManager &handler_mgr, - unsigned max_doc_size, std::string_view request_url) + unsigned max_doc_size, std::string_view request_url, const IncludeUrlValidator *url_validator) : _curr_state(STOPPED), _parser(max_doc_size, request_url), _n_prescanned_nodes(0), @@ -57,8 +103,12 @@ EsiProcessor::EsiProcessor(void *cont_addr, HttpDataFetcher &fetcher, Variables _n_try_blocks_processed(0), _handler_manager(handler_mgr), _request_url{request_url}, + _url_validator(url_validator), _cont_addr(cont_addr) { + if (_url_validator) { + _handler_fetcher = std::make_unique(_fetcher, *_url_validator); + } } bool @@ -614,6 +664,27 @@ EsiProcessor::_handleVars(const char *str, int str_len) bool EsiProcessor::_handleHtmlComment(const DocNodeList::iterator &curr_node) { + // Reject a nested at any depth inside the wrapper's content. + // The wrapper exists to hide ESI markup from non-ESI parsers; its contents + // are raw ESI tags, never another wrapper. Catching it on the raw bytes + // (rather than on the parsed top-level inner_nodes) closes the case where + // the nested wrapper is held inside child_nodes of , , + // , , , or , where it + // would still be expanded later by _preprocess. Mirrors the parser's own + // opening-tag rule in EsiParser::_findOpeningTag: " inside is not allowed", __FUNCTION__); + Stats::increment(Stats::N_PARSE_ERRS); + return false; + } + } + } + DocNodeList inner_nodes; if (!_parser.parse(inner_nodes, curr_node->data, curr_node->data_len)) { TSError("[%s] Couldn't parse html comment node content", __FUNCTION__); @@ -683,13 +754,31 @@ EsiProcessor::_preprocess(DocNodeList &node_list, int &n_prescanned_nodes) } const string &expanded_url = _expression.expand(raw_url); if (!expanded_url.size()) { - TSError("[%s] Couldn't expand raw URL [%.*s]", __FUNCTION__, int(raw_url.size()), raw_url.data()); + // Redact any user:pass@ before logging so credentials in the include + // template don't leak into diags.log, matching the rejection path below. + std::string const safe_raw = IncludeUrlValidator::redactUserInfo(raw_url); + TSError("[%s] Couldn't expand raw URL [%s]", __FUNCTION__, safe_raw.c_str()); Stats::increment(Stats::N_INCLUDE_ERRS); continue; } + if (_url_validator) { + auto reason = _url_validator->validate(expanded_url); + if (reason != IncludeUrlValidator::OK) { + // Redact any user:pass@ before logging so credentials in the + // include template or its expansion don't leak into diags.log. + std::string const safe_expanded = IncludeUrlValidator::redactUserInfo(expanded_url); + std::string const safe_raw = IncludeUrlValidator::redactUserInfo(raw_url); + TSError("[%s] Rejecting include URL [%s] (raw [%s]): %s", __FUNCTION__, safe_expanded.c_str(), safe_raw.c_str(), + IncludeUrlValidator::reasonString(reason)); + Stats::increment(Stats::N_INCLUDE_ERRS); + continue; + } + } + if (!_fetcher.addFetchRequest(expanded_url)) { - TSError("[%s] Couldn't add fetch request for URL [%.*s]", __FUNCTION__, int(raw_url.size()), raw_url.data()); + std::string const safe_expanded = IncludeUrlValidator::redactUserInfo(expanded_url); + TSError("[%s] Couldn't add fetch request for URL [%s]", __FUNCTION__, safe_expanded.c_str()); Stats::increment(Stats::N_INCLUDE_ERRS); continue; } @@ -703,7 +792,11 @@ EsiProcessor::_preprocess(DocNodeList &node_list, int &n_prescanned_nodes) SpecialIncludeHandler *handler; IncludeHandlerMap::const_iterator map_iter = _include_handlers.find(handler_id); if (map_iter == _include_handlers.end()) { - handler = _handler_manager.getHandler(_esi_vars, _expression, _fetcher, handler_id); + // Hand the validating wrapper to special-include handlers when a + // validator is configured; otherwise fall back to the raw + // fetcher (preserves behavior when SSRF guards are off). + HttpDataFetcher &handler_fetcher = _handler_fetcher ? *_handler_fetcher : _fetcher; + handler = _handler_manager.getHandler(_esi_vars, _expression, handler_fetcher, handler_id); if (!handler) { TSError("[%s] Couldn't create handler with id [%s]", __FUNCTION__, handler_id.c_str()); Stats::increment(Stats::N_SPCL_INCLUDE_ERRS); diff --git a/plugins/esi/lib/EsiProcessor.h b/plugins/esi/lib/EsiProcessor.h index 0baef0f1a6e..5cd579f9492 100644 --- a/plugins/esi/lib/EsiProcessor.h +++ b/plugins/esi/lib/EsiProcessor.h @@ -23,6 +23,7 @@ #pragma once +#include #include #include #include @@ -30,6 +31,7 @@ #include "DocNode.h" #include "EsiParser.h" #include "HttpDataFetcher.h" +#include "IncludeUrlValidator.h" #include "Variables.h" #include "Expression.h" #include "SpecialIncludeHandler.h" @@ -46,7 +48,8 @@ class EsiProcessor }; EsiProcessor(void *cont_addr, HttpDataFetcher &fetcher, EsiLib::Variables &variables, const EsiLib::HandlerManager &handler_mgr, - unsigned max_doc_size, std::string_view request_url = ""); + unsigned max_doc_size, std::string_view request_url = "", + const EsiLib::IncludeUrlValidator *url_validator = nullptr); /** Initializes the processor with the context of the request to be processed */ bool start(); @@ -166,8 +169,13 @@ class EsiProcessor TryBlockList _try_blocks; int _n_try_blocks_processed; - const EsiLib::HandlerManager &_handler_manager; - std::string _request_url; + const EsiLib::HandlerManager &_handler_manager; + std::string _request_url; + const EsiLib::IncludeUrlValidator *_url_validator; + // Wraps _fetcher with the validator and is passed to special-include + // handlers so they cannot reach TSFetchUrl with an unvalidated URL. + // nullptr when no validator is configured. + std::unique_ptr _handler_fetcher; static const char *INCLUDE_DATA_ID_ATTR; diff --git a/plugins/esi/lib/IncludeUrlValidator.cc b/plugins/esi/lib/IncludeUrlValidator.cc new file mode 100644 index 00000000000..5bb9c0c51fe --- /dev/null +++ b/plugins/esi/lib/IncludeUrlValidator.cc @@ -0,0 +1,441 @@ +/** @file + + Validator for esi:include src URLs to mitigate SSRF. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "IncludeUrlValidator.h" + +#include +#include +#include +#include +#include +#include + +using std::string_view; + +namespace EsiLib +{ +namespace +{ + // Backtracking limit for the allowlist match. PCRE2 stops and reports + // PCRE2_ERROR_MATCHLIMIT once this many match steps are taken, bounding + // worst-case CPU per validation against attacker-influenced hostnames. + // Matches the value used by the regex_remap plugin. + constexpr uint32_t ALLOW_REGEX_MATCH_LIMIT = 1750; + + bool + iequals(string_view a, string_view b) + { + if (a.size() != b.size()) { + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (std::tolower(static_cast(a[i])) != std::tolower(static_cast(b[i]))) { + return false; + } + } + return true; + } + + bool + iendswith(string_view s, string_view suffix) + { + if (s.size() < suffix.size()) { + return false; + } + return iequals(s.substr(s.size() - suffix.size()), suffix); + } + + // RFC 1918, loopback, link-local, unspecified, broadcast, and CGNAT. + bool + ipv4IsPrivate(const in_addr &a) + { + uint32_t h = ntohl(a.s_addr); + + // 0.0.0.0/8 (unspecified / "this network") + if ((h & 0xFF000000u) == 0x00000000u) { + return true; + } + // 10.0.0.0/8 + if ((h & 0xFF000000u) == 0x0A000000u) { + return true; + } + // 100.64.0.0/10 (CGNAT) + if ((h & 0xFFC00000u) == 0x64400000u) { + return true; + } + // 127.0.0.0/8 + if ((h & 0xFF000000u) == 0x7F000000u) { + return true; + } + // 169.254.0.0/16 (link-local, includes cloud metadata 169.254.169.254) + if ((h & 0xFFFF0000u) == 0xA9FE0000u) { + return true; + } + // 172.16.0.0/12 + if ((h & 0xFFF00000u) == 0xAC100000u) { + return true; + } + // 192.0.0.0/24 (IETF protocol assignments) and 192.0.2.0/24 (TEST-NET-1) + if ((h & 0xFFFFFF00u) == 0xC0000000u || (h & 0xFFFFFF00u) == 0xC0000200u) { + return true; + } + // 192.168.0.0/16 + if ((h & 0xFFFF0000u) == 0xC0A80000u) { + return true; + } + // 198.18.0.0/15 (benchmarking) + if ((h & 0xFFFE0000u) == 0xC6120000u) { + return true; + } + // 198.51.100.0/24 (TEST-NET-2), 203.0.113.0/24 (TEST-NET-3) + if ((h & 0xFFFFFF00u) == 0xC6336400u || (h & 0xFFFFFF00u) == 0xCB007100u) { + return true; + } + // 224.0.0.0/4 (multicast), 240.0.0.0/4 (reserved), 255.255.255.255 + if ((h & 0xF0000000u) == 0xE0000000u || (h & 0xF0000000u) == 0xF0000000u) { + return true; + } + return false; + } + + bool + ipv6IsPrivate(const in6_addr &a) + { + // ::/128 unspecified, ::1/128 loopback + bool all_zero = true; + for (int i = 0; i < 15; ++i) { + if (a.s6_addr[i] != 0) { + all_zero = false; + break; + } + } + if (all_zero && (a.s6_addr[15] == 0 || a.s6_addr[15] == 1)) { + return true; + } + // fe80::/10 link-local + if (a.s6_addr[0] == 0xfe && (a.s6_addr[1] & 0xc0) == 0x80) { + return true; + } + // fc00::/7 unique local + if ((a.s6_addr[0] & 0xfe) == 0xfc) { + return true; + } + // ff00::/8 multicast + if (a.s6_addr[0] == 0xff) { + return true; + } + // ::ffff:0:0/96 IPv4-mapped — fall through to IPv4 check + bool is_v4_mapped = true; + for (int i = 0; i < 10; ++i) { + if (a.s6_addr[i] != 0) { + is_v4_mapped = false; + break; + } + } + if (is_v4_mapped && a.s6_addr[10] == 0xff && a.s6_addr[11] == 0xff) { + in_addr v4; + std::memcpy(&v4.s_addr, &a.s6_addr[12], 4); + return ipv4IsPrivate(v4); + } + // 64:ff9b::/96 NAT64 — the last 32 bits are an embedded IPv4. Recurse + // through ipv4IsPrivate so private IPv4 destinations (e.g. + // 64:ff9b::0a00:0001 → 10.0.0.1) cannot bypass the denylist in + // NAT64-enabled environments. + if (a.s6_addr[0] == 0x00 && a.s6_addr[1] == 0x64 && a.s6_addr[2] == 0xff && a.s6_addr[3] == 0x9b) { + bool nat64_zero_middle = true; + for (int i = 4; i < 12; ++i) { + if (a.s6_addr[i] != 0) { + nat64_zero_middle = false; + break; + } + } + if (nat64_zero_middle) { + in_addr v4; + std::memcpy(&v4.s_addr, &a.s6_addr[12], 4); + return ipv4IsPrivate(v4); + } + } + return false; + } + + // Non-canonical numeric IPv4 forms — decimal ("2130706433"), octal + // ("017700000001"), hex ("0x7f000001"), and shortcut forms like "127.1" — + // are accepted by inet_aton but not inet_pton. They have no legitimate use + // in an include URL and are a standard SSRF-filter evasion. They must be + // rejected even when --allow-private-include-hosts is set, since that flag + // only relaxes the private-range denylist, not these evasion forms. + // Strips any zone id and trailing FQDN-root dot before checking, matching + // the normalization in isPrivateHost. + bool + isNonCanonicalNumericIPv4(string_view host) + { + if (auto pct = host.find('%'); pct != string_view::npos) { + host = host.substr(0, pct); + } + if (!host.empty() && host.back() == '.') { + host.remove_suffix(1); + } + + std::string h(host); + in_addr v4{}; + if (inet_pton(AF_INET, h.c_str(), &v4) == 1) { + return false; // canonical dotted-quad, handled by the normal denylist + } + return inet_aton(h.c_str(), &v4) != 0; + } + +} // namespace + +bool +IncludeUrlValidator::splitUrl(string_view url, string_view &scheme, string_view &host) +{ + auto sep = url.find("://"); + if (sep == string_view::npos || sep == 0) { + return false; + } + scheme = url.substr(0, sep); + + string_view rest = url.substr(sep + 3); + if (rest.empty()) { + return false; + } + + // Strip userinfo "user:pass@". Be careful: '@' may appear in path; only look + // before the authority terminator. + auto authority_end = rest.find_first_of("/?#"); + string_view authority = (authority_end == string_view::npos) ? rest : rest.substr(0, authority_end); + + auto at = authority.rfind('@'); + if (at != string_view::npos) { + authority.remove_prefix(at + 1); + } + + if (authority.empty()) { + return false; + } + + // IPv6 literal in brackets: [....] optionally followed by ":port". + if (authority.front() == '[') { + auto rb = authority.find(']'); + if (rb == string_view::npos) { + return false; + } + // The closing ']' must terminate the authority or be immediately followed + // by the port separator ':'. Anything else (e.g. "[::1]evil.com") is + // malformed; reject it rather than silently extracting "::1" as the host + // while the real URL points elsewhere. + if (rb + 1 != authority.size() && authority[rb + 1] != ':') { + return false; + } + host = authority.substr(1, rb - 1); + } else { + auto colon = authority.find(':'); + host = (colon == string_view::npos) ? authority : authority.substr(0, colon); + } + return !host.empty(); +} + +bool +IncludeUrlValidator::isPrivateHost(string_view host) +{ + // RFC 6874 IPv6 scope id: "fe80::1%eth0" (or URL-encoded + // "fe80::1%25eth0" after splitUrl strips the brackets). inet_pton + // rejects the '%', so strip from there before parsing — and treat + // the very presence of a zone id as private, because scope ids only + // make sense for link-local / non-global addresses. + bool had_zone = false; + if (auto pct = host.find('%'); pct != string_view::npos) { + had_zone = true; + host = host.substr(0, pct); + } + // Strip a single trailing FQDN-root dot: "localhost." resolves the + // same as "localhost"; without this, the hostname checks below would + // be trivially bypassed. + if (!host.empty() && host.back() == '.') { + host.remove_suffix(1); + } + + std::string h(host); + in_addr v4{}; + if (inet_pton(AF_INET, h.c_str(), &v4) == 1) { + return ipv4IsPrivate(v4); + } + // Non-canonical IPv4 numeric forms — decimal integer ("2130706433"), + // octal ("017700000001"), hex ("0x7f000001"), and shortcut forms + // like "127.1" — are accepted by inet_aton but not by inet_pton. + // They have no legitimate use in ESI include URLs and are routinely + // used to evade SSRF filters that only canonicalize the dotted-quad + // form. Reject outright instead of trying to apply ipv4IsPrivate to + // the parsed value. + if (inet_aton(h.c_str(), &v4) != 0) { + return true; + } + in6_addr v6{}; + if (inet_pton(AF_INET6, h.c_str(), &v6) == 1) { + return had_zone || ipv6IsPrivate(v6); + } + // Had a zone id but didn't parse as IPv6 — fail closed. + if (had_zone) { + return true; + } + // Non-IP hostname: deny obvious loopback aliases. Real hostnames that + // resolve to private space must be caught by the optional allowlist or + // upstream remap policy; we don't do DNS here. + if (iequals(host, "localhost") || iendswith(host, ".localhost")) { + return true; + } + return false; +} + +bool +IncludeUrlValidator::setHostAllowRegex(const std::string &pattern) +{ + std::string error; + int erroffset = -1; + if (!_allow_regex.compile(pattern, error, erroffset, RE_CASE_INSENSITIVE)) { + _has_allow_regex = false; + return false; + } + // Cap backtracking so a pathological pattern/host can't burn unbounded CPU + // on this hot path; an exceeded limit surfaces from exec() as a negative + // return and is treated as "not allowlisted" (fail closed). + _match_context.set_match_limit(ALLOW_REGEX_MATCH_LIMIT); + _has_allow_regex = true; + return true; +} + +std::string +IncludeUrlValidator::redactUserInfo(string_view url) +{ + // Locate the authority, then its userinfo. The authority begins after "://" + // for an absolute URL, or after a leading "//" for a protocol-relative URL + // ("//user:pass@host/path"). If there is an '@' before the authority + // terminator ('/', '?', or '#'), everything up to and including the '@' is + // userinfo and may carry credentials — replace it with "***@". This runs on + // rejection paths for URLs that fail validation too, so it must not require + // a valid scheme. Check the leading "//" first: a protocol-relative URL has + // no scheme, and a later "://" (e.g. inside a query) must not be mistaken + // for the authority delimiter. + size_t authority_begin; + if (url.substr(0, 2) == "//") { + authority_begin = 2; + } else if (auto scheme_sep = url.find("://"); scheme_sep != string_view::npos) { + authority_begin = scheme_sep + 3; + } else { + return std::string{url}; + } + size_t authority_end = url.find_first_of("/?#", authority_begin); + string_view authority = (authority_end == string_view::npos) ? url.substr(authority_begin) : + url.substr(authority_begin, authority_end - authority_begin); + auto at_in_authority = authority.rfind('@'); + if (at_in_authority == string_view::npos) { + return std::string{url}; + } + std::string redacted; + redacted.reserve(url.size()); + redacted.append(url.substr(0, authority_begin)); + redacted.append("***@"); + redacted.append(url.substr(authority_begin + at_in_authority + 1)); + return redacted; +} + +IncludeUrlValidator::Reason +IncludeUrlValidator::validate(string_view url) const +{ + // Reject ASCII control characters / whitespace to avoid request splitting in TSFetchUrl request construction. + for (unsigned char c : url) { + if (c <= 0x20 || c == 0x7f) { + return MALFORMED; + } + } + + string_view scheme; + string_view host; + if (!splitUrl(url, scheme, host)) { + return MALFORMED; + } + if (!iequals(scheme, "http") && !iequals(scheme, "https")) { + return BAD_SCHEME; + } + // Non-canonical numeric IPv4 forms are SSRF evasion and are rejected + // unconditionally — the allow-private-hosts escape hatch only relaxes the + // private-range denylist, not these forms. + if (isNonCanonicalNumericIPv4(host)) { + return PRIVATE_HOST; + } + // An RFC 6874 zone id (e.g. "fe80::1%25eth0", with the '%' URL-encoded as + // "%25") selects a network interface and is only meaningful for link-local + // / non-global addresses. Reject any host carrying a '%' unconditionally so + // it can never be enabled by accident through the private-hosts escape + // hatch. + if (host.find('%') != string_view::npos) { + return PRIVATE_HOST; + } + if (!_allow_private_hosts && isPrivateHost(host)) { + return PRIVATE_HOST; + } + if (_has_allow_regex) { + // Normalize the host the same way isPrivateHost does so an + // allowlist for "example.com" still matches "example.com." and + // doesn't accidentally match a host carrying a zone id. + string_view bare = host; + if (auto pct = bare.find('%'); pct != string_view::npos) { + bare = bare.substr(0, pct); + } + if (!bare.empty() && bare.back() == '.') { + bare.remove_suffix(1); + } + std::string h(bare); + // RE_FULL_MATCH requires the pattern to span the entire host (the + // std::regex_match semantics this replaced). exec() returns the capture + // count on success, 0 if it matched but the match buffer was too small for + // all capture groups, and a negative value for no match / error / exceeded + // match limit. We only care whether the host matched, not the captures, so + // any non-negative return is a match; only a negative return fails closed. + RegexMatches matches; + if (_allow_regex.exec(h, matches, RE_FULL_MATCH, &_match_context) < 0) { + return NOT_ALLOWLISTED; + } + } + return OK; +} + +const char * +IncludeUrlValidator::reasonString(Reason r) +{ + switch (r) { + case OK: + return "ok"; + case MALFORMED: + return "malformed-url"; + case BAD_SCHEME: + return "bad-scheme"; + case PRIVATE_HOST: + return "private-host"; + case NOT_ALLOWLISTED: + return "not-allowlisted"; + } + return "unknown"; +} + +} // namespace EsiLib diff --git a/plugins/esi/lib/IncludeUrlValidator.h b/plugins/esi/lib/IncludeUrlValidator.h new file mode 100644 index 00000000000..29f60abe6fb --- /dev/null +++ b/plugins/esi/lib/IncludeUrlValidator.h @@ -0,0 +1,86 @@ +/** @file + + Validator for esi:include src URLs to mitigate SSRF. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include +#include + +#include "tsutil/Regex.h" + +namespace EsiLib +{ +class IncludeUrlValidator +{ +public: + enum Reason { + OK = 0, + MALFORMED = 1, + BAD_SCHEME = 2, + PRIVATE_HOST = 3, + NOT_ALLOWLISTED = 4, + }; + + IncludeUrlValidator() = default; + ~IncludeUrlValidator() = default; + + // Compiles a regex that, when set, requires the post-expansion hostname to + // fully match. Returns false on invalid pattern; the caller is responsible + // for failing initialization closed (this security control should never be + // silently disabled by a typo'd pattern). + bool setHostAllowRegex(const std::string &pattern); + + void + setAllowPrivateHosts(bool b) + { + _allow_private_hosts = b; + } + + // Validates an already-expanded include URL. Cheap and pure. + Reason validate(std::string_view url) const; + + static const char *reasonString(Reason r); + + // Returns the URL with any userinfo ("user:pass@") replaced by "***@", + // so credentials don't leak into log lines. Leaves URLs without + // userinfo unchanged. Tolerates unparseable templates (raw URLs + // containing $(...) before expansion). + static std::string redactUserInfo(std::string_view url); + + // Exposed for unit tests. + static bool splitUrl(std::string_view url, std::string_view &scheme, std::string_view &host); + static bool isPrivateHost(std::string_view host); + +private: + bool _allow_private_hosts{false}; + bool _has_allow_regex{false}; + // PCRE2-backed (ts::Regex) instead of std::regex: the allowlist is matched + // in a hot path against attacker-influenced hostnames, and a backtracking + // engine is vulnerable to catastrophic-backtracking DoS. _match_context + // carries a match (backtracking) limit so worst-case CPU time per match is + // bounded; exceeding it fails closed (treated as "not allowlisted"). + Regex _allow_regex; + RegexMatchContext _match_context; +}; + +} // namespace EsiLib diff --git a/plugins/esi/test/CMakeLists.txt b/plugins/esi/test/CMakeLists.txt index c9bc8c2d5f7..67a9057026f 100644 --- a/plugins/esi/test/CMakeLists.txt +++ b/plugins/esi/test/CMakeLists.txt @@ -29,15 +29,27 @@ macro(ADD_ESI_TEST NAME) ) endmacro() +# esicore does not link ts::tsutil (see plugins/esi/lib/CMakeLists.txt for why), so test +# executables that pull in tsutil symbols transitively via esicore must link tsutil +# themselves. Plugins (esi.so, combo_handler.so) don't need this because they resolve +# tsutil symbols dynamically against traffic_server at dlopen time. add_esi_test(test_docnode docnode_test.cc) -target_link_libraries(test_docnode PRIVATE Catch2::Catch2WithMain esi-common esicore) +target_link_libraries(test_docnode PRIVATE Catch2::Catch2WithMain esi-common esicore ts::tsutil) add_esi_test(test_parser parser_test.cc) -target_link_libraries(test_parser PRIVATE Catch2::Catch2WithMain esi-common esicore) +target_link_libraries(test_parser PRIVATE Catch2::Catch2WithMain esi-common esicore ts::tsutil) add_esi_test(test_processor processor_test.cc) -target_link_libraries(test_processor PRIVATE Catch2::Catch2WithMain esi-common esicore) +target_link_libraries(test_processor PRIVATE Catch2::Catch2WithMain esi-common esicore ts::tsutil) add_esi_test(test_utils utils_test.cc) target_link_libraries(test_utils PRIVATE Catch2::Catch2WithMain esi-common) add_esi_test(test_vars vars_test.cc) -target_link_libraries(test_vars PRIVATE Catch2::Catch2WithMain esi-common esicore) +target_link_libraries(test_vars PRIVATE Catch2::Catch2WithMain esi-common esicore ts::tsutil) add_esi_test(test_gzip gzip_test.cc) target_link_libraries(test_gzip PRIVATE Catch2::Catch2WithMain esi-common) +add_esi_test(test_include_url_validator include_url_validator_test.cc) +target_link_libraries(test_include_url_validator PRIVATE Catch2::Catch2WithMain esi-common esicore ts::tsutil) +# combo_handler_utils.cc lives in the parent plugin source dir; compile +# it directly into the test binary since it is bundled into combo_handler.so +# rather than its own library. +add_esi_test(test_combo_handler_utils combo_handler_utils_test.cc ${CMAKE_CURRENT_SOURCE_DIR}/../combo_handler_utils.cc) +target_include_directories(test_combo_handler_utils PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/..) +target_link_libraries(test_combo_handler_utils PRIVATE Catch2::Catch2WithMain) diff --git a/plugins/esi/test/combo_handler_utils_test.cc b/plugins/esi/test/combo_handler_utils_test.cc new file mode 100644 index 00000000000..3325497b3da --- /dev/null +++ b/plugins/esi/test/combo_handler_utils_test.cc @@ -0,0 +1,197 @@ +/** @file + + Unit tests for combo_handler::parse_cache_control_value(). + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include + +#include "combo_handler_utils.h" + +using combo_handler::CacheControlValue; +using combo_handler::parse_cache_control_value; + +TEST_CASE("parse_cache_control_value parses max-age digits") +{ + auto p = parse_cache_control_value("max-age=300"); + REQUIRE(p.has_max_age); + REQUIRE(p.max_age == 300u); + REQUIRE_FALSE(p.is_private); + REQUIRE_FALSE(p.is_immutable); +} + +TEST_CASE("parse_cache_control_value honors max-age=0") +{ + // max-age=0 is a valid "must revalidate" directive; the parser must + // expose it so the caller can drive the combined response down to 0 + // rather than treating it as if the directive were absent. + auto p = parse_cache_control_value("max-age=0"); + REQUIRE(p.has_max_age); + REQUIRE(p.max_age == 0u); +} + +TEST_CASE("parse_cache_control_value rejects max-age with no digits") +{ + // "max-age=" or "max-age=foo" must not masquerade as max-age=0. If + // they did, callers that honor zero would force the combined + // response to no-cache on every garbage upstream value. + { + auto p = parse_cache_control_value("max-age="); + REQUIRE_FALSE(p.has_max_age); + } + { + auto p = parse_cache_control_value("max-age=foo"); + REQUIRE_FALSE(p.has_max_age); + } + { + auto p = parse_cache_control_value("max-age"); + REQUIRE_FALSE(p.has_max_age); + } +} + +TEST_CASE("parse_cache_control_value clamps overflow to UINT_MAX") +{ + // Pathological upstream values must not be misread as a small TTL. + // The parser clamps overflow to UINT_MAX so the min-merge in the + // caller never selects this object as the minimum. + auto p = parse_cache_control_value("max-age=99999999999999999999"); + REQUIRE(p.has_max_age); + REQUIRE(p.max_age == std::numeric_limits::max()); +} + +TEST_CASE("parse_cache_control_value tolerates whitespace around '='") +{ + auto p = parse_cache_control_value("max-age = 42"); + REQUIRE(p.has_max_age); + REQUIRE(p.max_age == 42u); +} + +TEST_CASE("parse_cache_control_value is case-insensitive on the token") +{ + { + auto p = parse_cache_control_value("MAX-AGE=300"); + REQUIRE(p.has_max_age); + REQUIRE(p.max_age == 300u); + } + { + auto p = parse_cache_control_value("Max-Age=7"); + REQUIRE(p.has_max_age); + REQUIRE(p.max_age == 7u); + } +} + +TEST_CASE("parse_cache_control_value recognizes private") +{ + { + auto p = parse_cache_control_value("private"); + REQUIRE(p.is_private); + REQUIRE_FALSE(p.has_max_age); + REQUIRE_FALSE(p.is_immutable); + } + { + auto p = parse_cache_control_value("Private"); + REQUIRE(p.is_private); + } + { + auto p = parse_cache_control_value("PRIVATE"); + REQUIRE(p.is_private); + } +} + +TEST_CASE("parse_cache_control_value recognizes immutable") +{ + { + auto p = parse_cache_control_value("immutable"); + REQUIRE(p.is_immutable); + REQUIRE_FALSE(p.has_max_age); + REQUIRE_FALSE(p.is_private); + } + { + auto p = parse_cache_control_value("Immutable"); + REQUIRE(p.is_immutable); + } +} + +TEST_CASE("parse_cache_control_value enforces directive boundary for private") +{ + // A longer token that merely starts with "private" must not be treated as + // the private directive. Without a boundary check, "privatee" would flip + // the whole combined response to private. + { + auto p = parse_cache_control_value("privatee"); + REQUIRE_FALSE(p.is_private); + REQUIRE_FALSE(p.has_max_age); + REQUIRE_FALSE(p.is_immutable); + } + { + auto p = parse_cache_control_value("private-cache"); + REQUIRE_FALSE(p.is_private); + } + // The directive may be followed by whitespace or by '=' (field-name form, + // e.g. private="set-cookie"); both are valid boundaries. + { + auto p = parse_cache_control_value("private "); + REQUIRE(p.is_private); + } + { + auto p = parse_cache_control_value(R"(private="set-cookie")"); + REQUIRE(p.is_private); + } +} + +TEST_CASE("parse_cache_control_value enforces directive boundary for immutable") +{ + { + auto p = parse_cache_control_value("immutableX"); + REQUIRE_FALSE(p.is_immutable); + REQUIRE_FALSE(p.has_max_age); + REQUIRE_FALSE(p.is_private); + } + // immutable takes no value, so '=' is not a valid boundary for it. + { + auto p = parse_cache_control_value("immutable=1"); + REQUIRE_FALSE(p.is_immutable); + } + { + auto p = parse_cache_control_value("immutable "); + REQUIRE(p.is_immutable); + } +} + +TEST_CASE("parse_cache_control_value ignores unrelated tokens") +{ + // Cache-Control directives the combo_handler does not aggregate + // (no-cache, no-store, public, must-revalidate, ...) should all + // come back with every flag false. + auto p = parse_cache_control_value("no-cache"); + REQUIRE_FALSE(p.has_max_age); + REQUIRE_FALSE(p.is_private); + REQUIRE_FALSE(p.is_immutable); +} + +TEST_CASE("parse_cache_control_value handles empty input") +{ + auto p = parse_cache_control_value(""); + REQUIRE_FALSE(p.has_max_age); + REQUIRE_FALSE(p.is_private); + REQUIRE_FALSE(p.is_immutable); +} diff --git a/plugins/esi/test/include_url_validator_test.cc b/plugins/esi/test/include_url_validator_test.cc new file mode 100644 index 00000000000..0144e450804 --- /dev/null +++ b/plugins/esi/test/include_url_validator_test.cc @@ -0,0 +1,206 @@ +/** @file + + Unit tests for IncludeUrlValidator. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include "IncludeUrlValidator.h" + +using EsiLib::IncludeUrlValidator; + +TEST_CASE("splitUrl parses scheme and host") +{ + std::string_view scheme; + std::string_view host; + + REQUIRE(IncludeUrlValidator::splitUrl("http://example.com/foo", scheme, host)); + REQUIRE(scheme == "http"); + REQUIRE(host == "example.com"); + + REQUIRE(IncludeUrlValidator::splitUrl("https://EXAMPLE.com:8443/path?x=1", scheme, host)); + REQUIRE(host == "EXAMPLE.com"); + + REQUIRE(IncludeUrlValidator::splitUrl("http://[::1]:80/", scheme, host)); + REQUIRE(host == "::1"); + + REQUIRE(IncludeUrlValidator::splitUrl("http://user:pw@example.com/x", scheme, host)); + REQUIRE(host == "example.com"); + + REQUIRE_FALSE(IncludeUrlValidator::splitUrl("no-scheme", scheme, host)); + REQUIRE_FALSE(IncludeUrlValidator::splitUrl("://nohost/", scheme, host)); + REQUIRE_FALSE(IncludeUrlValidator::splitUrl("http://", scheme, host)); + + // A bracketed IPv6 authority must be followed only by end-of-authority or a + // ":port" separator. Trailing junk after ']' (e.g. "[::1]evil.com") would + // otherwise mis-parse the host as "::1" while the URL points elsewhere. + REQUIRE_FALSE(IncludeUrlValidator::splitUrl("http://[::1]evil.com/", scheme, host)); + REQUIRE_FALSE(IncludeUrlValidator::splitUrl("http://[fe80::1]x/path", scheme, host)); + REQUIRE_FALSE(IncludeUrlValidator::splitUrl("http://[]/", scheme, host)); + REQUIRE_FALSE(IncludeUrlValidator::splitUrl("http://[::1", scheme, host)); + // Valid bracketed forms still parse. + REQUIRE(IncludeUrlValidator::splitUrl("http://[2001:db8::1]/p", scheme, host)); + REQUIRE(host == "2001:db8::1"); + REQUIRE(IncludeUrlValidator::splitUrl("http://[::1]", scheme, host)); + REQUIRE(host == "::1"); +} + +TEST_CASE("isPrivateHost recognizes private ranges") +{ + // IPv4 ranges that must be denied. + REQUIRE(IncludeUrlValidator::isPrivateHost("127.0.0.1")); + REQUIRE(IncludeUrlValidator::isPrivateHost("10.0.0.1")); + REQUIRE(IncludeUrlValidator::isPrivateHost("172.16.0.1")); + REQUIRE(IncludeUrlValidator::isPrivateHost("192.168.5.5")); + REQUIRE(IncludeUrlValidator::isPrivateHost("169.254.169.254")); // cloud metadata + REQUIRE(IncludeUrlValidator::isPrivateHost("0.0.0.0")); + REQUIRE(IncludeUrlValidator::isPrivateHost("100.64.1.1")); // CGNAT + // IPv6. + REQUIRE(IncludeUrlValidator::isPrivateHost("::1")); + REQUIRE(IncludeUrlValidator::isPrivateHost("fe80::1")); + REQUIRE(IncludeUrlValidator::isPrivateHost("fc00::abcd")); + REQUIRE(IncludeUrlValidator::isPrivateHost("::ffff:10.0.0.1")); // IPv4-mapped private + REQUIRE(IncludeUrlValidator::isPrivateHost("64:ff9b::a00:1")); // NAT64 wrapping 10.0.0.1 + REQUIRE(IncludeUrlValidator::isPrivateHost("64:ff9b::a9fe:a9fe")); // NAT64 wrapping 169.254.169.254 + REQUIRE_FALSE(IncludeUrlValidator::isPrivateHost("64:ff9b::808:808")); // NAT64 wrapping 8.8.8.8 + // RFC 6874 IPv6 scope id: stripped before parsing, and the very + // presence of a zone id means scoped/local — deny outright. + REQUIRE(IncludeUrlValidator::isPrivateHost("fe80::1%eth0")); + REQUIRE(IncludeUrlValidator::isPrivateHost("fe80::1%25eth0")); // URL-encoded form + REQUIRE(IncludeUrlValidator::isPrivateHost("2001:db8::1%eth0")); // global address + zone → still scoped + REQUIRE(IncludeUrlValidator::isPrivateHost("malformed-ipv6%eth0")); // had zone but didn't parse → fail closed + // Names. + REQUIRE(IncludeUrlValidator::isPrivateHost("localhost")); + REQUIRE(IncludeUrlValidator::isPrivateHost("LocalHost")); + REQUIRE(IncludeUrlValidator::isPrivateHost("foo.localhost")); + // Trailing FQDN-root dot must not bypass the loopback check. + REQUIRE(IncludeUrlValidator::isPrivateHost("localhost.")); + REQUIRE(IncludeUrlValidator::isPrivateHost("foo.localhost.")); + // Non-canonical IPv4 numeric forms must not slip past the denylist as + // "hostnames". Common evasion forms decoded back to 127.0.0.1: + REQUIRE(IncludeUrlValidator::isPrivateHost("2130706433")); // decimal int + REQUIRE(IncludeUrlValidator::isPrivateHost("017700000001")); // octal + REQUIRE(IncludeUrlValidator::isPrivateHost("0x7f000001")); // hex + REQUIRE(IncludeUrlValidator::isPrivateHost("0x7f.0x0.0x0.0x1")); // mixed hex octets + REQUIRE(IncludeUrlValidator::isPrivateHost("127.1")); // 2-part compact + REQUIRE(IncludeUrlValidator::isPrivateHost("127.0.1")); // 3-part compact + // Even non-canonical numeric forms for "public" IPs are rejected: an + // include URL written this way is never a legitimate hostname and + // operators should use the dotted-quad form. + REQUIRE(IncludeUrlValidator::isPrivateHost("0x08080808")); // hex 8.8.8.8 + REQUIRE(IncludeUrlValidator::isPrivateHost("134744072")); // decimal 8.8.8.8 + // DNS hostnames that happen to start with a digit are still hostnames. + REQUIRE_FALSE(IncludeUrlValidator::isPrivateHost("1example.com")); + // Public. + REQUIRE_FALSE(IncludeUrlValidator::isPrivateHost("8.8.8.8")); + REQUIRE_FALSE(IncludeUrlValidator::isPrivateHost("2001:4860:4860::8888")); + REQUIRE_FALSE(IncludeUrlValidator::isPrivateHost("example.com")); + REQUIRE_FALSE(IncludeUrlValidator::isPrivateHost("example.com.")); +} + +TEST_CASE("validate enforces scheme and private-host denylist by default") +{ + IncludeUrlValidator v; + REQUIRE(v.validate("http://example.com/foo") == IncludeUrlValidator::OK); + REQUIRE(v.validate("https://example.com/") == IncludeUrlValidator::OK); + // file:///etc/passwd has an empty authority and is rejected as MALFORMED before the scheme check. + REQUIRE(v.validate("file:///etc/passwd") == IncludeUrlValidator::MALFORMED); + REQUIRE(v.validate("file://host/etc/passwd") == IncludeUrlValidator::BAD_SCHEME); + REQUIRE(v.validate("gopher://example.com/") == IncludeUrlValidator::BAD_SCHEME); + REQUIRE(v.validate("not-a-url") == IncludeUrlValidator::MALFORMED); + REQUIRE(v.validate("http://169.254.169.254/latest/meta-data/") == IncludeUrlValidator::PRIVATE_HOST); + REQUIRE(v.validate("http://localhost:8080/admin") == IncludeUrlValidator::PRIVATE_HOST); + REQUIRE(v.validate("http://[::1]/") == IncludeUrlValidator::PRIVATE_HOST); + // Trailing-dot bypass: "localhost." must not slip past the denylist. + REQUIRE(v.validate("http://localhost./admin") == IncludeUrlValidator::PRIVATE_HOST); + // RFC 6874 IPv6 zone id: URL-encoded "%25" appears in the host after + // splitUrl strips the brackets; the scoped form must be rejected. + REQUIRE(v.validate("http://[fe80::1%25eth0]/x") == IncludeUrlValidator::PRIVATE_HOST); +} + +TEST_CASE("validate respects allow-private-hosts escape hatch") +{ + IncludeUrlValidator v; + v.setAllowPrivateHosts(true); + REQUIRE(v.validate("http://10.0.0.5/svc") == IncludeUrlValidator::OK); + REQUIRE(v.validate("http://localhost/x") == IncludeUrlValidator::OK); + // Canonical dotted-quad loopback is allowed in this mode. + REQUIRE(v.validate("http://127.0.0.1/x") == IncludeUrlValidator::OK); + // Scheme is still enforced. + REQUIRE(v.validate("ftp://10.0.0.5/x") == IncludeUrlValidator::BAD_SCHEME); + // The escape hatch relaxes the private-range denylist, but non-canonical + // numeric IPv4 forms are SSRF evasion and stay rejected even here — for + // private addresses... + REQUIRE(v.validate("http://0x7f000001/x") == IncludeUrlValidator::PRIVATE_HOST); // hex 127.0.0.1 + REQUIRE(v.validate("http://2130706433/x") == IncludeUrlValidator::PRIVATE_HOST); // decimal 127.0.0.1 + REQUIRE(v.validate("http://017700000001/x") == IncludeUrlValidator::PRIVATE_HOST); // octal 127.0.0.1 + REQUIRE(v.validate("http://127.1/x") == IncludeUrlValidator::PRIVATE_HOST); // 2-part compact + // ...and for public addresses (never a legitimate include host). + REQUIRE(v.validate("http://0x08080808/x") == IncludeUrlValidator::PRIVATE_HOST); // hex 8.8.8.8 + REQUIRE(v.validate("http://134744072/x") == IncludeUrlValidator::PRIVATE_HOST); // decimal 8.8.8.8 + // An RFC 6874 zone id selects a (link-local) interface and is rejected even + // in this mode, so it can never be enabled by accident through the escape + // hatch. + REQUIRE(v.validate("http://[fe80::1%25eth0]/x") == IncludeUrlValidator::PRIVATE_HOST); +} + +TEST_CASE("validate enforces optional host allowlist regex") +{ + IncludeUrlValidator v; + REQUIRE(v.setHostAllowRegex(R"((.+\.)?example\.com)")); + REQUIRE(v.validate("http://example.com/x") == IncludeUrlValidator::OK); + REQUIRE(v.validate("http://api.example.com/x") == IncludeUrlValidator::OK); + REQUIRE(v.validate("http://EXAMPLE.com/x") == IncludeUrlValidator::OK); + REQUIRE(v.validate("http://evil.com/x") == IncludeUrlValidator::NOT_ALLOWLISTED); + // Regex doesn't bypass the private-host denylist (private check runs first). + REQUIRE(v.validate("http://127.0.0.1/x") == IncludeUrlValidator::PRIVATE_HOST); + // Allowlist must see the normalized host: an operator pattern for + // "example.com" matches an include URL with the FQDN-root form. + REQUIRE(v.validate("http://example.com./x") == IncludeUrlValidator::OK); + REQUIRE(v.validate("http://api.example.com./x") == IncludeUrlValidator::OK); +} + +TEST_CASE("setHostAllowRegex rejects invalid pattern") +{ + IncludeUrlValidator v; + REQUIRE_FALSE(v.setHostAllowRegex("(unclosed")); + // After a failed compile, validator falls back to no-allowlist behavior. + // The caller (TSPluginInit) is responsible for failing the plugin so + // this fallback is never reached in practice; the test pins the + // per-instance contract. + REQUIRE(v.validate("http://example.com/x") == IncludeUrlValidator::OK); +} + +TEST_CASE("redactUserInfo strips credentials before logging") +{ + // Plain userinfo replaced with "***@". + REQUIRE(IncludeUrlValidator::redactUserInfo("http://user:pass@example.com/x") == "http://***@example.com/x"); + REQUIRE(IncludeUrlValidator::redactUserInfo("http://user@example.com/x") == "http://***@example.com/x"); + // No userinfo: unchanged. + REQUIRE(IncludeUrlValidator::redactUserInfo("http://example.com/x") == "http://example.com/x"); + // '@' only in the path/query must not be misidentified as userinfo. + REQUIRE(IncludeUrlValidator::redactUserInfo("http://example.com/path@notuser") == "http://example.com/path@notuser"); + REQUIRE(IncludeUrlValidator::redactUserInfo("http://example.com/?q=a@b") == "http://example.com/?q=a@b"); + // Unparseable / pre-expansion templates: tolerated. + REQUIRE(IncludeUrlValidator::redactUserInfo("not-a-url") == "not-a-url"); + // Empty userinfo (just "@") is still redacted so the form is consistent. + REQUIRE(IncludeUrlValidator::redactUserInfo("http://@example.com/") == "http://***@example.com/"); +} diff --git a/plugins/experimental/access_control/plugin.cc b/plugins/experimental/access_control/plugin.cc index f5817c6e624..53defce8f8a 100644 --- a/plugins/experimental/access_control/plugin.cc +++ b/plugins/experimental/access_control/plugin.cc @@ -24,16 +24,20 @@ #include /* strftime */ -#include "common.h" /* Common definitions */ -#include "config.h" /* AccessControlConfig */ -#include "access_control.h" /* AccessToken */ -#include "ts/remap.h" /* TSRemapInterface, TSRemapStatus, apiInfo */ -#include "ts/ts.h" /* ATS API */ -#include "utils.h" /* cryptoBase64Decode.* functions */ -#include "headers.h" /* getHeader, setHeader, removeHeader */ +#include "common.h" /* Common definitions */ +#include "config.h" /* AccessControlConfig */ +#include "access_control.h" /* AccessToken */ +#include "ts/remap.h" /* TSRemapInterface, TSRemapStatus, apiInfo */ +#include "ts/ts.h" /* ATS API */ +#include "tsutil/LocalBuffer.h" /* ts::LocalBuffer */ +#include "utils.h" /* cryptoBase64Decode.* functions */ +#include "headers.h" /* getHeader, setHeader, removeHeader */ static const std::string_view UNKNOWN{"unknown"}; +// Stack reservation @c ts::LocalBuffer uses for the base64 cookie decode buffer. +static constexpr size_t COOKIE_DECODE_STACK_BUFFER_SIZE = 8 * 1024; + static const char * getEventName(TSEvent event) { @@ -514,7 +518,9 @@ enforceAccessControl(TSHttpTxn txnp, TSRemapRequestInfo *rri, AccessControlConfi * example, using Base64 [RFC4648]. */ size_t decodedCookieBufferSize = cryptoBase64DecodeSize(cookie.c_str(), cookie.size()); - char decodedCookie[decodedCookieBufferSize]; + ts::LocalBuffer decodedCookieBuf(decodedCookieBufferSize); + // decodedCookieBuf owns the storage; decodedCookie is a non-owning view into it. + char *decodedCookie = decodedCookieBuf.data(); size_t decryptedCookieSize = cryptoModifiedBase64Decode(cookie.c_str(), cookie.size(), decodedCookie, decodedCookieBufferSize); if (0 < decryptedCookieSize) { if (auto token = config->_tokenFactory->getAccessToken(); nullptr != token) { diff --git a/plugins/experimental/jax_fingerprint/ja3/test.cc b/plugins/experimental/jax_fingerprint/ja3/test.cc index 9826ef24059..53d597fa375 100644 --- a/plugins/experimental/jax_fingerprint/ja3/test.cc +++ b/plugins/experimental/jax_fingerprint/ja3/test.cc @@ -59,6 +59,12 @@ TEST_CASE("ja3 word buffer encoding") CHECK("" == got); } + SECTION("nullptr with len 1 - early return must not deref") + { + auto got{ja3::encode_word_buffer(nullptr, 1)}; + CHECK("" == got); + } + SECTION("1 value") { auto got{ja3::encode_word_buffer(buf, 2)}; @@ -70,6 +76,55 @@ TEST_CASE("ja3 word buffer encoding") auto got{ja3::encode_word_buffer(buf, 10)}; CHECK("5-8-256" == got); } + + SECTION("all GREASE - skip-loop consumes buffer, no emit") + { + unsigned char const grease_buf[]{0x0a, 0x0a, 0xda, 0xda}; + auto got{ja3::encode_word_buffer(grease_buf, 4)}; + CHECK("" == got); + } + + SECTION("trailing GREASE - last pair is GREASE, no trailing dash") + { + unsigned char const buf2[]{0x00, 0x05, 0x0a, 0x0a}; + auto got{ja3::encode_word_buffer(buf2, 4)}; + CHECK("5" == got); + } + + SECTION("odd length 1 - single trailing byte must not be read as a word") + { + unsigned char const odd_buf[]{0x42}; + auto got{ja3::encode_word_buffer(odd_buf, 1)}; + CHECK("" == got); + } + + SECTION("odd length 3 - last byte without pair must be ignored") + { + unsigned char const odd_buf[]{0x00, 0x05, 0x42}; + auto got{ja3::encode_word_buffer(odd_buf, 3)}; + CHECK("5" == got); + } + + SECTION("odd length 3 after GREASE - skip-loop must not read past end") + { + unsigned char const odd_buf[]{0x0a, 0x0a, 0x42}; + auto got{ja3::encode_word_buffer(odd_buf, 3)}; + CHECK("" == got); + } + + SECTION("odd length 5 - tail loop must reject trailing single byte") + { + unsigned char const odd_buf[]{0x00, 0x05, 0x00, 0x08, 0x42}; + auto got{ja3::encode_word_buffer(odd_buf, 5)}; + CHECK("5-8" == got); + } + + SECTION("supported_groups path: 3-byte extension body, 1-byte tail") + { + unsigned char const ext_body[]{0x00, 0x01, 0x02}; + auto got{ja3::encode_word_buffer(ext_body + 2, 1)}; + CHECK("" == got); + } } TEST_CASE("ja3 integer buffer encoding") diff --git a/plugins/experimental/jax_fingerprint/ja3/utils.cc b/plugins/experimental/jax_fingerprint/ja3/utils.cc index b54e5066c83..5947e67870d 100644 --- a/plugins/experimental/jax_fingerprint/ja3/utils.cc +++ b/plugins/experimental/jax_fingerprint/ja3/utils.cc @@ -67,17 +67,21 @@ std::string encode_word_buffer(unsigned char const *buf, int const len) { std::string result; - auto it{buf}; - while ((it < (buf + len)) && ja3_should_ignore(from_big_endian(it[0], it[1]))) { + if (len < 2) { + return result; + } + auto it{buf}; + auto const end{buf + len}; + while ((it < end - 1) && ja3_should_ignore(from_big_endian(it[0], it[1]))) { it += 2; } - if (it < (buf + len)) { + if (it < end - 1) { // Benchmarks show that reserving buf.size() - 1 space in the string here // would have no impact on performance. Since the string may not even need // that much due to GREASE values present in the buffer, we don't do it. result.append(std::to_string(from_big_endian(it[0], it[1]))); it += 2; - for (; it < buf + len; it += 2) { + for (; it < end - 1; it += 2) { auto const value{from_big_endian(it[0], it[1])}; if (!ja3_should_ignore(value)) { result.push_back('-'); diff --git a/plugins/experimental/rate_limit/ip_reputation.cc b/plugins/experimental/rate_limit/ip_reputation.cc index 1853861c496..427806630c0 100644 --- a/plugins/experimental/rate_limit/ip_reputation.cc +++ b/plugins/experimental/rate_limit/ip_reputation.cc @@ -120,8 +120,8 @@ SieveLru::parseYaml(const YAML::Node &node) uint32_t cur_size = pow(2, 1 + _size - _num_buckets); - _map.reserve(pow(2, _size + 1)); // Allow for all the sieve LRUs - _buckets.reserve(_num_buckets + 1); // One extra bucket, for the deny list + _map.reserve(pow(2, _size + 1)); // Allow for all the sieve LRUs + _buckets.resize(_num_buckets + 1); // One extra bucket, for the deny list // Create the other buckets, in smaller and smaller sizes (power of 2) for (uint32_t i = lastBucket(); i <= entryBucket(); ++i) { diff --git a/plugins/experimental/rate_limit/sni_selector.cc b/plugins/experimental/rate_limit/sni_selector.cc index 0f441465b4f..c1c2eec7ea7 100644 --- a/plugins/experimental/rate_limit/sni_selector.cc +++ b/plugins/experimental/rate_limit/sni_selector.cc @@ -20,6 +20,8 @@ #include "sni_selector.h" +extern int gVCIdx; + std::atomic SniSelector::_instance = nullptr; /////////////////////////////////////////////////////////////////////////////// @@ -218,7 +220,7 @@ sni_queue_cont(TSCont cont, TSEvent /* event ATS_UNUSED */, void * /* edata ATS_ if (owner) { // Don't operate on the aliases // Try to enable some queued VCs (if any) if there are slots available - while (limiter->size() > 0 && limiter->reserve() != ReserveStatus::RESERVED) { // Can't be UNLIMITED here + while (limiter->size() > 0 && limiter->reserve() == ReserveStatus::RESERVED) { auto [vc, contp, start_time] = limiter->pop(); std::chrono::milliseconds delay = std::chrono::duration_cast(now - start_time); @@ -239,6 +241,8 @@ sni_queue_cont(TSCont cont, TSEvent /* event ATS_UNUSED */, void * /* edata ATS_ (void)contp; Dbg(dbg_ctl, "Queued VC is too old (%ldms), erroring out", static_cast(age.count())); + TSUserArgSet(vc, gVCIdx, nullptr); + limiter->selector()->release(); TSVConnReenableEx(vc, TS_EVENT_ERROR); limiter->incrementMetric(RATE_LIMITER_METRIC_EXPIRED); } diff --git a/plugins/experimental/rate_limit/txn_limiter.cc b/plugins/experimental/rate_limit/txn_limiter.cc index 6eef9de0f56..489e47c0cbb 100644 --- a/plugins/experimental/rate_limit/txn_limiter.cc +++ b/plugins/experimental/rate_limit/txn_limiter.cc @@ -74,7 +74,7 @@ txn_queue_cont(TSCont cont, TSEvent /* event ATS_UNUSED */, void * /* edata ATS_ QueueTime now = std::chrono::system_clock::now(); // Only do this once per "loop" // Try to enable some queued txns (if any) if there are slots available - while (limiter->size() > 0 && limiter->reserve() != ReserveStatus::FULL) { // Can't be UNLIMITED here + while (limiter->size() > 0 && limiter->reserve() == ReserveStatus::RESERVED) { auto [txnp, contp, start_time] = limiter->pop(); std::chrono::milliseconds delay = std::chrono::duration_cast(now - start_time); diff --git a/plugins/experimental/stale_response/BodyData.h b/plugins/experimental/stale_response/BodyData.h index 0f4d02e40c6..5eb4caece76 100644 --- a/plugins/experimental/stale_response/BodyData.h +++ b/plugins/experimental/stale_response/BodyData.h @@ -63,6 +63,10 @@ struct BodyData { bool intercept_active = false; bool key_hash_active = false; uint32_t key_hash = 0; + // This BodyData object's contribution to ConfigInfo::body_data_memory_usage. + // The aggregate is used to enforce the plugin memory cap; this value lets + // cleanup subtract exactly this object's reservation, and only once. + int64_t memory_accounted = 0; private: struct Chunk { diff --git a/plugins/experimental/stale_response/ServerIntercept.cc b/plugins/experimental/stale_response/ServerIntercept.cc index 01b83a8cae8..0de1702da71 100644 --- a/plugins/experimental/stale_response/ServerIntercept.cc +++ b/plugins/experimental/stale_response/ServerIntercept.cc @@ -142,6 +142,7 @@ connShutdownDataDestory(SContData *cont_data) SRDBG(TAG_BAD, "[%s] didnt delete async active", __FUNCTION__); } } else { + body_memory_release(cont_data->plugin_config, cont_data->pBody); delete cont_data->pBody; } // clean up my cont diff --git a/plugins/experimental/stale_response/stale_response.cc b/plugins/experimental/stale_response/stale_response.cc index 0f1b4f2f46f..80b27082e61 100644 --- a/plugins/experimental/stale_response/stale_response.cc +++ b/plugins/experimental/stale_response/stale_response.cc @@ -69,6 +69,7 @@ create_response_info(void) resp_info->http_hdr_loc = TSHttpHdrCreate(resp_info->http_hdr_buf); resp_info->parser = TSHttpParserCreate(); resp_info->parsed = false; + resp_info->status = TS_HTTP_STATUS_NONE; return resp_info; } @@ -143,6 +144,26 @@ create_state_info(TSHttpTxn txnp, TSCont contp) return state; } +/*-----------------------------------------------------------------------------------------------*/ +void +body_memory_release(ConfigInfo *plugin_config, BodyData *pBody) +{ + if (!pBody) { + return; + } + + TSMutexLock(plugin_config->body_data_mutex); + int64_t memory_accounted = pBody->memory_accounted; + pBody->memory_accounted = 0; + if (memory_accounted > 0) { + plugin_config->body_data_memory_usage -= memory_accounted; + if (plugin_config->body_data_memory_usage < 0) { + plugin_config->body_data_memory_usage = 0; + } + } + TSMutexUnlock(plugin_config->body_data_mutex); +} + /*-----------------------------------------------------------------------------------------------*/ static void free_state_info(StateInfo *state) @@ -188,6 +209,7 @@ free_state_info(StateInfo *state) // this should be null but check and delete if (state->sie_body) { + body_memory_release(state->plugin_config, state->sie_body); delete state->sie_body; } state->sie_body = nullptr; @@ -287,7 +309,11 @@ async_remove_active(uint32_t key_hash, ConfigInfo *plugin_config) TSMutexLock(plugin_config->body_data_mutex); UintBodyMap::iterator pos = plugin_config->body_data->find(key_hash); if (pos != plugin_config->body_data->end()) { - plugin_config->body_data_memory_usage -= (pos->second)->getSize(); + plugin_config->body_data_memory_usage -= (pos->second)->memory_accounted; + if (plugin_config->body_data_memory_usage < 0) { + plugin_config->body_data_memory_usage = 0; + } + (pos->second)->memory_accounted = 0; delete pos->second; plugin_config->body_data->erase(pos); wasActive = true; @@ -419,7 +445,91 @@ get_cached_header_info(StateInfo *state) } /*-----------------------------------------------------------------------------------------------*/ +/** Record that the current origin fetch exceeded the plugin body memory cap. + * + * This centralizes the transition into the over-memory state so callers do not + * double-count the condition in the plugin statistic. + * + * @param[in,out] state Fetch state that should fall back instead of saving the + * new origin response. + */ static void +fetch_mark_over_max_memory(StateInfo *state) +{ + if (state->over_max_memory) { + return; + } + + state->over_max_memory = true; + SRDBG(TAG, "[%s] {%u} Over memory usage %" PRId64, __FUNCTION__, state->req_info->key_hash, + aync_memory_total_get(state->plugin_config)); + TSStatIntIncrement(state->plugin_config->rfc_stat_memory_over, 1); +} + +/*-----------------------------------------------------------------------------------------------*/ +/** Try to reserve plugin body memory for data about to be appended to a body. + * + * This keeps the plugin-wide memory total and the per-body accounting in sync: + * the aggregate enforces the configured cap, while @a pBody records exactly how + * much this object must release later. + * + * @param[in,out] plugin_config Plugin configuration that owns the shared memory + * accounting state. + * @param[in,out] pBody Body buffer receiving the reserved data. + * @param[in] size Number of bytes to reserve. + * @return @c true if the reservation fits within the cap, or @a size is zero or + * negative; @c false if reserving @a size would exceed the cap. + */ +static bool +body_memory_reserve(ConfigInfo *plugin_config, BodyData *pBody, int64_t size) +{ + bool reserved = false; + + if (size <= 0) { + return true; + } + + TSMutexLock(plugin_config->body_data_mutex); + if (size <= plugin_config->max_body_data_memory_usage - plugin_config->body_data_memory_usage) { + plugin_config->body_data_memory_usage += size; + pBody->memory_accounted += size; + reserved = true; + } + TSMutexUnlock(plugin_config->body_data_mutex); + + return reserved; +} + +/*-----------------------------------------------------------------------------------------------*/ +/** Determine how many origin response bytes an SIE/SWR fetch should read. + * + * This returns one byte more than the configured finite cap so the read path can + * detect an oversized origin response. The returned value is passed to + * @c TSVConnRead in @c fetch_resource(); any overflow byte then reaches + * @c fetch_save_response(), where @c body_memory_reserve() rejects the chunk and + * the fetch is marked over-memory instead of being treated as a successful, + * exactly-at-cap response. + * + * @param[in] plugin_config Plugin configuration holding the body memory cap. + * @return The VConn read limit to use for the origin fetch. + */ +static int64_t +fetch_read_limit(ConfigInfo *plugin_config) +{ + int64_t limit = plugin_config->max_body_data_memory_usage; + + if (limit < 0) { + return 0; + } + if (limit == INT64_MAX) { + return INT64_MAX; + } + // The `+ 1` is for a sentinel byte marking overflow. See the function comment above for details. + return limit + 1; +} + +/*-----------------------------------------------------------------------------------------------*/ +static bool fetch_save_response(StateInfo *state, BodyData *pBody) { TSIOBufferBlock block; @@ -429,14 +539,16 @@ fetch_save_response(StateInfo *state, BodyData *pBody) while (block != nullptr) { start = TSIOBufferBlockReadStart(block, state->resp_io_buf_reader, &avail); if (avail > 0) { - pBody->addChunk(start, avail); - // increase body_data_memory_usage only if content stored in plugin_config->body_data - if (pBody->key_hash_active) { - aync_memory_total_add(state->plugin_config, avail); + if (!body_memory_reserve(state->plugin_config, pBody, avail)) { + fetch_mark_over_max_memory(state); + return false; } + pBody->addChunk(start, avail); } block = TSIOBufferBlockNext(block); } + + return true; } /*-----------------------------------------------------------------------------------------------*/ @@ -467,33 +579,67 @@ fetch_parse_response(StateInfo *state) } /*-----------------------------------------------------------------------------------------------*/ -static void +static bool fetch_read_the_data(StateInfo *state) { - // always save data - if (state->cur_save_body) { - fetch_save_response(state, state->cur_save_body); - } else { - SRDBG(TAG_BAD, "[%s] no BodyData", __FUNCTION__); - } - // get the resp code if (!state->resp_info->parsed) { fetch_parse_response(state); } - // Consume data + int64_t avail = TSIOBufferReaderAvail(state->resp_io_buf_reader); + if (avail == TS_ERROR) { + TSError("[%s] Error while getting number of bytes available", __FUNCTION__); + state->fetch_error = true; + return false; + } + + if (state->sie_active && state->resp_info->parsed && valid_sie_status(state->resp_info->status)) { + TSIOBufferReaderConsume(state->resp_io_buf_reader, avail); + TSVIONDoneSet(state->r_vio, TSVIONDoneGet(state->r_vio) + avail); + return false; + } + + if (state->cur_save_body) { + if (!fetch_save_response(state, state->cur_save_body)) { + TSIOBufferReaderConsume(state->resp_io_buf_reader, avail); + TSVIONDoneSet(state->r_vio, TSVIONDoneGet(state->r_vio) + avail); + return false; + } + } else { + SRDBG(TAG_BAD, "[%s] no BodyData", __FUNCTION__); + } + TSIOBufferReaderConsume(state->resp_io_buf_reader, avail); TSVIONDoneSet(state->r_vio, TSVIONDoneGet(state->r_vio) + avail); + return true; } /*-----------------------------------------------------------------------------------------------*/ static void fetch_finish(StateInfo *state) { + bool const response_header_parsed = state->resp_info && state->resp_info->parsed; + TSHttpStatus const response_status = response_header_parsed ? state->resp_info->status : TS_HTTP_STATUS_NONE; + bool const fetch_failed = state->fetch_error || (!response_header_parsed && !state->over_max_memory); + + if (!response_header_parsed) { + SRDBG(TAG_BAD, "[%s] {%u} origin response header was not parsed", __FUNCTION__, state->req_info->key_hash); + } + SRDBG(TAG, "[%s] {%u} swr=%d sie=%d", __FUNCTION__, state->req_info->key_hash, state->swr_active, state->sie_active); if (state->swr_active) { SRDBG(TAG, "[%s] {%u} SWR Unlock URL / Post request", __FUNCTION__, state->req_info->key_hash); - if (state->sie_active && valid_sie_status(state->resp_info->status)) { + if (fetch_failed) { + SRDBG(TAG_BAD, "[%s] {%u} SWR fetch failed", __FUNCTION__, state->req_info->key_hash); + if (!async_remove_active(state->req_info->key_hash, state->plugin_config)) { + SRDBG(TAG_BAD, "[%s] {%u} didnt delete async active", __FUNCTION__, state->req_info->key_hash); + } + } else if (state->over_max_memory) { + SRDBG(TAG, "[%s] {%u} SWR response exceeded memory limit", __FUNCTION__, state->req_info->key_hash); + if (!async_remove_active(state->req_info->key_hash, state->plugin_config)) { + SRDBG(TAG_BAD, "[%s] {%u} didnt delete async active", __FUNCTION__, state->req_info->key_hash); + } + } else if (state->sie_active && valid_sie_status(response_status)) { SRDBG(TAG, "[%s] {%u} SWR Bad Data skipping", __FUNCTION__, state->req_info->key_hash); if (!async_remove_active(state->req_info->key_hash, state->plugin_config)) { SRDBG(TAG_BAD, "[%s] {%u} didnt delete async active", __FUNCTION__, state->req_info->key_hash); @@ -504,8 +650,8 @@ fetch_finish(StateInfo *state) } } else // state->sie_active { - SRDBG(TAG, "[%s] {%u} SIE in sync path Reenable %d", __FUNCTION__, state->req_info->key_hash, state->resp_info->status); - if (valid_sie_status(state->resp_info->status)) { + SRDBG(TAG, "[%s] {%u} SIE in sync path Reenable %d", __FUNCTION__, state->req_info->key_hash, response_status); + if (valid_sie_status(response_status)) { SRDBG(TAG, "[%s] {%u} SIE sending stale data", __FUNCTION__, state->req_info->key_hash); if (state->plugin_config->log_info.object && (state->plugin_config->log_info.all || state->plugin_config->log_info.stale_if_error)) { @@ -516,6 +662,12 @@ fetch_finish(StateInfo *state) } // send out the stale data send_stale_response(state); + } else if (fetch_failed) { + SRDBG(TAG_BAD, "[%s] SIE {%u} fetch failed; sending stale data", __FUNCTION__, state->req_info->key_hash); + send_stale_response(state); + } else if (state->over_max_memory) { + SRDBG(TAG, "[%s] SIE {%u} response exceeded memory limit; sending stale data", __FUNCTION__, state->req_info->key_hash); + send_stale_response(state); } else { SRDBG(TAG, "[%s] SIE {%u} sending new data", __FUNCTION__, state->req_info->key_hash); // load the data as if we are OS by ServerIntercept @@ -555,8 +707,14 @@ fetch_consume(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) case TS_EVENT_VCONN_READ_READY: // save the data and parse header if needed - fetch_read_the_data(state); - TSVIOReenable(state->r_vio); + if (fetch_read_the_data(state)) { + TSVIOReenable(state->r_vio); + } else { + TSVConnAbort(state->vconn, TS_VC_CLOSE_ABORT); + fetch_finish(state); + free_state_info(state); + TSContDestroy(contp); + } break; case TS_EVENT_VCONN_READ_COMPLETE: @@ -632,7 +790,7 @@ fetch_resource(TSCont contp, TSEvent, void *) // connect , setup read , write assert(state->req_info->client_addr != nullptr); state->vconn = TSHttpConnect(state->req_info->client_addr); - state->r_vio = TSVConnRead(state->vconn, consume_contp, state->resp_io_buf, INT64_MAX); + state->r_vio = TSVConnRead(state->vconn, consume_contp, state->resp_io_buf, fetch_read_limit(state->plugin_config)); state->w_vio = TSVConnWrite(state->vconn, consume_contp, state->req_io_buf_reader, TSIOBufferReaderAvail(state->req_io_buf_reader)); @@ -734,13 +892,13 @@ transaction_handler(TSCont contp, TSEvent event, void *edata) state->swr_active = ((((state->txn_start - chi->date) + 1) < (chi->max_age + chi->stale_while_revalidate)) && chi->stale_while_revalidate); state->sie_active = ((((state->txn_start - chi->date) + 1) < (chi->max_age + chi->stale_if_error)) && chi->stale_if_error); - state->over_max_memory = (aync_memory_total_get(plugin_config) > plugin_config->max_body_data_memory_usage); + state->over_max_memory = (aync_memory_total_get(plugin_config) >= plugin_config->max_body_data_memory_usage); SRDBG(TAG, "[%s] {%u} CacheLookup Stale swr=%d sie=%d over=%d", __FUNCTION__, state->req_info->key_hash, state->swr_active, state->sie_active, state->over_max_memory); // see if we are using too much memory and if so do not swr/sie if (state->over_max_memory) { - SRDBG(TAG, "[%s] {%u} Over memory Usage %" PRId64, __FUNCTION__, state->req_info->key_hash, + SRDBG(TAG, "[%s] {%u} Over memory usage %" PRId64, __FUNCTION__, state->req_info->key_hash, aync_memory_total_get(plugin_config)); TSStatIntIncrement(state->plugin_config->rfc_stat_memory_over, 1); } diff --git a/plugins/experimental/stale_response/stale_response.h b/plugins/experimental/stale_response/stale_response.h index a0a9071ea58..8d9f9be8fff 100644 --- a/plugins/experimental/stale_response/stale_response.h +++ b/plugins/experimental/stale_response/stale_response.h @@ -130,6 +130,7 @@ struct StateInfo { bool swr_active = false; bool sie_active = false; bool over_max_memory = false; + bool fetch_error = false; TSIOBuffer req_io_buf = nullptr; TSIOBuffer resp_io_buf = nullptr; TSIOBufferReader req_io_buf_reader = nullptr; @@ -150,6 +151,7 @@ struct StateInfo { BodyData *async_check_active(uint32_t key_hash, ConfigInfo *plugin_config); bool async_check_and_add_active(uint32_t key_hash, ConfigInfo *plugin_config); bool async_remove_active(uint32_t key_hash, ConfigInfo *plugin_config); +void body_memory_release(ConfigInfo *plugin_config, BodyData *pBody); // 500, 502, 503, 504 inline bool diff --git a/plugins/experimental/stream_editor/stream_editor.cc b/plugins/experimental/stream_editor/stream_editor.cc index 6883b55734f..8c0cb25fcc7 100644 --- a/plugins/experimental/stream_editor/stream_editor.cc +++ b/plugins/experimental/stream_editor/stream_editor.cc @@ -565,7 +565,7 @@ struct contdata_t { TSIOBuffer out_buf = nullptr; TSIOBufferReader out_rd = nullptr; TSVIO out_vio = nullptr; - ruleset_up_t rules; + ruleset_up_t rules{std::make_unique()}; std::string contbuf; size_t contbuf_sz = 0; int64_t bytes_in = 0; diff --git a/plugins/experimental/txn_box/plugin/src/Modifier.cc b/plugins/experimental/txn_box/plugin/src/Modifier.cc index cfef23a0fa8..6e5beaff8e0 100644 --- a/plugins/experimental/txn_box/plugin/src/Modifier.cc +++ b/plugins/experimental/txn_box/plugin/src/Modifier.cc @@ -25,12 +25,15 @@ #include "txn_box/Config.h" #include "txn_box/Comparison.h" #include "txn_box/yaml_util.h" +#include "tsutil/LocalBuffer.h" using swoc::Errata; using swoc::Rv; using swoc::TextView; using namespace swoc::literals; +static constexpr size_t FILTER_LIST_LOCAL_BUFFER_SIZE = 32; + Errata Modifier::define(swoc::TextView name, Modifier::Worker const &f) { @@ -430,10 +433,10 @@ Mod_filter::operator()(Context &ctx, Feature &feature) { Feature zret{}; if (feature.is_list()) { - auto src = std::get(feature); - auto farray = static_cast(alloca(sizeof(Feature) * src.count())); - feature_type_for dst{farray, src.count()}; - unsigned dst_idx = 0; + auto src = std::get(feature); + ts::LocalBuffer farray(src.count()); + feature_type_for dst = {farray.data(), src.count()}; + unsigned dst_idx = 0; for (Feature f = feature; !is_nil(f); f = cdr(f)) { Feature item = car(f); auto c = _cases(ctx, item); diff --git a/plugins/experimental/txn_box/plugin/src/ts_util.cc b/plugins/experimental/txn_box/plugin/src/ts_util.cc index 3df9c750ed3..a1881768956 100644 --- a/plugins/experimental/txn_box/plugin/src/ts_util.cc +++ b/plugins/experimental/txn_box/plugin/src/ts_util.cc @@ -23,9 +23,6 @@ #include #include #include -#if __has_include() -#include -#endif #include @@ -36,6 +33,8 @@ #include #include +#include "tscore/ink_config.h" +#include "tsutil/LocalBuffer.h" #include "txn_box/ts_util.h" using swoc::BufferWriter; @@ -57,6 +56,9 @@ DbgCtl txn_box_dbg_ctl{DEBUG_TAG}; namespace ts { namespace swoc = ::swoc; // Import to avoid global naming weirdness. + +static constexpr size_t HOST_FIELD_LOCAL_BUFFER_SIZE = TS_MAX_HOST_NAME_LEN; + /* ------------------------------------------------------------------------------------ */ const swoc::Lexicon TSRecordDataTypeNames{ @@ -503,8 +505,10 @@ ts::HttpRequest::host_set(swoc::TextView const &host) auto text = field.value(); TextView host_token, port_token; if (swoc::IPEndpoint::tokenize(text, &host_token, &port_token)) { - size_t n = host.size() + 1 + port_token.size(); - swoc::FixedBufferWriter w{static_cast(alloca(n)), n}; + size_t n = host.size() + 1 + port_token.size(); + + ts::LocalBuffer buffer(n); + swoc::FixedBufferWriter w{buffer.data(), n}; if (port_token.size()) { w.print("{}:{}", host, port_token); } else { @@ -532,8 +536,10 @@ ts::HttpRequest::port_set(in_port_t port) auto text = field.value(); TextView host_token, port_token; if (swoc::IPEndpoint::tokenize(text, &host_token, &port_token)) { - size_t n = host_token.size() + 1 + std::numeric_limits::max_digits10; - swoc::FixedBufferWriter w{static_cast(alloca(n)), n}; + size_t n = host_token.size() + 1 + std::numeric_limits::digits10 + 1; + + ts::LocalBuffer buffer(n); + swoc::FixedBufferWriter w{buffer.data(), n}; w.write(host_token); if (port > 0) { w.write(':'); diff --git a/plugins/experimental/uri_signing/normalize.cc b/plugins/experimental/uri_signing/normalize.cc index 7783a86de08..7be1bc1e8b8 100644 --- a/plugins/experimental/uri_signing/normalize.cc +++ b/plugins/experimental/uri_signing/normalize.cc @@ -23,21 +23,25 @@ #include +#include "tsutil/LocalBuffer.h" + /* Remove Dot Algorithm outlined in RFC3986 section 5.2.4 * Function writes normalizes path and writes to ret_buffer */ int remove_dot_segments(const char *path, int path_ct, char *ret_buffer, int buff_ct) { - /* Ensure buffer is at least the size of the path */ - if (buff_ct < path_ct) { + /* Validate pointers and ensure buffer has room for the path plus NUL terminator */ + if (path == nullptr || ret_buffer == nullptr || path_ct < 0 || buff_ct <= path_ct) { PluginDebug("Path buffer not large enough"); return -1; } /* Create an input buffer that we can change */ - char inBuff[path_ct + 1]; - memset(inBuff, 0, path_ct + 1); - strcpy(inBuff, path); + size_t inBuff_size = static_cast(path_ct) + 1; + ts::LocalBuffer inBuff_storage(inBuff_size); + char *inBuff = inBuff_storage.data(); + memset(inBuff, 0, inBuff_size); + memcpy(inBuff, path, path_ct); const char *path_end = inBuff + path_ct; char *seg_start = inBuff; @@ -108,20 +112,18 @@ remove_dot_segments(const char *path, int path_ct, char *ret_buffer, int buff_ct } /* Write subsequent characters to buffer */ - while (*seg_start != '/') { + while (*seg_start != '/' && *seg_start != '\0') { *write_buffer = *seg_start; write_buffer++; - if (*seg_start == 0) { - break; - } seg_start++; } } seg_start = seg_end; } + *write_buffer = '\0'; PluginDebug("Normalized Path: %s", ret_buffer); - return strlen(ret_buffer); + return static_cast(write_buffer - ret_buffer); } /* Function percent decodes uri_ct characters of the string uri and writes it to the decoded_uri @@ -196,10 +198,11 @@ percent_decode(const char *uri, int uri_ct, char *decoded_uri, bool lower) } /* This function takes a uri and an initialized buffer to populate with the normalized uri. - * Returns non zero for error + * Returns non zero for error. * - * The buffer provided must be at least the length of the uri + 1 as the normalized uri will - * potentially be one char larger than the original uri if a backslash is added to the path. + * The buffer provided must be at least uri_ct + 2 bytes: the normalized uri can be one + * character larger than the original (a trailing '/' added when the path is empty) plus + * one byte for the NUL terminator. * * The normalization function returns a string with the following modifications * 1. Lowecase protocol/domain @@ -207,24 +210,31 @@ percent_decode(const char *uri, int uri_ct, char *decoded_uri, bool lower) * 3. Alphabetical percent encoded octet values are toupper * 4. Non-reserved percent encoded octet values are decoded * 5. The Port is removed if it is default - * 6. Defaults to a single backslash for the path segment if path segment is empty + * 6. Defaults to a single forward slash for the path segment if path segment is empty */ int normalize_uri(const char *uri, int uri_ct, char *normal_uri, int normal_ct) { + /* Validate inputs before any pointer arithmetic or dereferences. + * The output buffer must hold uri_ct + 2 bytes: the URI itself plus a + * possible trailing '/' added when the authority has an empty path, plus + * the NUL terminator. The overflow-safe form of normal_ct < uri_ct + 2 + * is normal_ct - 1 <= uri_ct. + */ + if (uri == nullptr || normal_uri == nullptr || uri_ct < 0 || normal_ct < 2 || normal_ct - 1 <= uri_ct) { + PluginDebug("Buffer to Normalize URI not large enough."); + return -1; + } + PluginDebug("Normalizing URI: %s", uri); /* Buffer provided must be large enough to store the uri plus one additional char */ const char *uri_end = uri + uri_ct; const char *buff_end = normal_uri + normal_ct; - if ((normal_uri == nullptr) || (normal_uri && normal_ct < uri_ct + 1)) { - PluginDebug("Buffer to Normalize URI not large enough."); - return -1; - } - /* Initialize a path buffer to pass to path normalization function later on */ - char path_buffer[normal_ct]; + ts::LocalBuffer path_buffer_storage(normal_ct); + char *path_buffer = path_buffer_storage.data(); memset(path_buffer, 0, normal_ct); /* Comp variables store starting/ending indexes for each uri component as uri is parsed. diff --git a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc index 0672c522456..bcba4f9d4c7 100644 --- a/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc +++ b/plugins/experimental/uri_signing/unit_tests/uri_signing_test.cc @@ -21,6 +21,7 @@ */ #include +#include extern "C" { #include @@ -34,6 +35,7 @@ extern "C" { #include "../config.h" #include "tscore/Version.h" +#include "tsutil/LocalBuffer.h" AppVersionInfo appVersionInfo; @@ -169,43 +171,47 @@ jwt_parsing_helper(const char *jwt_string) bool normalize_uri_helper(const char *uri, const char *expected_normal) { - size_t uri_ct = strlen(uri); - int buff_size = uri_ct + 2; + size_t uri_ct = strlen(uri); + size_t requested_ct = uri_ct + 2; int err; - char *uri_normal = static_cast(malloc(buff_size)); - memset(uri_normal, 0, buff_size); - err = normalize_uri(uri, uri_ct, uri_normal, buff_size); + ts::LocalBuffer uri_normal(requested_ct); + memset(uri_normal.data(), 0, requested_ct); + + err = normalize_uri(uri, static_cast(uri_ct), uri_normal.data(), static_cast(requested_ct)); if (err) { - free(uri_normal); return false; } - if (expected_normal && strcmp(expected_normal, uri_normal) == 0) { - free(uri_normal); + if (expected_normal && strcmp(expected_normal, uri_normal.data()) == 0) { return true; } - free(uri_normal); return false; } bool remove_dot_helper(const char *path, const char *expected_path) { - fprintf(stderr, "Removing Dot Segments from Path: %s\n", path); size_t path_ct = strlen(path); - path_ct++; - int new_ct; - char path_buffer[path_ct]; - memset(path_buffer, 0, path_ct); - new_ct = remove_dot_segments(path, path_ct, path_buffer, path_ct); + if (path_ct > 120) { + fprintf(stderr, "Removing Dot Segments from Path: %.120s... (%zu bytes)\n", path, path_ct); + } else { + fprintf(stderr, "Removing Dot Segments from Path: %s\n", path); + } + size_t requested_ct = path_ct + 1; + int new_ct; + + ts::LocalBuffer path_buffer(requested_ct); + memset(path_buffer.data(), 0, requested_ct); + + new_ct = remove_dot_segments(path, static_cast(path_ct), path_buffer.data(), static_cast(requested_ct)); if (new_ct < 0) { return false; - } else if (strcmp(expected_path, path_buffer) == 0) { + } else if (strcmp(expected_path, path_buffer.data()) == 0) { return true; } else { return false; @@ -216,22 +222,23 @@ bool jws_parsing_helper(const char *uri, const char *paramName, const char *expected_strip) { bool resp; - size_t uri_ct = strlen(uri); - size_t strip_ct = 0; + size_t uri_ct = strlen(uri); + size_t strip_ct = 0; + size_t requested_ct = uri_ct + 1; - char *uri_strip = static_cast(malloc(uri_ct + 1)); - memset(uri_strip, 0, uri_ct + 1); + ts::LocalBuffer uri_strip(requested_ct); + memset(uri_strip.data(), 0, requested_ct); - cjose_jws_t *jws = get_jws_from_uri(uri, uri_ct, paramName, uri_strip, uri_ct, &strip_ct); + cjose_jws_t *jws = get_jws_from_uri(uri, uri_ct, paramName, uri_strip.data(), requested_ct, &strip_ct); if (jws) { resp = true; if (expected_strip != nullptr) { - if (strcmp(uri_strip, expected_strip) != 0) { + if (strcmp(uri_strip.data(), expected_strip) != 0) { resp = false; } } else { // expected_strip == nullptr means we expect uri_strip to be empty - if (uri_strip[0] != '\0') { + if (uri_strip.data()[0] != '\0') { resp = false; } } @@ -239,7 +246,6 @@ jws_parsing_helper(const char *uri, const char *paramName, const char *expected_ resp = false; } cjose_jws_release(jws); - free(uri_strip); return resp; } @@ -441,7 +447,7 @@ TEST_CASE("2", "[JWSFromURLTest]") } } -TEST_CASE("3", "[RemoveDotSegmentsTest]") +TEST_CASE("3", "[RemoveDotSegmentsTest][large-path]") { INFO("TEST 3, Test Removal of Dot Segments From Paths"); @@ -534,6 +540,24 @@ TEST_CASE("3", "[RemoveDotSegmentsTest]") { REQUIRE(remove_dot_helper("/foo/bar/././something/../foobar", "/foo/bar/foobar")); } + + SECTION("Large path normalization scenario") + { + std::string large_path = "/" + std::string(70000, 'a'); + REQUIRE(remove_dot_helper(large_path.c_str(), large_path.c_str())); + } + + SECTION("500 plus dot-segment normalization scenario") + { + std::string many_segments; + many_segments.reserve(4 * 512 + 4); + for (int i = 0; i < 512; ++i) { + many_segments += "/../"; + } + many_segments += "bar"; + REQUIRE(remove_dot_helper(many_segments.c_str(), "/bar")); + } + fprintf(stderr, "\n"); } @@ -650,6 +674,133 @@ TEST_CASE("4", "[NormalizeTest]") { REQUIRE(!normalize_uri_helper("http://?/", nullptr)); } + + SECTION("Userinfo with colon-separated credentials and default port") + { + REQUIRE(normalize_uri_helper("https://admin:443@cdn.example.com:443/content/video.mp4", + "https://admin:443@cdn.example.com/content/video.mp4")); + } + + SECTION("Userinfo with numeric password over http with default port") + { + REQUIRE(normalize_uri_helper("http://user:80@origin.example.net:80/assets/img.png", + "http://user:80@origin.example.net/assets/img.png")); + } + + SECTION("Userinfo numeric password over http without host port") + { + REQUIRE( + normalize_uri_helper("http://deploy:80@origin.example.net/release/v2", "http://deploy:80@origin.example.net/release/v2")); + } + + SECTION("Userinfo with colon but no host port") + { + REQUIRE( + normalize_uri_helper("https://token:443@storage.example.io/bucket/obj", "https://token:443@storage.example.io/bucket/obj")); + } + + SECTION("Userinfo with non-default numeric value and host default port") + { + REQUIRE( + normalize_uri_helper("https://svc:8080@api.example.com:443/v1/resource", "https://svc:8080@api.example.com/v1/resource")); + } + + SECTION("Userinfo containing only digits after colon") + { + REQUIRE(normalize_uri_helper("http://node:3000@cluster.local:80/healthz", "http://node:3000@cluster.local/healthz")); + } + + SECTION("Userinfo with empty password and host port") + { + REQUIRE(normalize_uri_helper("https://user:@files.example.org:443/doc.pdf", "https://user:@files.example.org/doc.pdf")); + } + + SECTION("Simple username without password and host default port") + { + REQUIRE(normalize_uri_helper("http://anonymous@mirror.example.com:80/pub/archive.tar.gz", + "http://anonymous@mirror.example.com/pub/archive.tar.gz")); + } + + SECTION("Userinfo with percent-encoded colon and host default port") + { + REQUIRE( + normalize_uri_helper("https://user%3Aname:pass@host.example.com:443/path", "https://user%3Aname:pass@host.example.com/path")); + } + + SECTION("Userinfo with multiple colons") + { + REQUIRE(normalize_uri_helper("http://a:b:c@www.example.com:80/index.html", "http://a:b:c@www.example.com/index.html")); + } + + SECTION("Userinfo preserves case while host is lowered") + { + REQUIRE(normalize_uri_helper("https://MyUser:MyPass@WWW.EXAMPLE.COM:443/Path", "https://MyUser:MyPass@www.example.com/Path")); + } + + SECTION("Userinfo with non-default host port preserved") + { + REQUIRE( + normalize_uri_helper("https://ops:deploy@internal.example.com:8443/api", "https://ops:deploy@internal.example.com:8443/api")); + } + + SECTION("Userinfo with query string and fragment") + { + REQUIRE(normalize_uri_helper("http://cache:secret@edge.example.net:80/video?quality=hd#t=10", + "http://cache:secret@edge.example.net/video?quality=hd#t=10")); + } + + SECTION("Userinfo with encoded at-sign in username") + { + REQUIRE(normalize_uri_helper("http://foo%40bar:baz@www.example.com:80/", "http://foo%40bar:baz@www.example.com/")); + } + + SECTION("Long userinfo with embedded port-like substring") + { + REQUIRE(normalize_uri_helper("https://serviceaccount:443secret@backend.example.com:443/rpc", + "https://serviceaccount:443secret@backend.example.com/rpc")); + } + + SECTION("Userinfo digits matching port pattern but not at boundary") + { + REQUIRE(normalize_uri_helper("http://x:12380@lb.example.com:80/status", "http://x:12380@lb.example.com/status")); + } + + SECTION("Userinfo and host both without port") + { + REQUIRE(normalize_uri_helper("https://readonly:tok@repo.example.com/org/project", + "https://readonly:tok@repo.example.com/org/project")); + } + + SECTION("Userinfo with path dot-segment removal") + { + REQUIRE(normalize_uri_helper("https://ci:runner@build.example.com:443/workspace/../output/artifact.zip", + "https://ci:runner@build.example.com/output/artifact.zip")); + } + + SECTION("FQDN-style userinfo with port-like suffix and host default port") + { + REQUIRE(normalize_uri_helper("https://registry.internal:443@cdn.example.com:443/v2/image/manifests/latest", + "https://registry.internal:443@cdn.example.com/v2/image/manifests/latest")); + } + + SECTION("FQDN-style userinfo with port-like suffix and host without port") + { + REQUIRE(normalize_uri_helper("https://upstream.proxy.local:443@edge.example.net/assets/bundle.js", + "https://upstream.proxy.local:443@edge.example.net/assets/bundle.js")); + } + + SECTION("FQDN-style userinfo with http default port value") + { + REQUIRE(normalize_uri_helper("http://cache.node.dc1:80@origin.example.com:80/media/stream.m3u8", + "http://cache.node.dc1:80@origin.example.com/media/stream.m3u8")); + } + + SECTION("FQDN-style userinfo without port and host default port") + { + REQUIRE(normalize_uri_helper("https://forwarder.mesh.internal@gateway.example.com:443/api/v1/tokens", + "https://forwarder.mesh.internal@gateway.example.com/api/v1/tokens")); + } + fprintf(stderr, "\n"); } @@ -784,15 +935,18 @@ TEST_CASE("7", "[TestsConfig]") bool jws_validation_helper(const char *url, const char *package, struct config *cfg) { - size_t url_ct = strlen(url); - size_t strip_ct = 0; - char uri_strip[url_ct + 1]; - memset(uri_strip, 0, sizeof uri_strip); - cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package, uri_strip, url_ct, &strip_ct); + size_t url_ct = strlen(url); + size_t strip_ct = 0; + size_t requested_ct = url_ct + 1; + + ts::LocalBuffer uri_strip(requested_ct); + memset(uri_strip.data(), 0, requested_ct); + + cjose_jws_t *jws = get_jws_from_uri(url, url_ct, package, uri_strip.data(), requested_ct, &strip_ct); if (!jws) { return false; } - struct jwt *jwt = validate_jws(jws, cfg, uri_strip, strip_ct); + struct jwt *jwt = validate_jws(jws, cfg, uri_strip.data(), strip_ct); cjose_jws_release(jws); if (!jwt) { return false; diff --git a/plugins/experimental/url_sig/url_sig.cc b/plugins/experimental/url_sig/url_sig.cc index 7e55d90b30a..5f4350ac48c 100644 --- a/plugins/experimental/url_sig/url_sig.cc +++ b/plugins/experimental/url_sig/url_sig.cc @@ -622,6 +622,10 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) case AF_INET: Dbg(dbg_ctl, "ip->sa_family: AF_INET"); has_path_params == false ? (pp = strstr(cp, "&")) : (pp = strstr(cp, ";")); + if (pp == nullptr) { + err_log(url, url_len, "Malformed C parameter: missing delimiter."); + goto deny; + } if ((pp - cp) > INET_ADDRSTRLEN - 1 || (pp - cp) < 4) { err_log(url, url_len, "IP address string too long or short."); goto deny; @@ -639,6 +643,10 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) case AF_INET6: Dbg(dbg_ctl, "ip->sa_family: AF_INET6"); has_path_params == false ? (pp = strstr(cp, "&")) : (pp = strstr(cp, ";")); + if (pp == nullptr) { + err_log(url, url_len, "Malformed C parameter: missing delimiter."); + goto deny; + } if ((pp - cp) > INET6_ADDRSTRLEN - 1 || (pp - cp) < 4) { err_log(url, url_len, "IP address string too long or short."); goto deny; diff --git a/plugins/header_rewrite/cidr.h b/plugins/header_rewrite/cidr.h new file mode 100644 index 00000000000..40ab692b424 --- /dev/null +++ b/plugins/header_rewrite/cidr.h @@ -0,0 +1,68 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// CIDR masking helpers for %{CIDR:...}, kept header-only for unit testing. +#pragma once + +#include +#include +#include +#include + +// /0 yields a 0 mask, avoiding the undefined `<< 32`. Out-of-range prefixes +// are clamped into [0, 32] to keep the shift well-defined. +inline in_addr_t +cidr_v4_mask(int prefix) +{ + if (prefix <= 0) { + return 0; + } + if (prefix >= 32) { + return htonl(UINT32_MAX); + } + return htonl(UINT32_MAX << (32 - prefix)); +} + +// Trailing bytes to clear, plus a high-bit mask for the partial byte (0xff if aligned). +// Out-of-range prefixes are clamped into [0, 128] so the math stays well-defined. +inline void +cidr_v6_params(int prefix, int &zero_bytes, unsigned char &mask) +{ + if (prefix < 0) { + prefix = 0; + } else if (prefix > 128) { + prefix = 128; + } + + int const rem_bits = prefix % 8; + + zero_bytes = (128 - prefix) / 8; + mask = rem_bits ? static_cast(0xff << (8 - rem_bits)) : 0xff; +} + +// Clear the trailing zero_bytes, then keep the high bits of the byte above them. +inline void +cidr_apply_v6(in6_addr &addr, int zero_bytes, unsigned char mask) +{ + if (zero_bytes > 0) { + memset(&addr.s6_addr[16 - zero_bytes], 0, zero_bytes); + } + if (mask != 0xff) { + addr.s6_addr[16 - zero_bytes - 1] &= mask; + } +} diff --git a/plugins/header_rewrite/conditions.cc b/plugins/header_rewrite/conditions.cc index 1953864d798..e3ee85227ef 100644 --- a/plugins/header_rewrite/conditions.cc +++ b/plugins/header_rewrite/conditions.cc @@ -35,6 +35,7 @@ #include "ts/ts.h" #include "conditions.h" +#include "cidr.h" #include "lulu.h" static const sockaddr *getClientAddr(TSHttpTxn txnp, int txn_private_slot); @@ -1074,8 +1075,7 @@ ConditionCidr::set_qualifier(const std::string &q) Dbg(pi_dbg_ctl, "\tParsing %%{CIDR:%s} qualifier", q.c_str()); cidr = strtol(q.c_str(), &endp, 10); if (cidr >= 0 && cidr <= 32) { - _v4_mask.s_addr = UINT32_MAX >> (32 - cidr); - _v4_cidr = cidr; + _v4_cidr = cidr; if (endp && (*endp == ',' || *endp == '/' || *endp == ':')) { cidr = strtol(endp + 1, nullptr, 10); if (cidr >= 0 && cidr <= 128) { @@ -1128,12 +1128,7 @@ ConditionCidr::append_value(std::string &s, const Resources &res) char resource[INET6_ADDRSTRLEN]; struct in6_addr ipv6 = reinterpret_cast(addr)->sin6_addr; - if (_v6_zero_bytes > 0) { - memset(&ipv6.s6_addr[16 - _v6_zero_bytes], 0, _v6_zero_bytes); - } - if (_v6_mask != 0xff) { - ipv6.s6_addr[16 - _v6_zero_bytes] &= _v6_mask; - } + cidr_apply_v6(ipv6, _v6_zero_bytes, _v6_mask); inet_ntop(AF_INET6, &ipv6, resource, INET6_ADDRSTRLEN); if (resource[0]) { s += resource; @@ -1149,9 +1144,8 @@ ConditionCidr::append_value(std::string &s, const Resources &res) void ConditionCidr::_create_masks() { - _v4_mask.s_addr = htonl(UINT32_MAX << (32 - _v4_cidr)); - _v6_zero_bytes = (128 - _v6_cidr) / 8; - _v6_mask = 0xff >> ((128 - _v6_cidr) % 8); + _v4_mask.s_addr = cidr_v4_mask(_v4_cidr); + cidr_v6_params(_v6_cidr, _v6_zero_bytes, _v6_mask); } void diff --git a/plugins/header_rewrite/conditions.h b/plugins/header_rewrite/conditions.h index 65020646b88..809742d1a96 100644 --- a/plugins/header_rewrite/conditions.h +++ b/plugins/header_rewrite/conditions.h @@ -213,7 +213,7 @@ class ConditionCookie : public Condition end = buf + buf_len; while (start < end) { - if (strncasecmp(start, name, name_len) != 0) { + if (end - start < name_len || strncasecmp(start, name, name_len) != 0) { goto skip; } diff --git a/plugins/header_rewrite/header_rewrite_test.cc b/plugins/header_rewrite/header_rewrite_test.cc index 6f6026e3df3..68997debd08 100644 --- a/plugins/header_rewrite/header_rewrite_test.cc +++ b/plugins/header_rewrite/header_rewrite_test.cc @@ -26,8 +26,11 @@ #include #include #include +#include +#include #include "parser.h" +#include "cidr.h" #if TS_USE_HRW_MAXMINDDB #include @@ -728,10 +731,91 @@ test_maxmind_geo() } #endif +static std::string +cidr_mask_v6_str(const char *addr_str, int prefix) +{ + in6_addr addr{}; + + if (inet_pton(AF_INET6, addr_str, &addr) != 1) { + return ""; + } + + int zero_bytes = 0; + unsigned char mask = 0; + + cidr_v6_params(prefix, zero_bytes, mask); + cidr_apply_v6(addr, zero_bytes, mask); + + char out[INET6_ADDRSTRLEN]{}; + + if (inet_ntop(AF_INET6, &addr, out, sizeof(out)) == nullptr) { + return ""; + } + + return std::string(out); +} + +int +test_cidr() +{ + int errors = 0; + + // IPv4 masks, in network byte order. /0 must be 0 (and must not shift by 32). + // Out-of-range prefixes clamp to [0, 32]. + const std::pair v4cases[] = { + {-1, 0 }, // clamps to /0 + {0, 0 }, + {24, htonl(0xFFFFFF00u)}, + {32, htonl(0xFFFFFFFFu)}, + {33, htonl(0xFFFFFFFFu)}, // clamps to /32 + }; + + for (auto const &[prefix, expect] : v4cases) { + in_addr_t const got = cidr_v4_mask(prefix); + + if (got != expect) { + std::cerr << "FAIL: cidr_v4_mask(/" << prefix << ") = " << std::hex << got << ", expected " << expect << std::dec + << std::endl; + ++errors; + } else { + std::cout << " PASS: cidr_v4_mask(/" << prefix << ")" << std::endl; + } + } + + // IPv6 masks. The source has bits set in every byte so non-byte-aligned + // prefixes (the actual regression) produce distinct results. Out-of-range + // prefixes clamp to [0, 128]. + const char *src = "2001:db8:abcd:ef13:3456:789a:bcde:f012"; + const std::pair v6cases[] = { + {129, "2001:db8:abcd:ef13:3456:789a:bcde:f012"}, // clamps to /128 + {128, "2001:db8:abcd:ef13:3456:789a:bcde:f012"}, + {64, "2001:db8:abcd:ef13::" }, + {63, "2001:db8:abcd:ef12::" }, + {60, "2001:db8:abcd:ef10::" }, + {52, "2001:db8:abcd:e000::" }, + {48, "2001:db8:abcd::" }, + {0, "::" }, + {-1, "::" }, // clamps to /0 + }; + + for (auto const &[prefix, expect] : v6cases) { + std::string const got = cidr_mask_v6_str(src, prefix); + + if (got != expect) { + std::cerr << "FAIL: " << src << " /" << prefix << " = " << got << ", expected " << expect << std::endl; + ++errors; + } else { + std::cout << " PASS: " << src << " /" << prefix << " = " << got << std::endl; + } + } + + return errors; +} + int main() { - if (test_parsing() || test_processing() || test_tokenizer()) { + if (test_parsing() || test_processing() || test_tokenizer() || test_cidr()) { return 1; } diff --git a/plugins/header_rewrite/operators.cc b/plugins/header_rewrite/operators.cc index 133991228cd..bdb9af636e7 100644 --- a/plugins/header_rewrite/operators.cc +++ b/plugins/header_rewrite/operators.cc @@ -58,7 +58,11 @@ handleFetchEvents(TSCont cont, TSEvent event, void *edata) TSHttpHdrTypeSet(hdr_buf, hdr_loc, TS_HTTP_TYPE_RESPONSE); if (TSHttpHdrParseResp(parser, hdr_buf, hdr_loc, &data_start, data_end) == TS_PARSE_DONE) { - TSHttpTxnErrorBodySet(http_txn, TSstrdup(data_start), (data_end - data_start), nullptr); + size_t body_len = data_end - data_start; + char *body = static_cast(TSmalloc(body_len + 1)); + memcpy(body, data_start, body_len); + body[body_len] = '\0'; + TSHttpTxnErrorBodySet(http_txn, body, body_len, nullptr); } else { TSWarning("[%s] Unable to parse set-custom-body fetch response", __FUNCTION__); } @@ -1012,7 +1016,7 @@ CookieHelper::cookieModifyHelper(const char *cookies, const size_t cookies_len, for (; idx < cookies_len && std::isspace(cookies[idx]); idx++) { ; } - if (0 == strncmp(cookies + idx, cookie_key.c_str(), cookie_key.size())) { + if (cookies_len - idx >= cookie_key.size() && 0 == memcmp(cookies + idx, cookie_key.c_str(), cookie_key.size())) { size_t key_start_idx = idx; // advance to past the name and any subsequent spaces for (idx += cookie_key.size(); idx < cookies_len && std::isspace(cookies[idx]); idx++) { @@ -1030,12 +1034,7 @@ CookieHelper::cookieModifyHelper(const char *cookies, const size_t cookies_len, for (; idx < cookies_len && cookies[idx] != ';'; idx++) { ; } - // If we have not reached the end and there is a space after the - // semi-colon, advance one char - if (idx + 1 < cookies_len && std::isspace(cookies[idx + 1])) { - idx++; - } - // cookie value is found + // idx now points at the ';' ending this pair, or at cookies_len. size_t value_end_idx = idx; if (CookieHelper::COOKIE_OP_SET == cookie_op) { updated_cookies.append(cookies, value_start_idx); @@ -1045,10 +1044,15 @@ CookieHelper::cookieModifyHelper(const char *cookies, const size_t cookies_len, } if (CookieHelper::COOKIE_OP_DEL == cookie_op) { - // +1 to skip the semi-colon after the cookie_value updated_cookies.append(cookies, key_start_idx); + // Drop the deleted pair's trailing ';' and one following space, if present. if (value_end_idx < cookies_len) { - updated_cookies.append(cookies + value_end_idx + 1, cookies_len - value_end_idx - 1); + size_t tail_idx = value_end_idx + 1; + + if (tail_idx < cookies_len && std::isspace(cookies[tail_idx])) { + tail_idx++; + } + updated_cookies.append(cookies + tail_idx, cookies_len - tail_idx); } // if the cookie to delete is the last pair, // the semi-colon before this pair needs to be deleted diff --git a/plugins/header_rewrite/statement.h b/plugins/header_rewrite/statement.h index 0a354fc694c..82db6752af6 100644 --- a/plugins/header_rewrite/statement.h +++ b/plugins/header_rewrite/statement.h @@ -253,10 +253,14 @@ class Statement return false; } - // Scope-aware helpers for state variable access + // Scope-aware state accessors. A null handle (internal txns have no session) + // would trip a release assert in TSUserArg*, so treat it as unset. uint64_t _get_state_data(TSUserArgType scope, const Resources &res) const { + if (!_check_state_handle(scope, res)) { + return 0; + } if (scope == TS_USER_ARGS_SSN) { return reinterpret_cast(TSUserArgGet(res.state.ssnp, _ssn_slot)); } @@ -266,6 +270,9 @@ class Statement void _set_state_data(TSUserArgType scope, const Resources &res, uint64_t data) const { + if (!_check_state_handle(scope, res)) { + return; + } if (scope == TS_USER_ARGS_SSN) { TSUserArgSet(res.state.ssnp, _ssn_slot, reinterpret_cast(data)); } else { diff --git a/plugins/ja3_fingerprint/ja3_fingerprint.cc b/plugins/ja3_fingerprint/ja3_fingerprint.cc index 4f9f3028cf7..05ada1df758 100644 --- a/plugins/ja3_fingerprint/ja3_fingerprint.cc +++ b/plugins/ja3_fingerprint/ja3_fingerprint.cc @@ -153,14 +153,14 @@ custom_get_ja3(SSL *ssl) result.push_back(','); // Get elliptic curves - if (SSL_client_hello_get0_ext(ssl, 0x0a, &buf, &len) == 1) { + if (SSL_client_hello_get0_ext(ssl, 0x0a, &buf, &len) == 1 && len >= 2) { // Skip first 2 bytes since we already have length result.append(ja3::encode_word_buffer(buf + 2, len - 2)); } result.push_back(','); // Get elliptic curve point formats - if (SSL_client_hello_get0_ext(ssl, 0x0b, &buf, &len) == 1) { + if (SSL_client_hello_get0_ext(ssl, 0x0b, &buf, &len) == 1 && len >= 1) { // Skip first byte since we already have length result.append(ja3::encode_byte_buffer(buf + 1, len - 1)); } diff --git a/plugins/ja3_fingerprint/ja3_utils.cc b/plugins/ja3_fingerprint/ja3_utils.cc index b54e5066c83..5947e67870d 100644 --- a/plugins/ja3_fingerprint/ja3_utils.cc +++ b/plugins/ja3_fingerprint/ja3_utils.cc @@ -67,17 +67,21 @@ std::string encode_word_buffer(unsigned char const *buf, int const len) { std::string result; - auto it{buf}; - while ((it < (buf + len)) && ja3_should_ignore(from_big_endian(it[0], it[1]))) { + if (len < 2) { + return result; + } + auto it{buf}; + auto const end{buf + len}; + while ((it < end - 1) && ja3_should_ignore(from_big_endian(it[0], it[1]))) { it += 2; } - if (it < (buf + len)) { + if (it < end - 1) { // Benchmarks show that reserving buf.size() - 1 space in the string here // would have no impact on performance. Since the string may not even need // that much due to GREASE values present in the buffer, we don't do it. result.append(std::to_string(from_big_endian(it[0], it[1]))); it += 2; - for (; it < buf + len; it += 2) { + for (; it < end - 1; it += 2) { auto const value{from_big_endian(it[0], it[1])}; if (!ja3_should_ignore(value)) { result.push_back('-'); diff --git a/plugins/ja3_fingerprint/test_utils.cc b/plugins/ja3_fingerprint/test_utils.cc index 1e5de5e25d5..973d6d8dbcb 100644 --- a/plugins/ja3_fingerprint/test_utils.cc +++ b/plugins/ja3_fingerprint/test_utils.cc @@ -59,6 +59,12 @@ TEST_CASE("ja3 word buffer encoding") CHECK("" == got); } + SECTION("nullptr with len 1 - early return must not deref") + { + auto got{ja3::encode_word_buffer(nullptr, 1)}; + CHECK("" == got); + } + SECTION("1 value") { auto got{ja3::encode_word_buffer(buf, 2)}; @@ -70,6 +76,55 @@ TEST_CASE("ja3 word buffer encoding") auto got{ja3::encode_word_buffer(buf, 10)}; CHECK("5-8-256" == got); } + + SECTION("all GREASE - skip-loop consumes buffer, no emit") + { + unsigned char const grease_buf[]{0x0a, 0x0a, 0xda, 0xda}; + auto got{ja3::encode_word_buffer(grease_buf, 4)}; + CHECK("" == got); + } + + SECTION("trailing GREASE - last pair is GREASE, no trailing dash") + { + unsigned char const buf2[]{0x00, 0x05, 0x0a, 0x0a}; + auto got{ja3::encode_word_buffer(buf2, 4)}; + CHECK("5" == got); + } + + SECTION("odd length 1 - single trailing byte must not be read as a word") + { + unsigned char const odd_buf[]{0x42}; + auto got{ja3::encode_word_buffer(odd_buf, 1)}; + CHECK("" == got); + } + + SECTION("odd length 3 - last byte without pair must be ignored") + { + unsigned char const odd_buf[]{0x00, 0x05, 0x42}; + auto got{ja3::encode_word_buffer(odd_buf, 3)}; + CHECK("5" == got); + } + + SECTION("odd length 3 after GREASE - skip-loop must not read past end") + { + unsigned char const odd_buf[]{0x0a, 0x0a, 0x42}; + auto got{ja3::encode_word_buffer(odd_buf, 3)}; + CHECK("" == got); + } + + SECTION("odd length 5 - tail loop must reject trailing single byte") + { + unsigned char const odd_buf[]{0x00, 0x05, 0x00, 0x08, 0x42}; + auto got{ja3::encode_word_buffer(odd_buf, 5)}; + CHECK("5-8" == got); + } + + SECTION("supported_groups path: 3-byte extension body, 1-byte tail") + { + unsigned char const ext_body[]{0x00, 0x01, 0x02}; + auto got{ja3::encode_word_buffer(ext_body + 2, 1)}; + CHECK("" == got); + } } TEST_CASE("ja3 integer buffer encoding") diff --git a/plugins/lua/ts_lua.cc b/plugins/lua/ts_lua.cc index eeaa4f03a5b..f4af729070b 100644 --- a/plugins/lua/ts_lua.cc +++ b/plugins/lua/ts_lua.cc @@ -519,7 +519,7 @@ ts_lua_remap_plugin_init(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) instance_conf = (ts_lua_instance_conf *)ih; main_ctx = static_cast(pthread_getspecific(lua_state_key)); - if (main_ctx == nullptr) { + if (main_ctx == nullptr || static_cast(main_ctx - ts_lua_main_ctx_array) >= instance_conf->states) { req_id = __sync_fetch_and_add(&ts_lua_http_next_id, 1); main_ctx = &ts_lua_main_ctx_array[req_id % instance_conf->states]; pthread_setspecific(lua_state_key, main_ctx); @@ -529,9 +529,10 @@ ts_lua_remap_plugin_init(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) http_ctx = ts_lua_create_http_ctx(main_ctx, instance_conf); - http_ctx->txnp = rh; - http_ctx->has_hook = 0; - http_ctx->rri = rri; + http_ctx->txnp = rh; + http_ctx->has_hook = 0; + http_ctx->from_remap = (rri != nullptr) ? 1 : 0; + http_ctx->rri = rri; if (rri != nullptr) { http_ctx->client_request_bufp = rri->requestBufp; http_ctx->client_request_hdrp = rri->requestHdrp; @@ -565,6 +566,11 @@ ts_lua_remap_plugin_init(void *ih, TSHttpTxn rh, TSRemapRequestInfo *rri) lua_pop(L, 1); + // rri lives on the caller's stack; clear it so post-remap hooks + // see ts.remap.* return nil per the documented do_remap-only + // context. Destructors gate handle release on from_remap, not rri. + http_ctx->rri = nullptr; + if (http_ctx->has_hook) { Dbg(dbg_ctl, "[%s] has txn hook -> adding txn close hook handler to release resources", __FUNCTION__); TSHttpTxnHookAdd(rh, TS_HTTP_TXN_CLOSE_HOOK, contp); @@ -615,7 +621,7 @@ vconnHookHandler(TSCont contp, TSEvent event, void *edata) ts_lua_instance_conf *conf = (ts_lua_instance_conf *)TSContDataGet(contp); main_ctx = static_cast(pthread_getspecific(lua_g_state_key)); - if (main_ctx == NULL) { + if (main_ctx == NULL || static_cast(main_ctx - ts_lua_g_main_ctx_array) >= conf->states) { req_id = __sync_fetch_and_add(&ts_lua_g_http_next_id, 1); Dbg(dbg_ctl, "[%s] req_id for vconn handler: %" PRId64, __FUNCTION__, req_id); main_ctx = &ts_lua_g_main_ctx_array[req_id % conf->states]; @@ -690,7 +696,7 @@ globalHookHandler(TSCont contp, TSEvent event ATS_UNUSED, void *edata) ts_lua_instance_conf *conf = (ts_lua_instance_conf *)TSContDataGet(contp); main_ctx = static_cast(pthread_getspecific(lua_g_state_key)); - if (main_ctx == nullptr) { + if (main_ctx == nullptr || static_cast(main_ctx - ts_lua_g_main_ctx_array) >= conf->states) { req_id = __sync_fetch_and_add(&ts_lua_g_http_next_id, 1); Dbg(dbg_ctl, "[%s] req_id: %" PRId64, __FUNCTION__, req_id); main_ctx = &ts_lua_g_main_ctx_array[req_id % conf->states]; diff --git a/plugins/lua/ts_lua_common.h b/plugins/lua/ts_lua_common.h index 4f203fe0c17..6d032c2b78e 100644 --- a/plugins/lua/ts_lua_common.h +++ b/plugins/lua/ts_lua_common.h @@ -148,6 +148,7 @@ typedef struct { ts_lua_instance_conf *instance_conf; int has_hook; + int from_remap; TSRemapRequestInfo *rri; diff --git a/plugins/lua/ts_lua_fetch.cc b/plugins/lua/ts_lua_fetch.cc index 45a80557c55..df6d4953b7f 100644 --- a/plugins/lua/ts_lua_fetch.cc +++ b/plugins/lua/ts_lua_fetch.cc @@ -202,16 +202,16 @@ ts_lua_fetch_multi(lua_State *L) static int ts_lua_fetch_one_item(lua_State *L, const char *url, size_t url_len, ts_lua_fetch_info *fi) { - TSCont contp; - int tb, flags, host_len, n; - int cl, ht, ua; - const char *method, *key, *value, *body, *opt; - const char *addr, *ptr, *host; - size_t method_len, key_len, value_len, body_len; - size_t addr_len, opt_len, i, left; - char c; - struct sockaddr clientaddr; - char buf[32]; + TSCont contp; + int tb, flags, host_len, n; + int cl, ht, ua; + const char *method, *key, *value, *body, *opt; + const char *addr, *ptr, *host; + size_t method_len, key_len, value_len, body_len; + size_t addr_len, opt_len, i, left; + char c; + struct sockaddr_storage clientaddr; + char buf[32]; tb = lua_istable(L, -1); @@ -259,15 +259,17 @@ ts_lua_fetch_one_item(lua_State *L, const char *url, size_t url_len, ts_lua_fetc if (lua_isstring(L, -1)) { addr = luaL_checklstring(L, -1, &addr_len); - if (TS_ERROR == TSIpStringToAddr(addr, addr_len, &clientaddr)) { + if (TS_ERROR == TSIpStringToAddr(addr, addr_len, reinterpret_cast(&clientaddr))) { TSError("[ts_lua][%s] Client ip parse failed! Using default.", TS_LUA_DEBUG_TAG); - if (TS_ERROR == TSIpStringToAddr(TS_LUA_FETCH_CLIENT_ADDRPORT, TS_LUA_FETCH_CLIENT_ADDRPORT_LEN, &clientaddr)) { + if (TS_ERROR == TSIpStringToAddr(TS_LUA_FETCH_CLIENT_ADDRPORT, TS_LUA_FETCH_CLIENT_ADDRPORT_LEN, + reinterpret_cast(&clientaddr))) { TSError("[ts_lua][%s] Default client ip parse failed!", TS_LUA_DEBUG_TAG); return 0; } } } else { - if (TS_ERROR == TSIpStringToAddr(TS_LUA_FETCH_CLIENT_ADDRPORT, TS_LUA_FETCH_CLIENT_ADDRPORT_LEN, &clientaddr)) { + if (TS_ERROR == TSIpStringToAddr(TS_LUA_FETCH_CLIENT_ADDRPORT, TS_LUA_FETCH_CLIENT_ADDRPORT_LEN, + reinterpret_cast(&clientaddr))) { TSError("[ts_lua][%s] Default client ip parse failed!", TS_LUA_DEBUG_TAG); return 0; } @@ -275,7 +277,8 @@ ts_lua_fetch_one_item(lua_State *L, const char *url, size_t url_len, ts_lua_fetc lua_pop(L, 1); } else { - if (TS_ERROR == TSIpStringToAddr(TS_LUA_FETCH_CLIENT_ADDRPORT, TS_LUA_FETCH_CLIENT_ADDRPORT_LEN, &clientaddr)) { + if (TS_ERROR == TSIpStringToAddr(TS_LUA_FETCH_CLIENT_ADDRPORT, TS_LUA_FETCH_CLIENT_ADDRPORT_LEN, + reinterpret_cast(&clientaddr))) { TSError("[ts_lua][%s] Default client ip parse failed!", TS_LUA_DEBUG_TAG); return 0; } @@ -315,7 +318,7 @@ ts_lua_fetch_one_item(lua_State *L, const char *url, size_t url_len, ts_lua_fetc TSContDataSet(contp, fi); fi->contp = contp; - fi->fch = TSFetchCreate(contp, method, url, "HTTP/1.1", &clientaddr, flags); + fi->fch = TSFetchCreate(contp, method, url, "HTTP/1.1", reinterpret_cast(&clientaddr), flags); /* header */ cl = ht = ua = 0; diff --git a/plugins/lua/ts_lua_server_request.cc b/plugins/lua/ts_lua_server_request.cc index 913b7fba478..03de506a9ac 100644 --- a/plugins/lua/ts_lua_server_request.cc +++ b/plugins/lua/ts_lua_server_request.cc @@ -1060,6 +1060,8 @@ ts_lua_server_request_get_method(lua_State *L) GET_HTTP_CONTEXT(http_ctx, L); + TS_LUA_CHECK_SERVER_REQUEST_HDR(http_ctx); + method = TSHttpHdrMethodGet(http_ctx->server_request_bufp, http_ctx->server_request_hdrp, &method_len); if (method && method_len) { @@ -1081,6 +1083,8 @@ ts_lua_server_request_set_method(lua_State *L) GET_HTTP_CONTEXT(http_ctx, L); + TS_LUA_CHECK_SERVER_REQUEST_HDR(http_ctx); + method = luaL_checklstring(L, 1, &method_len); if (method) { diff --git a/plugins/lua/ts_lua_transform.cc b/plugins/lua/ts_lua_transform.cc index 06311e78239..7d30877f4ad 100644 --- a/plugins/lua/ts_lua_transform.cc +++ b/plugins/lua/ts_lua_transform.cc @@ -31,6 +31,11 @@ ts_lua_client_entry(TSCont contp, TSEvent ev, void *edata) event = (int)ev; transform_ctx = (ts_lua_http_transform_ctx *)TSContDataGet(contp); + if (TSVConnClosedGet(contp)) { + ts_lua_destroy_http_transform_ctx(transform_ctx); + return 0; + } + n = 0; switch (event) { diff --git a/plugins/lua/ts_lua_util.cc b/plugins/lua/ts_lua_util.cc index 10baf236b79..14e9b8573cc 100644 --- a/plugins/lua/ts_lua_util.cc +++ b/plugins/lua/ts_lua_util.cc @@ -64,7 +64,7 @@ ts_lua_update_server_response_hdrp(ts_lua_http_ctx *http_ctx) void ts_lua_clear_http_ctx(ts_lua_http_ctx *http_ctx) { - if (http_ctx->rri == nullptr) { + if (!http_ctx->from_remap) { if (http_ctx->client_request_url != nullptr) { TSHandleMLocRelease(http_ctx->client_request_bufp, http_ctx->client_request_hdrp, http_ctx->client_request_url); http_ctx->client_request_url = nullptr; @@ -836,7 +836,7 @@ ts_lua_destroy_http_ctx(ts_lua_http_ctx *http_ctx) ci = &http_ctx->cinfo; - if (http_ctx->rri == nullptr) { + if (!http_ctx->from_remap) { if (http_ctx->client_request_url) { TSHandleMLocRelease(http_ctx->client_request_bufp, http_ctx->client_request_hdrp, http_ctx->client_request_url); } diff --git a/plugins/multiplexer/CMakeLists.txt b/plugins/multiplexer/CMakeLists.txt index e1e7d1c7307..6b01e36ddd4 100644 --- a/plugins/multiplexer/CMakeLists.txt +++ b/plugins/multiplexer/CMakeLists.txt @@ -29,3 +29,10 @@ add_atsplugin( target_compile_definitions(multiplexer PRIVATE PLUGIN_TAG="multiplexer") verify_remap_plugin(multiplexer) + +if(BUILD_TESTING) + add_executable(test_multiplexer_chunk_decoder unit_tests/test_chunk_decoder.cc chunk-decoder.cc) + target_include_directories(test_multiplexer_chunk_decoder PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}") + target_link_libraries(test_multiplexer_chunk_decoder PRIVATE Catch2::Catch2WithMain ts::tsutil) + add_catch2_test(NAME test_multiplexer_chunk_decoder COMMAND test_multiplexer_chunk_decoder) +endif() diff --git a/plugins/multiplexer/chunk-decoder.cc b/plugins/multiplexer/chunk-decoder.cc index 2b2cb43514a..995dc7058f1 100644 --- a/plugins/multiplexer/chunk-decoder.cc +++ b/plugins/multiplexer/chunk-decoder.cc @@ -22,59 +22,84 @@ */ #include #include +#include #include "chunk-decoder.h" +namespace +{ +int +parse_hex_digit(const char a) +{ + if (a >= '0' && a <= '9') { + return a - '0'; + } + if (a >= 'A' && a <= 'F') { + return a - 'A' + 10; + } + if (a >= 'a' && a <= 'f') { + return a - 'a' + 10; + } + return -1; +} +} // namespace + void ChunkDecoder::parseSizeCharacter(const char a) { assert(state_ == State::kSize); - if (a >= '0' && a <= '9') { - size_ = (size_ << 4) | (a - '0'); - } else if (a >= 'A' && a <= 'F') { - size_ = (size_ << 4) | (a - 'A' + 10); - } else if (a >= 'a' && a <= 'f') { - size_ = (size_ << 4) | (a - 'a' + 10); + const int digit = parse_hex_digit(a); + if (digit >= 0) { + constexpr int64_t max_chunk_size = std::numeric_limits::max(); + if (size_ > (max_chunk_size - digit) / 16) { + state_ = State::kInvalid; + size_ = 0; + return; + } + size_ = size_ * 16 + digit; } else if (a == '\r') { state_ = size_ == 0 ? State::kEndN : State::kDataN; } else { - assert(false); // invalid input + state_ = State::kInvalid; + return; } + return; } -int +int64_t ChunkDecoder::parseSize(const char *p, const int64_t s) { assert(p != nullptr); assert(s > 0); - int length = 0; - while (state_ != State::kData && *p != '\0' && length < s) { + int64_t length = 0; + while (state_ != State::kData && state_ != State::kInvalid && length < s) { assert(state_ < State::kUpperBound); // VALID RANGE switch (state_) { case State::kData: - case State::kInvalid: case State::kEnd: case State::kUpperBound: assert(false); break; + case State::kInvalid: + break; + case State::kDataN: - assert(*p == '\n'); state_ = (*p == '\n') ? State::kData : State::kInvalid; break; case State::kEndN: - assert(*p == '\n'); state_ = (*p == '\n') ? State::kEnd : State::kInvalid; - return length; + if (state_ == State::kEnd) { + return length; + } + break; case State::kSizeR: - assert(*p == '\r'); state_ = (*p == '\r') ? State::kSizeN : State::kInvalid; break; case State::kSizeN: - assert(*p == '\n'); state_ = (*p == '\n') ? State::kSize : State::kInvalid; break; @@ -84,7 +109,6 @@ ChunkDecoder::parseSize(const char *p, const int64_t s) } ++length; ++p; - assert(state_ != State::kInvalid); } return length; } @@ -96,17 +120,21 @@ ChunkDecoder::isSizeState() const state_ == State::kSizeR; } -int +int64_t ChunkDecoder::decode(const TSIOBufferReader &r) { assert(r != nullptr); + if (state_ == State::kInvalid) { + return -1; + } + if (state_ == State::kEnd) { return 0; } { - const int l = TSIOBufferReaderAvail(r); + const int64_t l = TSIOBufferReaderAvail(r); if (l == 0) { return 0; } else if (l < size_) { @@ -120,12 +148,19 @@ ChunkDecoder::decode(const TSIOBufferReader &r) // Trying to parse a size. if (isSizeState()) { - while (block != nullptr && size_ == 0) { + while (block != nullptr && size_ == 0 && !isInvalid()) { const char *p = TSIOBufferBlockReadStart(block, r, &size); + if (size == 0) { + block = TSIOBufferBlockNext(block); + continue; + } assert(p != nullptr); - const int i = parseSize(p, size); - size -= i; + const int64_t i = parseSize(p, size); + size -= i; TSIOBufferReaderConsume(r, i); + if (isInvalid()) { + return -1; + } if (state_ == State::kEnd) { assert(size_ == 0); return 0; @@ -137,7 +172,7 @@ ChunkDecoder::decode(const TSIOBufferReader &r) } } - int length = 0; + int64_t length = 0; while (block != nullptr && state_ == State::kData) { assert(size_ > 0); diff --git a/plugins/multiplexer/chunk-decoder.h b/plugins/multiplexer/chunk-decoder.h index 196aaed6c24..5b0cf13f44f 100644 --- a/plugins/multiplexer/chunk-decoder.h +++ b/plugins/multiplexer/chunk-decoder.h @@ -23,8 +23,8 @@ #pragma once -#include #include +#include /** Class to handle state for decoding chunked data. */ @@ -52,10 +52,16 @@ class ChunkDecoder /// Default Constructor. Construct to empty state of expected size 0. ChunkDecoder() {} - void parseSizeCharacter(const char); - int parseSize(const char *, const int64_t); - int decode(const TSIOBufferReader &); - bool isSizeState() const; + void parseSizeCharacter(const char); + int64_t parseSize(const char *, const int64_t); + int64_t decode(const TSIOBufferReader &); + bool isSizeState() const; + + inline bool + isInvalid() const + { + return state_ == State::kInvalid; + } inline bool isEnd() const diff --git a/plugins/multiplexer/fetcher.h b/plugins/multiplexer/fetcher.h index bc2a27a9d1f..612b1f2d2e9 100644 --- a/plugins/multiplexer/fetcher.h +++ b/plugins/multiplexer/fetcher.h @@ -137,10 +137,12 @@ template struct HttpTransaction { } static void - close(Self *const s) + close(TSCont c, Self *const s) { + assert(c != NULL); assert(s != NULL); TSVConnShutdown(s->vconnection_, 1, 0); + TSContDataSet(c, nullptr); delete s; } @@ -174,8 +176,7 @@ template struct HttpTransaction { Dbg(dbg_ctl, "HttpTransaction: ERROR"); self->t_.error(); self->abort(); - close(self); - TSContDataSet(c, nullptr); + close(c, self); break; case TS_EVENT_VCONN_EOS: Dbg(dbg_ctl, "HttpTransaction: EOS"); @@ -207,33 +208,46 @@ template struct HttpTransaction { available = TSIOBufferReaderAvail(self->in_->reader); } if (!self->parsingHeaders_) { + bool closed = false; if (self->chunkDecoder_ != NULL) { available = self->chunkDecoder_->decode(self->in_->reader); - if (available == 0) { + if (available < 0) { + self->t_.error(); + self->abort(); + close(c, self); + closed = true; + } else if (available == 0) { self->t_.data(self->in_->reader, available); } - while (available > 0) { + while (!closed && available > 0) { self->t_.data(self->in_->reader, available); TSIOBufferReaderConsume(self->in_->reader, available); available = self->chunkDecoder_->decode(self->in_->reader); + if (available < 0) { + self->t_.error(); + self->abort(); + close(c, self); + closed = true; + } } } else { self->t_.data(self->in_->reader, available); TSIOBufferReaderConsume(self->in_->reader, available); } + if (closed) { + break; + } } } if (e == TS_EVENT_VCONN_READ_COMPLETE || e == TS_EVENT_VCONN_EOS) { self->t_.done(); - close(self); - TSContDataSet(c, nullptr); + close(c, self); } else if (self->chunkDecoder_ != NULL && self->chunkDecoder_->isEnd()) { assert(self->parsingHeaders_ == false); assert(isChunkEncoding(self->parser_.buffer_, self->parser_.location_)); self->abort(); self->t_.done(); - close(self); - TSContDataSet(c, nullptr); + close(c, self); } else { TSVIOReenable(self->in_->vio); } @@ -260,8 +274,7 @@ template struct HttpTransaction { Dbg(dbg_ctl, "HttpTransaction: Timeout"); self->t_.timeout(); self->abort(); - close(self); - TSContDataSet(c, nullptr); + close(c, self); break; default: diff --git a/plugins/multiplexer/unit_tests/test_chunk_decoder.cc b/plugins/multiplexer/unit_tests/test_chunk_decoder.cc new file mode 100644 index 00000000000..04ff18b81f5 --- /dev/null +++ b/plugins/multiplexer/unit_tests/test_chunk_decoder.cc @@ -0,0 +1,155 @@ +/** @file + + Unit tests for the multiplexer chunk decoder. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "chunk-decoder.h" + +#include + +namespace +{ +struct FakeBlock { + const char *data = nullptr; + int64_t size = 0; + FakeBlock *next = nullptr; +}; + +FakeBlock *first_block = nullptr; +int64_t reader_avail = 0; +int64_t consumed = 0; + +TSIOBufferBlock +as_block(FakeBlock *block) +{ + return reinterpret_cast(block); +} + +FakeBlock * +as_fake_block(TSIOBufferBlock block) +{ + return reinterpret_cast(block); +} +} // namespace + +TSIOBufferBlock +TSIOBufferBlockNext(TSIOBufferBlock blockp) +{ + FakeBlock *block = as_fake_block(blockp); + + return block == nullptr ? nullptr : as_block(block->next); +} + +const char * +TSIOBufferBlockReadStart(TSIOBufferBlock blockp, TSIOBufferReader /* readerp ATS_UNUSED */, int64_t *avail) +{ + FakeBlock *block = as_fake_block(blockp); + + *avail = block == nullptr ? 0 : block->size; + return block == nullptr ? nullptr : block->data; +} + +TSIOBufferBlock +TSIOBufferReaderStart(TSIOBufferReader /* readerp ATS_UNUSED */) +{ + return as_block(first_block); +} + +void +TSIOBufferReaderConsume(TSIOBufferReader /* readerp ATS_UNUSED */, int64_t nbytes) +{ + consumed += nbytes; + reader_avail -= nbytes; +} + +int64_t +TSIOBufferReaderAvail(TSIOBufferReader /* readerp ATS_UNUSED */) +{ + return reader_avail; +} + +TEST_CASE("ChunkDecoder accepts representable chunk sizes", "[multiplexer][chunk-decoder]") +{ + ChunkDecoder decoder; + + CHECK(decoder.parseSize("7fffffffffffffff\r\n", 18) == 18); + CHECK_FALSE(decoder.isInvalid()); + CHECK_FALSE(decoder.isSizeState()); + CHECK_FALSE(decoder.isEnd()); +} + +TEST_CASE("ChunkDecoder rejects chunk sizes that exceed int64_t", "[multiplexer][chunk-decoder]") +{ + SECTION("one larger than INT64_MAX") + { + ChunkDecoder decoder; + + CHECK(decoder.parseSize("8000000000000000\r\n", 18) == 16); + CHECK(decoder.isInvalid()); + } + + SECTION("17 hex digits") + { + ChunkDecoder decoder; + + CHECK(decoder.parseSize("10000000000000000\r\n", 19) == 17); + CHECK(decoder.isInvalid()); + } +} + +TEST_CASE("ChunkDecoder rejects malformed chunk-size characters", "[multiplexer][chunk-decoder]") +{ + SECTION("unexpected alphabetic character") + { + ChunkDecoder decoder; + + CHECK(decoder.parseSize("1z\r\n", 4) == 2); + CHECK(decoder.isInvalid()); + } + + SECTION("embedded NUL byte") + { + ChunkDecoder decoder; + const char encoded[] = {'1', '\0', '\r', '\n'}; + + CHECK(decoder.parseSize(encoded, sizeof(encoded)) == 2); + CHECK(decoder.isInvalid()); + } +} + +TEST_CASE("ChunkDecoder skips empty IOBuffer blocks while parsing chunk sizes", "[multiplexer][chunk-decoder]") +{ + ChunkDecoder decoder; + FakeBlock data_block{"1\r\nx", 4, nullptr}; + FakeBlock empty_block{nullptr, 0, &data_block}; + + first_block = &empty_block; + reader_avail = data_block.size; + consumed = 0; + + CHECK(decoder.decode(reinterpret_cast(&empty_block)) == 1); + CHECK(consumed == 3); + CHECK_FALSE(decoder.isInvalid()); + + first_block = nullptr; + reader_avail = 0; + consumed = 0; +} diff --git a/plugins/prefetch/CMakeLists.txt b/plugins/prefetch/CMakeLists.txt index 28637b6f176..9f1d22f51f7 100644 --- a/plugins/prefetch/CMakeLists.txt +++ b/plugins/prefetch/CMakeLists.txt @@ -28,6 +28,7 @@ add_atsplugin( fetch_policy_lru.cc headers.cc pattern.cc + path.cc plugin.cc ) diff --git a/plugins/prefetch/fetch.cc b/plugins/prefetch/fetch.cc index b0d5c7615c1..99e72b02b4f 100644 --- a/plugins/prefetch/fetch.cc +++ b/plugins/prefetch/fetch.cc @@ -415,11 +415,11 @@ BgFetch::~BgFetch() bool BgFetch::schedule(BgFetchState *state, const PrefetchConfig &config, bool askPermission, TSMBuffer requestBuffer, TSMLoc requestHeaderLoc, TSHttpTxn txnp, const char *path, size_t pathLen, const String &cachekey, - bool removeQuery) + bool removeQuery, const char *query, size_t queryLen) { bool ret = false; BgFetch *fetch = new BgFetch(state, config, askPermission); - if (fetch->init(requestBuffer, requestHeaderLoc, txnp, path, pathLen, cachekey, removeQuery)) { + if (fetch->init(requestBuffer, requestHeaderLoc, txnp, path, pathLen, cachekey, removeQuery, query, queryLen)) { fetch->schedule(); ret = true; } else { @@ -459,7 +459,7 @@ BgFetch::addBytes(int64_t b) */ bool BgFetch::init(TSMBuffer reqBuffer, TSMLoc reqHdrLoc, TSHttpTxn txnp, const char *fetchPath, size_t fetchPathLen, - const String &cachekey, bool removeQuery) + const String &cachekey, bool removeQuery, const char *fetchQuery, size_t fetchQueryLen) { TSAssert(TS_NULL_MLOC == _headerLoc); TSAssert(TS_NULL_MLOC == _urlLoc); @@ -523,6 +523,15 @@ BgFetch::init(TSMBuffer reqBuffer, TSMLoc reqHdrLoc, TSHttpTxn txnp, const char } } + /* Replace the query string when a derived prefetch path supplied one. */ + if (nullptr != fetchQuery) { + if (TS_SUCCESS == TSUrlHttpQuerySet(_mbuf, _urlLoc, fetchQuery, fetchQueryLen)) { + PrefetchDebug("setting URL query to %.*s", (int)fetchQueryLen, fetchQuery); + } else { + PrefetchError("failed to set URL query to %.*s", (int)fetchQueryLen, fetchQuery); + } + } + /* Now set or remove the prefetch API header */ const String &header = _config.getApiHeader(); if (_config.isFront()) { diff --git a/plugins/prefetch/fetch.h b/plugins/prefetch/fetch.h index 66ae24d561a..ff42a639895 100644 --- a/plugins/prefetch/fetch.h +++ b/plugins/prefetch/fetch.h @@ -168,13 +168,13 @@ class BgFetch public: static bool schedule(BgFetchState *state, const PrefetchConfig &config, bool askPermission, TSMBuffer requestBuffer, TSMLoc requestHeaderLoc, TSHttpTxn txnp, const char *path, size_t pathLen, const String &cachekey, - bool removeQuery = false); + bool removeQuery = false, const char *query = nullptr, size_t queryLen = 0); private: BgFetch(BgFetchState *state, const PrefetchConfig &config, bool lock); ~BgFetch(); bool init(TSMBuffer requestBuffer, TSMLoc requestHeaderLoc, TSHttpTxn txnp, const char *fetchPath, size_t fetchPathLen, - const String &cacheKey, bool removeQuery = false); + const String &cacheKey, bool removeQuery = false, const char *fetchQuery = nullptr, size_t fetchQueryLen = 0); void schedule(); static int handler(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */); bool saveIp(TSHttpTxn txnp); diff --git a/plugins/prefetch/path.cc b/plugins/prefetch/path.cc new file mode 100644 index 00000000000..a89e16ce522 --- /dev/null +++ b/plugins/prefetch/path.cc @@ -0,0 +1,190 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/** + * @file path.cc + * @brief Prefetch path construction helpers. + */ + +#include "path.h" + +#include + +namespace +{ +String +getDirectoryPrefix(const String &path) +{ + const String::size_type lastSlash = path.find_last_of('/'); + + if (String::npos == lastSlash) { + return {}; + } + return path.substr(0, lastSlash + 1); +} + +bool +normalizePath(StringView path, String &normalized) +{ + const bool absolute = !path.empty() && '/' == path.front(); + StringList segments; + + normalized.clear(); + + for (String::size_type offset = 0; offset <= path.size();) { + const String::size_type nextSeparator = path.find('/', offset); + const StringView segment = path.substr(offset, nextSeparator - offset); + + if (segment.empty() || "." == segment) { + // Skip empty and current-directory segments. + } else if (".." == segment) { + if (segments.empty()) { + return false; + } + segments.pop_back(); + } else { + segments.emplace_back(segment); + } + + if (String::npos == nextSeparator) { + break; + } + offset = nextSeparator + 1; + } + + if (absolute) { + normalized.push_back('/'); + } + + for (const auto &segment : segments) { + if (!normalized.empty() && '/' != normalized.back()) { + normalized.push_back('/'); + } + normalized.append(segment); + } + + if (!path.empty() && '/' == path.back() && (normalized.empty() || '/' != normalized.back())) { + normalized.push_back('/'); + } + + return true; +} + +int +hexDigitValue(char c) +{ + if ('0' <= c && c <= '9') { + return c - '0'; + } + c = static_cast(std::tolower(static_cast(c))); + if ('a' <= c && c <= 'f') { + return c - 'a' + 10; + } + return -1; +} + +String +decodePathSeparatorsAndDots(StringView path) +{ + String decoded; + + decoded.reserve(path.size()); + for (String::size_type i = 0; i < path.size(); ++i) { + if ('\\' == path[i]) { + decoded.push_back('/'); + } else if ('%' == path[i] && i + 2 < path.size()) { + const int high = hexDigitValue(path[i + 1]); + const int low = hexDigitValue(path[i + 2]); + if (0 <= high && 0 <= low) { + const char c = static_cast((high << 4) | low); + if ('.' == c || '/' == c || '\\' == c) { + decoded.push_back('\\' == c ? '/' : c); + } else { + decoded.append(path.substr(i, 3)); + } + i += 2; + } else { + decoded.push_back(path[i]); + } + } else { + decoded.push_back(path[i]); + } + } + + return decoded; +} + +bool +isUnderPrefix(const String &path, const String &prefix) +{ + return prefix.empty() || path.rfind(prefix, 0) == 0; +} + +bool +startsWithSeparator(StringView path) +{ + return !path.empty() && ('/' == path.front() || '\\' == path.front()); +} +} // namespace + +bool +makeSafeRelativeFetchPath(const String ¤tPath, const String &relativePath, SafeRelativeFetchPath &fetchPath) +{ + fetchPath = {}; + + if (String::npos != relativePath.find('#')) { + return false; + } + + const String::size_type queryStart = relativePath.find('?'); + const StringView pathPart = + String::npos == queryStart ? StringView{relativePath} : StringView{relativePath}.substr(0, queryStart); + const String decodedPathPart = decodePathSeparatorsAndDots(pathPart); + + if (pathPart.empty() || startsWithSeparator(pathPart) || startsWithSeparator(decodedPathPart)) { + return false; + } + + const String basePrefix = getDirectoryPrefix(currentPath); + const String candidatePath{basePrefix + String{pathPart}}; + + String normalizedBasePrefix; + String normalizedCandidatePath; + if (!normalizePath(basePrefix, normalizedBasePrefix) || !normalizePath(candidatePath, normalizedCandidatePath)) { + return false; + } + + String validationBasePrefix; + String validationCandidatePath; + if (!normalizePath(decodePathSeparatorsAndDots(basePrefix), validationBasePrefix) || + !normalizePath(decodePathSeparatorsAndDots(candidatePath), validationCandidatePath)) { + return false; + } + + if (!isUnderPrefix(normalizedCandidatePath, normalizedBasePrefix) || + !isUnderPrefix(validationCandidatePath, validationBasePrefix)) { + return false; + } + + fetchPath.path = normalizedCandidatePath; + if (String::npos != queryStart) { + fetchPath.hasQuery = true; + fetchPath.query = relativePath.substr(queryStart + 1); + } + return true; +} diff --git a/plugins/prefetch/path.h b/plugins/prefetch/path.h new file mode 100644 index 00000000000..b154e81e259 --- /dev/null +++ b/plugins/prefetch/path.h @@ -0,0 +1,63 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/** + * @file path.h + * @brief Prefetch path construction helpers. + */ + +#pragma once + +#include "common.h" + +/** A normalized path and optional query for a derived prefetch request. */ +struct SafeRelativeFetchPath { + String path; ///< Normalized path component without a query or fragment. + String query; ///< Query string without the leading @c ?, meaningful when @c hasQuery is @c true. + bool hasQuery{false}; ///< Whether @c query should replace the cloned request query. +}; + +/** Build a normalized prefetch target from a relative path. + * + * The prefetch plugin accepts query and CMCD metadata as instructions to fetch + * a sibling object relative to the triggering object. Without this boundary + * check, an attacker can turn a request for an allowed object into an implicit + * background fetch for a different origin resource by using traversal segments + * such as @c ../ in that metadata. This is not a general rejection of HTTP URL + * paths containing dot segments; it enforces the plugin's relative prefetch + * contract before scheduling and caching a derived request. + * + * The path component of @a relativePath is resolved against the directory that + * contains @a currentPath. Query suffixes are returned separately so callers + * can set the URL query component instead of embedding @c ? in the path. + * Fragment suffixes are rejected because they are not sent in HTTP requests. + * The resulting path must remain under that original directory after + * dot-segment cleanup and after decoding encoded dot or separator octets used + * for validation. + * + * @param[in] currentPath The path of the request that is triggering the + * prefetch. + * @param[in] relativePath The query or CMCD supplied path to fetch relative to + * @a currentPath. + * @param[out] fetchPath The normalized fetch path and optional query when + * construction succeeds. + * @return @c true if @a fetchPath was populated with a safe target, @c false + * if @a relativePath is empty, absolute, contains a fragment, or would escape + * the current directory. + */ +bool makeSafeRelativeFetchPath(const String ¤tPath, const String &relativePath, SafeRelativeFetchPath &fetchPath); diff --git a/plugins/prefetch/plugin.cc b/plugins/prefetch/plugin.cc index a472bea0d50..37c8d60a176 100644 --- a/plugins/prefetch/plugin.cc +++ b/plugins/prefetch/plugin.cc @@ -32,6 +32,7 @@ #include "fetch_policy.h" #include "headers.h" #include "evaluate.h" +#include "path.h" static const char * getEventName(TSEvent event) @@ -636,15 +637,18 @@ contHandleFetch(const TSCont contp, TSEvent event, void *edata) PrefetchDebug("Current path: '%s'", currentPath.c_str()); PrefetchDebug("Parsed cmcd nor relpath: '%s'", relpath.c_str()); - const String::size_type lsi = currentPath.find_last_of("/"); - const String nextPath = currentPath.substr(0, lsi + 1) + relpath; - - PrefetchDebug("Next cmcd nor path: '%s'", nextPath.c_str()); + SafeRelativeFetchPath nextPath; + if (!makeSafeRelativeFetchPath(currentPath, relpath, nextPath)) { + PrefetchDebug("skipping unsafe cmcd nor path: '%s'", relpath.c_str()); + } else { + PrefetchDebug("Next cmcd nor path: '%s'", nextPath.path.c_str()); - constexpr bool askPermission = false; - constexpr bool removeQuery = true; - BgFetch::schedule(state, config, askPermission, reqBuffer, reqHdrLoc, txnp, nextPath.c_str(), nextPath.length(), - data->_cachekey, removeQuery); + constexpr bool askPermission = false; + constexpr bool removeQuery = true; + BgFetch::schedule(state, config, askPermission, reqBuffer, reqHdrLoc, txnp, nextPath.path.c_str(), + nextPath.path.length(), data->_cachekey, removeQuery, + nextPath.hasQuery ? nextPath.query.c_str() : nullptr, nextPath.query.length()); + } } } @@ -681,9 +685,9 @@ contHandleFetch(const TSCont contp, TSEvent event, void *edata) /* Trigger all necessary background fetches based on the query string(s) */ PrefetchDebug("currentQuery: %s", currentQuery.c_str()); - const size_t lastSlashIndex = currentPath.find_last_of("/"); - const size_t keyLen = config.getQueryKeyName().size(); - unsigned done = 1; + const String &queryKeyName = config.getQueryKeyName(); + const size_t keyLen = queryKeyName.size(); + unsigned done = 1; std::istringstream cStringStream(currentQuery); String param; @@ -691,15 +695,24 @@ contHandleFetch(const TSCont contp, TSEvent event, void *edata) if (!isQueryKeyParam(param, config.getQueryKeyName())) { continue; } + String nextFile = param.substr(keyLen + 1); // +1 for the '=' + if (nextFile.empty()) { + PrefetchDebug("skipping empty query prefetch path"); + continue; + } if (config.getFetchCount() < done++) { break; } - String nextFile = param.substr(keyLen + 1); // +1 for the '=' - String nextPath = currentPath.substr(0, lastSlashIndex + 1) + nextFile; + SafeRelativeFetchPath nextPath; + if (!makeSafeRelativeFetchPath(currentPath, nextFile, nextPath)) { + PrefetchDebug("skipping unsafe query prefetch path: '%s'", nextFile.c_str()); + continue; + } - PrefetchDebug("nextPath %s, cacheKey %s", nextPath.c_str(), data->_cachekey.c_str()); - BgFetch::schedule(state, config, /* askPermission */ false, reqBuffer, reqHdrLoc, txnp, nextPath.c_str(), - nextPath.length(), data->_cachekey); + PrefetchDebug("nextPath %s, cacheKey %s", nextPath.path.c_str(), data->_cachekey.c_str()); + BgFetch::schedule(state, config, /* askPermission */ false, reqBuffer, reqHdrLoc, txnp, nextPath.path.c_str(), + nextPath.path.length(), data->_cachekey, /* removeQuery */ false, + nextPath.hasQuery ? nextPath.query.c_str() : nullptr, nextPath.query.length()); } } } diff --git a/plugins/prefetch/test/CMakeLists.txt b/plugins/prefetch/test/CMakeLists.txt index a7c7b4405f5..111f75d674a 100644 --- a/plugins/prefetch/test/CMakeLists.txt +++ b/plugins/prefetch/test/CMakeLists.txt @@ -16,9 +16,13 @@ ####################### add_executable(test_evaluate test_evaluate.cc "${PROJECT_SOURCE_DIR}/common.cc" "${PROJECT_SOURCE_DIR}/evaluate.cc") +add_executable(test_prefetch_path test_path.cc "${PROJECT_SOURCE_DIR}/path.cc") target_link_libraries(test_evaluate PRIVATE Catch2::Catch2WithMain) +target_link_libraries(test_prefetch_path PRIVATE Catch2::Catch2WithMain) target_compile_definitions(test_evaluate PRIVATE PREFETCH_UNIT_TEST) +target_compile_definitions(test_prefetch_path PRIVATE PREFETCH_UNIT_TEST) add_catch2_test(NAME test_evaluate COMMAND test_evaluate) +add_catch2_test(NAME test_prefetch_path COMMAND test_prefetch_path) diff --git a/plugins/prefetch/test/test_path.cc b/plugins/prefetch/test/test_path.cc new file mode 100644 index 00000000000..dc08b42f9e2 --- /dev/null +++ b/plugins/prefetch/test/test_path.cc @@ -0,0 +1,67 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include + +#include "../path.h" + +TEST_CASE("Safe prefetch paths stay under the current directory", "[prefetch][path]") +{ + SafeRelativeFetchPath fetchPath; + + REQUIRE(makeSafeRelativeFetchPath("texts/demo-1.txt", "demo-2.txt", fetchPath)); + REQUIRE(fetchPath.path == "texts/demo-2.txt"); + REQUIRE_FALSE(fetchPath.hasQuery); + + REQUIRE(makeSafeRelativeFetchPath("texts/demo-1.txt", "./demo-2.txt", fetchPath)); + REQUIRE(fetchPath.path == "texts/demo-2.txt"); + REQUIRE_FALSE(fetchPath.hasQuery); + + REQUIRE(makeSafeRelativeFetchPath("texts/demo-1.txt", "segments/../demo-2.txt", fetchPath)); + REQUIRE(fetchPath.path == "texts/demo-2.txt"); + REQUIRE_FALSE(fetchPath.hasQuery); + + REQUIRE(makeSafeRelativeFetchPath("tests/query", "query?bar=baz", fetchPath)); + REQUIRE(fetchPath.path == "tests/query"); + REQUIRE(fetchPath.hasQuery); + REQUIRE(fetchPath.query == "bar=baz"); + + REQUIRE(makeSafeRelativeFetchPath("root.txt", "rooted", fetchPath)); + REQUIRE(fetchPath.path == "rooted"); + REQUIRE_FALSE(fetchPath.hasQuery); +} + +TEST_CASE("Unsafe prefetch paths cannot escape the current directory", "[prefetch][path]") +{ + SafeRelativeFetchPath fetchPath; + + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "?bar=baz", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "/demo-2.txt", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "/foo/../../bar", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "\\demo-2.txt", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "%2fdemo-2.txt", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "demo-2.txt#fragment", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "../private/secret.txt", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "foo/../../bar", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "segments/../../private/secret.txt", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "%2e%2e/private/secret.txt", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "..%2fprivate/secret.txt", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("texts/demo-1.txt", "..\\private\\secret.txt", fetchPath)); + REQUIRE_FALSE(makeSafeRelativeFetchPath("root.txt", "../private/secret.txt", fetchPath)); +} diff --git a/plugins/regex_remap/regex_remap.cc b/plugins/regex_remap/regex_remap.cc index 6eca1e8c33a..9c5d6e3834d 100644 --- a/plugins/regex_remap/regex_remap.cc +++ b/plugins/regex_remap/regex_remap.cc @@ -44,7 +44,7 @@ #include "tscore/ink_atomic.h" #include "tscore/ink_time.h" #include "tscore/ink_inet.h" - +#include "tsutil/LocalBuffer.h" #include "tsutil/Regex.h" static const char *PLUGIN_NAME = "regex_remap"; @@ -466,7 +466,7 @@ RemapRegex::compile(std::string &error, int &erroffset) // Get the lengths of the matching string(s), taking into account variable substitutions. // We also calculate a total length for the new string, which is the max length the -// substituted string can have (use it to allocate a buffer before calling substitute() ). +// substituted string can have (used for the ts::LocalBuffer). int RemapRegex::get_lengths(RegexMatches const &matches, int lengths[], TSRemapRequestInfo *rri, UrlComponents *req_url) { @@ -907,9 +907,9 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) TSRemapStatus retval = TSREMAP_DID_REMAP; RemapRegex *re = ri->first; int match_len = 0; - char *match_buf; - match_buf = static_cast(alloca(req_url.url_len + 32)); + // Cap the stack allocation to 16KB, a typical browser upper limit + ts::LocalBuffer match_buf(req_url.url_len + 32); if (ri->method) { // Prepend the URI path or URL with the HTTP method TSMBuffer mBuf; @@ -923,38 +923,38 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) if (match_len > 16) { match_len = 16; } - memcpy(match_buf, method, match_len); + memcpy(match_buf.data(), method, match_len); } } } if (ri->host && req_url.host && req_url.host_len > 0) { - memcpy(match_buf + match_len, "//", 2); - memcpy(match_buf + match_len + 2, req_url.host, req_url.host_len); + memcpy(match_buf.data() + match_len, "//", 2); + memcpy(match_buf.data() + match_len + 2, req_url.host, req_url.host_len); match_len += (req_url.host_len + 2); } - *(match_buf + match_len) = '/'; + *(match_buf.data() + match_len) = '/'; match_len++; if (req_url.path && req_url.path_len > 0) { - memcpy(match_buf + match_len, req_url.path, req_url.path_len); + memcpy(match_buf.data() + match_len, req_url.path, req_url.path_len); match_len += (req_url.path_len); } if (ri->query_string && req_url.query && req_url.query_len > 0) { - *(match_buf + match_len) = '?'; - memcpy(match_buf + match_len + 1, req_url.query, req_url.query_len); + *(match_buf.data() + match_len) = '?'; + memcpy(match_buf.data() + match_len + 1, req_url.query, req_url.query_len); match_len += (req_url.query_len + 1); } - match_buf[match_len] = '\0'; // NULL terminate the match string - Dbg(dbg_ctl, "Target match string is `%s'", match_buf); + match_buf.data()[match_len] = '\0'; // NULL terminate the match string + Dbg(dbg_ctl, "Target match string is `%s'", match_buf.data()); RegexMatches matches(MATCHCOUNT); // Apply the regular expressions, in order. First one wins. while (re) { // Since we check substitutions on parse time, we don't need to reset ovector - auto match_result = re->match(match_buf, matches); + auto match_result = re->match(match_buf.data(), matches); if (match_result >= 0) { int new_len = re->get_lengths(matches, lengths, rri, &req_url); @@ -1025,13 +1025,13 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) } if (new_len > 0) { - char *dest; + // Cap the stack allocation to 16KB, a typical browser upper limit + ts::LocalBuffer dest(new_len + 8); - dest = static_cast(alloca(new_len + 8)); - dest_len = re->substitute(dest, matches, lengths, txnp, rri, &req_url, lowercase_substitutions); + dest_len = re->substitute(dest.data(), matches, lengths, txnp, rri, &req_url, lowercase_substitutions); Dbg(dbg_ctl, "New URL is estimated to be %d bytes long, or less", new_len); - Dbg(dbg_ctl, "New URL is %s (length %d)", dest, dest_len); + Dbg(dbg_ctl, "New URL is %s (length %d)", dest.data(), dest_len); Dbg(dbg_ctl, " matched rule %d [%s]", re->order(), re->regex()); // Check for a quick response, if the status option is set @@ -1051,7 +1051,7 @@ TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri) // Now parse the new URL, which can also be the redirect URL if (dest_len > 0) { - const char *start = dest; + const char *start = dest.data(); // Setup the new URL if (TS_PARSE_ERROR == TSUrlParse(rri->requestBufp, rri->requestUrl, &start, start + dest_len)) { diff --git a/plugins/slice/server.cc b/plugins/slice/server.cc index ffcf653de94..7105d5157bd 100644 --- a/plugins/slice/server.cc +++ b/plugins/slice/server.cc @@ -585,9 +585,26 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data) } break; case BlockState::ActiveRef: { // Mark the reference block for "skip". - int64_t const blockbytes = data->m_config->m_blockbytes; - int64_t const firstblock = data->m_req_range.firstBlockFor(blockbytes); - int64_t const blockpos = firstblock * blockbytes; + int64_t const blockbytes = data->m_config->m_blockbytes; + int64_t const firstblock = data->m_req_range.firstBlockFor(blockbytes); + int64_t const blockpos = firstblock * blockbytes; + int64_t const range_beg = data->m_req_range.m_beg; + + // Once the content no longer reaches the requested first byte, the client range is unsatisfiable. + if (data->m_contentlen <= range_beg) { + if (data->m_config->canLogError()) { + ERROR_LOG("Content length %" PRId64 " shrunk below requested range start %" PRId64, data->m_contentlen, range_beg); + } + data->m_upstream.abort(); + data->m_blockstate = BlockState::Fail; + if (data->m_dnstream.m_write.isOpen()) { + TSVIOReenable(data->m_dnstream.m_write.m_vio); + } else { + shutdown(contp, data); + } + return; + } + int64_t const firstblockbytes = std::min(blockbytes, data->m_contentlen - blockpos); data->m_blockskip = firstblockbytes; diff --git a/plugins/webp_transform/ImageTransform.cc b/plugins/webp_transform/ImageTransform.cc index 809a5e5e989..e4ebc7f4d7c 100644 --- a/plugins/webp_transform/ImageTransform.cc +++ b/plugins/webp_transform/ImageTransform.cc @@ -16,8 +16,13 @@ limitations under the License. */ -#include +#include +#include +#include +#include #include +#include +#include #include #include @@ -79,6 +84,97 @@ has_signature_for(std::string_view data, ImageEncoding encoding) return false; } + +// Cap the buffered (encoded) response body. 16 MiB fits every +// realistic image asset while keeping the worst case bounded. The default is +// overridable with the max_buffer_size plugin argument (see TSPluginInit). +constexpr size_t DEFAULT_MAX_BUFFERED_IMAGE_SIZE = 16ULL * 1024 * 1024; +size_t max_buffered_image_size = DEFAULT_MAX_BUFFERED_IMAGE_SIZE; + +// Parse a byte count with an optional binary suffix (K, M, or G, 1024-based), +// for the max_buffer_size argument. Returns 0 on any malformed input so the +// caller can reject it and keep the default. +size_t +parse_size(const char *value) +{ + // strtoull() skips leading whitespace and then accepts an optional sign, so a + // value like "-1" would wrap to a huge size_t and silently disable the cap. + // Reject any signed input up front. Skip the SAME whitespace set strtoull() + // skips (the full isspace() set, not just space/tab); otherwise a value like + // "\n-1" slips past this guard and strtoull() still wraps it. + const char *p = value; + while (std::isspace(static_cast(*p))) { + ++p; + } + if (*p == '-' || *p == '+') { + return 0; + } + + errno = 0; + char *end = nullptr; + auto scaled = std::strtoull(p, &end, 10); + if (errno != 0 || end == p) { + return 0; + } + size_t multiplier = 1; + if (*end != '\0') { + if (end[1] != '\0') { // at most one suffix character + return 0; + } + switch (*end) { + case 'k': + case 'K': + multiplier = 1024ULL; + break; + case 'm': + case 'M': + multiplier = 1024ULL * 1024; + break; + case 'g': + case 'G': + multiplier = 1024ULL * 1024 * 1024; + break; + default: + return 0; + } + } + // Treat a multiply that would overflow size_t as invalid input rather than + // letting the cap wrap to an unintended (small) value. + if (scaled > std::numeric_limits::max() / multiplier) { + return 0; + } + return static_cast(scaled) * multiplier; +} + +// Decode-side limits. A small crafted image can declare huge +// dimensions and decode into a multi-gigabyte pixel buffer even though its +// encoded form fits under max_buffered_image_size. width/height/area bound the +// dimensions of a single decode; memory/map bound ImageMagick's pixel-cache RAM +// and disk(0) makes an over-limit decode fail as a caught Magick::Error rather +// than spilling to disk. area is sized to one full pixel cache (64 Mpixels at 8 +// bytes per pixel for a Q16 build is 512 MiB) so the dimension and RAM limits +// are mutually consistent. The RAM limits are process-wide and shared across +// concurrent decodes, so under load an over-budget decode reverts to the +// original bytes rather than converting. +constexpr size_t MAX_IMAGE_DIMENSION_PX = 16000; +constexpr size_t MAX_IMAGE_AREA_PX = 64ULL * 1024 * 1024; +constexpr size_t MAX_PIXEL_CACHE_BYTES = 512ULL * 1024 * 1024; + +// Map an encoding to the client-facing Content-Type it should be labeled with. +const char * +content_type_for(ImageEncoding encoding) +{ + switch (encoding) { + case ImageEncoding::webp: + return "image/webp"; + case ImageEncoding::jpeg: + return "image/jpeg"; + case ImageEncoding::png: + return "image/png"; + default: + return nullptr; + } +} } // namespace class ImageTransform : public TransformationPlugin @@ -98,8 +194,17 @@ class ImageTransform : public TransformationPlugin void handleReadResponseHeaders(Transaction &transaction) override { - transaction.getServerResponse().getHeaders()["Vary"] = "Accept"; // to have a separate cache entry - + // Label the server response so both the cached transform and the client + // copy carry the target type. On a degraded transform (pass-through or + // decode error) the body is the original encoding but the label still says + // the target; handleSendResponseHeaders below corrects the client-facing + // copy using _input_content_type in that case. The cached label can still + // end up wrong on a degraded transform; fixing that without mislabeling + // the cache is tracked as a separate correctness issue. + if (const char *ctype = content_type_for(_transform_image_type); ctype != nullptr) { + transaction.getServerResponse().getHeaders()["Content-Type"] = ctype; + } + transaction.getServerResponse().getHeaders()["Vary"] = "Accept"; // separate cache entry per Accept Dbg(webp_dbg_ctl, "url %s", transaction.getServerRequest().getUrl().getUrlString().c_str()); transaction.resume(); } @@ -107,50 +212,75 @@ class ImageTransform : public TransformationPlugin void handleSendResponseHeaders(Transaction &transaction) override { - if (_transform_image_type == _input_image_type) { - transaction.getClientResponse().getHeaders()["Content-Type"] = _input_content_type; + // If the body exceeded the cap we produced no transformed + // body. The client response headers are not built until setOutputComplete, + // so we can still turn the 200 into an error here, giving the client a + // clear failure with an empty body instead of a truncated 200. (We cannot + // use Transaction::error() for this; it asserts once the response is in + // flight.) + if (_refused) { + Response &response = transaction.getClientResponse(); + response.setStatusCode(HTTP_STATUS_BAD_GATEWAY); + response.setReasonPhrase("Bad Gateway"); + // Drop the image labeling the read hook added for a conversion that did + // not happen; this is an empty error response, not an image. + response.getHeaders().erase("Content-Type"); + response.getHeaders().erase("Vary"); transaction.resume(); return; } - switch (_transform_image_type) { - case ImageEncoding::webp: - transaction.getClientResponse().getHeaders()["Content-Type"] = "image/webp"; - break; - case ImageEncoding::jpeg: - transaction.getClientResponse().getHeaders()["Content-Type"] = "image/jpeg"; - break; - case ImageEncoding::png: - transaction.getClientResponse().getHeaders()["Content-Type"] = "image/png"; - break; - case ImageEncoding::unknown: - // do nothing - break; + // Signature mismatch or decode failure reverted us to the original + // encoding (see pass_through()); relabel the client-facing response to + // match the body we actually sent. + if (_transform_image_type == _input_image_type) { + transaction.getClientResponse().getHeaders()["Content-Type"] = _input_content_type; } - transaction.resume(); } void consume(std::string_view data) override { - _img.write(data.data(), data.length()); + // The response body is buffered in full before being handed + // to ImageMagick. A malicious or just unusually large origin response can + // drive the proxy to OOM, so the buffer is bounded by a per-transaction + // cap. When the cap is exceeded we drop what we have and stop buffering; + // handleInputComplete then produces no body and handleSendResponseHeaders + // turns the response into a 502. Because the transform emits nothing until + // then, the client gets an error with an empty body rather than the + // oversized image. Transaction::error() cannot be used here: it asserts + // once the response is in flight. + if (_refused) { + return; + } + if (_img.size() + data.length() > max_buffered_image_size) { + TSError("[webp_transform] response body exceeds cap %zu, returning 502", max_buffered_image_size); + _refused = true; + _img.clear(); + _img.shrink_to_fit(); + return; + } + _img.append(data.data(), data.length()); } void handleInputComplete() override { - std::string input_data = _img.str(); + if (_refused) { + setOutputComplete(); // no body produced; handleSendResponseHeaders turns this into a 502 + return; + } - if (!has_signature_for(input_data, _input_image_type)) { + if (!has_signature_for(_img, _input_image_type)) { TSError("[webp_transform] input body does not match its declared image encoding: %d, length: %zu", - static_cast(_input_image_type), input_data.length()); - pass_through(input_data); + static_cast(_input_image_type), _img.length()); + pass_through(_img); setOutputComplete(); return; } - Blob input_blob(input_data.data(), input_data.length()); + Blob input_blob(_img.data(), _img.length()); Image image; try { @@ -168,12 +298,24 @@ class ImageTransform : public TransformationPlugin } image.write(&output_blob); produce(std::string_view(reinterpret_cast(output_blob.data()), output_blob.length())); - } catch (Magick::Warning &warning) { + } catch (const Magick::Warning &warning) { TSError("ImageMagick++ warning: %s", warning.what()); pass_through(std::string_view(reinterpret_cast(input_blob.data()), input_blob.length())); - } catch (Magick::Error &error) { - TSError("ImageMagick++ error: %s _image_type: %d input_data.length(): %zu", error.what(), (int)_transform_image_type, - input_data.length()); + } catch (const Magick::Error &error) { + TSError("ImageMagick++ error: %s _image_type: %d input length: %zu", error.what(), (int)_transform_image_type, _img.length()); + pass_through(std::string_view(reinterpret_cast(input_blob.data()), input_blob.length())); + } catch (const std::exception &e) { + // ImageMagick++ can throw other exception types (e.g. + // std::bad_alloc on huge or malformed inputs). Catch them so an + // uncaught exception does not terminate the process. Log the input type + // and buffered length so a memory-pressure attack (repeated bad_alloc on + // large inputs) is distinguishable from a one-off decode hiccup. + TSError("[webp_transform] std::exception during transform: %s _image_type: %d input length: %zu", e.what(), + (int)_transform_image_type, _img.length()); + pass_through(std::string_view(reinterpret_cast(input_blob.data()), input_blob.length())); + } catch (...) { + TSError("[webp_transform] unknown exception during transform _image_type: %d input length: %zu", (int)_transform_image_type, + _img.length()); pass_through(std::string_view(reinterpret_cast(input_blob.data()), input_blob.length())); } @@ -192,10 +334,11 @@ class ImageTransform : public TransformationPlugin _transform_image_type = _input_image_type; } - std::stringstream _img; - std::string _input_content_type; - ImageEncoding _input_image_type; - ImageEncoding _transform_image_type; + std::string _img; + std::string _input_content_type; + bool _refused = false; + ImageEncoding _input_image_type; + ImageEncoding _transform_image_type; }; class GlobalHookPlugin : public GlobalPlugin @@ -239,15 +382,64 @@ class GlobalHookPlugin : public GlobalPlugin // If we might need to convert check to see if what the browser supports if (transaction_convert_to_webp == true || transaction_convert_to_jpeg == true) { + // When the origin advertises a body larger than the cap, + // decline the transform up front so the original response passes through + // untouched rather than being buffered up to the cap and then forwarded + // under a transformed Content-Type. Bodies without a Content-Length are + // still bounded by the per-transaction cap inside ImageTransform. + bool content_length_usable = false; + std::string content_length = transaction.getServerResponse().getHeaders().values("Content-Length"); + if (!content_length.empty()) { + const char *cstr = content_length.c_str(); + // Reject a leading sign: strtoull() would wrap a negative value to a + // huge size_t and spuriously decline the transform. + bool signed_input = (*cstr == '-' || *cstr == '+'); + + errno = 0; + char *end = nullptr; + unsigned long long declared = std::strtoull(cstr, &end, 10); + + // Require the entire header value to be a single integer (optional + // trailing whitespace only). Headers::values() comma-joins duplicate + // Content-Length headers, so "1,20971520" would otherwise parse as "1" + // and bypass this up-front decline. + while (*end == ' ' || *end == '\t') { + ++end; + } + bool fully_parsed = (errno == 0) && (end != cstr) && (*end == '\0') && !signed_input; + + if (fully_parsed && declared > max_buffered_image_size) { + Dbg(webp_dbg_ctl, "origin Content-Length %llu exceeds cap %zu, not transforming", declared, max_buffered_image_size); + transaction.resume(); + return; + } + // A fully parsed Content-Length at or under the cap guarantees the body + // fits and the transform will complete, so the result stays cacheable. + content_length_usable = fully_parsed; + } + std::string accept = transaction.getServerRequest().getHeaders().values("Accept"); bool webp_supported = accept.find("image/webp") != std::string::npos; Dbg(webp_dbg_ctl, "Accept: %s webp_suppported: %d", accept.c_str(), webp_supported); + // Without a usable Content-Length the body may exceed the cap mid-stream + // and be refused, which yields an empty 502 to the client. The cacheable + // object would be the origin 200 plus the empty transform output relabeled + // with the target Content-Type, so mark such responses no-store to keep a + // poisoned 200/empty-body entry out of the cache. Bodies declared over the + // cap are declined above; bodies at or under the cap transform fully and + // remain cacheable. if (webp_supported == true && transaction_convert_to_webp == true) { Dbg(webp_dbg_ctl, "Content type is either jpeg or png. Converting to webp"); + if (!content_length_usable) { + TSHttpTxnServerRespNoStoreSet(static_cast(transaction.getAtsHandle()), 1); + } transaction.addPlugin(new ImageTransform(transaction, ctype, input_image_type, ImageEncoding::webp)); } else if (webp_supported == false && transaction_convert_to_jpeg == true) { Dbg(webp_dbg_ctl, "Content type is webp. Converting to jpeg"); + if (!content_length_usable) { + TSHttpTxnServerRespNoStoreSet(static_cast(transaction.getAtsHandle()), 1); + } transaction.addPlugin(new ImageTransform(transaction, ctype, input_image_type, ImageEncoding::jpeg)); } else { Dbg(webp_dbg_ctl, "Nothing to convert"); @@ -265,21 +457,47 @@ TSPluginInit(int argc, const char *argv[]) return; } - if (argc >= 2) { - std::string option(argv[1]); - if (option.find("convert_to_webp") != std::string::npos) { + constexpr std::string_view max_buffer_prefix = "max_buffer_size="; + + bool convert_specified = false; + for (int i = 1; i < argc; ++i) { + std::string_view arg(argv[i]); + bool recognized = false; + // Independent checks (not mutually exclusive) so the legacy comma-combined + // form "convert_to_jpeg,convert_to_webp" in a single argument still enables + // both directions. + if (arg.find("convert_to_webp") != std::string_view::npos) { Dbg(webp_dbg_ctl, "Configured to convert to webp"); config_convert_to_webp = true; + convert_specified = true; + recognized = true; } - if (option.find("convert_to_jpeg") != std::string::npos) { + if (arg.find("convert_to_jpeg") != std::string_view::npos) { Dbg(webp_dbg_ctl, "Configured to convert to jpeg"); config_convert_to_jpeg = true; + convert_specified = true; + recognized = true; + } + if (arg.substr(0, max_buffer_prefix.size()) == max_buffer_prefix) { + size_t parsed = parse_size(argv[i] + max_buffer_prefix.size()); + if (parsed == 0) { + TSError("[webp_transform] invalid %.*s, keeping default %zu bytes", static_cast(arg.size()), arg.data(), + max_buffered_image_size); + } else { + max_buffered_image_size = parsed; + Dbg(webp_dbg_ctl, "max buffered image size set to %zu bytes", max_buffered_image_size); + } + recognized = true; } - if (config_convert_to_webp == false && config_convert_to_jpeg == false) { - TSError("Unknown option: %s", option.c_str()); + if (!recognized) { + TSError("[webp_transform] unknown option: %.*s", static_cast(arg.size()), arg.data()); } - } else { - Dbg(webp_dbg_ctl, "Default configuration is to convert both webp and jpeg"); + } + + // If no conversion direction was named, default to converting both, matching + // the no-argument behavior. + if (!convert_specified) { + Dbg(webp_dbg_ctl, "No conversion direction given; converting both webp and jpeg"); config_convert_to_webp = true; config_convert_to_jpeg = true; } @@ -288,5 +506,18 @@ TSPluginInit(int argc, const char *argv[]) stat_convert_to_jpeg.init("plugin." TAG ".convert_to_jpeg", Stat::SYNC_SUM, false); InitializeMagick(""); + + // Bound the decode so a small image declaring huge dimensions + // cannot decode into a multi-gigabyte pixel buffer. disk(0) turns an + // over-budget decode into a fast Magick::Error (which we catch and revert) + // rather than letting ImageMagick spill the oversized pixel cache to disk. + // See the limit constants above for how the dimension and RAM caps line up. + Magick::ResourceLimits::width(MAX_IMAGE_DIMENSION_PX); + Magick::ResourceLimits::height(MAX_IMAGE_DIMENSION_PX); + Magick::ResourceLimits::area(MAX_IMAGE_AREA_PX); // max width*height in pixels held in the cache + Magick::ResourceLimits::memory(MAX_PIXEL_CACHE_BYTES); // heap pixel-cache budget + Magick::ResourceLimits::map(MAX_PIXEL_CACHE_BYTES); // memory-mapped pixel-cache budget + Magick::ResourceLimits::disk(0); // no disk-backed spill; fail rather than thrash + plugin = new GlobalHookPlugin(); } diff --git a/plugins/xdebug/xdebug.cc b/plugins/xdebug/xdebug.cc index 7983c290503..32cd8631372 100644 --- a/plugins/xdebug/xdebug.cc +++ b/plugins/xdebug/xdebug.cc @@ -393,7 +393,7 @@ InjectRemapHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) const char *toUrlStr = getRemapUrlStr(txn, TSRemapToUrlGet, toUrlStrLen); char buf[2048]; - int len = snprintf(buf, sizeof(buf), "from=%*s, to=%*s", fromUrlStrLen, fromUrlStr, toUrlStrLen, toUrlStr); + int len = snprintf(buf, sizeof(buf), "from=%.*s, to=%.*s", fromUrlStrLen, fromUrlStr, toUrlStrLen, toUrlStr); if (fromUrlStr != NotFound) { TSfree(const_cast(fromUrlStr)); @@ -402,7 +402,12 @@ InjectRemapHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr) TSfree(const_cast(toUrlStr)); } - TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, -1 /* idx */, buf, len) == TS_SUCCESS); + if (len > 0) { + if (static_cast(len) >= sizeof(buf)) { + len = sizeof(buf) - 1; + } + TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, -1 /* idx */, buf, len) == TS_SUCCESS); + } TSHandleMLocRelease(buffer, hdr, dst); } } diff --git a/src/api/InkAPI.cc b/src/api/InkAPI.cc index 5500b44a6c8..9aabc670044 100644 --- a/src/api/InkAPI.cc +++ b/src/api/InkAPI.cc @@ -1650,7 +1650,9 @@ TSMimeFieldValueGet(TSMBuffer /* bufp ATS_UNUSED */, TSMLoc field_obj, int idx, } } -static void +// Returns false when the value exceeds the uint16_t field-length limit and was +// rejected by mime_field_value_set, so callers can surface TS_ERROR. +static bool TSMimeFieldValueSet(TSMBuffer bufp, TSMLoc field_obj, int idx, const char *value, int length) { MIMEFieldSDKHandle *handle = reinterpret_cast(field_obj); @@ -1663,10 +1665,10 @@ TSMimeFieldValueSet(TSMBuffer bufp, TSMLoc field_obj, int idx, const char *value if (idx >= 0) { mime_field_value_set_comma_val(heap, handle->mh, handle->field_ptr, idx, std::string_view{value, static_cast(length)}); - } else { - mime_field_value_set(heap, handle->mh, handle->field_ptr, - std::string_view{value, static_cast(length)}, true); + return true; } + return mime_field_value_set(heap, handle->mh, handle->field_ptr, + std::string_view{value, static_cast(length)}, true); } static void @@ -1896,7 +1898,13 @@ TSMimeHdrFieldCreateNamed(TSMBuffer bufp, TSMLoc mh_mloc, const char *name, int HdrHeap *heap = ((reinterpret_cast(bufp))->m_heap); MIMEFieldSDKHandle *h = sdk_alloc_field_handle(bufp, mh); h->field_ptr = mime_field_create_named(heap, mh, std::string_view{name, static_cast(name_len)}); - *locp = reinterpret_cast(h); + if (h->field_ptr == nullptr) { + // The name exceeds the uint16_t field-length limit; nothing was created. + sdk_free_field_handle(bufp, h); + *locp = nullptr; + return TS_ERROR; + } + *locp = reinterpret_cast(h); return TS_SUCCESS; } @@ -2104,12 +2112,15 @@ TSMimeHdrFieldNameSet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, const char *name mime_hdr_field_detach(handle->mh, handle->field_ptr, false); } - handle->field_ptr->name_set(heap, handle->mh, std::string_view{name, static_cast(length)}); + bool const stored = + handle->field_ptr->name_set(heap, handle->mh, std::string_view{name, static_cast(length)}); if (attached) { mime_hdr_field_attach(handle->mh, handle->field_ptr, 1, nullptr); } - return TS_SUCCESS; + // A rejected oversized name leaves the field's prior name intact; report the + // failure so the plugin knows the set did not take effect. + return stored ? TS_SUCCESS : TS_ERROR; } TSReturnCode @@ -2259,8 +2270,7 @@ TSMimeHdrFieldValueStringSet(TSMBuffer bufp, TSMLoc hdr, TSMLoc field, int idx, length = strlen(value); } - TSMimeFieldValueSet(bufp, field, idx, value, length); - return TS_SUCCESS; + return TSMimeFieldValueSet(bufp, field, idx, value, length) ? TS_SUCCESS : TS_ERROR; } TSReturnCode @@ -5068,7 +5078,7 @@ TSHttpTxnNextHopNamedStrategyGet(TSHttpTxn txnp, const char *name) auto sm = reinterpret_cast(txnp); - sdk_assert(sdk_sanity_check_null_ptr((void *)sm->m_remap) == TS_SUCCESS); + sdk_assert(sdk_sanity_check_null_ptr((void *)sm->m_remap.get()) == TS_SUCCESS); sdk_assert(sdk_sanity_check_null_ptr((void *)sm->m_remap->strategyFactory) == TS_SUCCESS); // HttpSM has a reference count handle to UrlRewrite which has a @@ -6742,7 +6752,8 @@ TSHttpTxnRedirectUrlSet(TSHttpTxn txnp, const char *url, const int url_len) sm->redirect_url = const_cast(url); sm->redirect_url_len = url_len; sm->enable_redirection = true; - sm->redirection_tries = 0; + // Don't reset HttpSM::redirection_tries here: a per-hop reset defeats the number_of_redirections + // limit that HttpSM enforces, allowing an unbounded redirect chain. // Make sure we allow for at least one redirection. if (sm->t_state.txn_conf->number_of_redirections <= 0) { diff --git a/src/cripts/Bundles/HRWBridge.cc b/src/cripts/Bundles/HRWBridge.cc index 6649a32be9f..e2c2ed648f1 100644 --- a/src/cripts/Bundles/HRWBridge.cc +++ b/src/cripts/Bundles/HRWBridge.cc @@ -39,7 +39,7 @@ class ID : public detail::HRWBridge ID(const cripts::string_view &id); ~ID() override = default; - cripts::string_view value(cripts::Context *context) override; + cripts::string_view value(cripts::Context *context, cripts::string &scratch) override; private: Type _type = Type::none; @@ -59,24 +59,24 @@ ID::ID(const cripts::string_view &id) : super_type(id) } cripts::string_view -ID::value(cripts::Context *context) +ID::value(cripts::Context *context, cripts::string &scratch) { switch (_type) { case Type::REQUEST: - _value = cripts::UUID::Request::_get(context); + scratch = cripts::UUID::Request::_get(context); break; case Type::PROCESS: - _value = cripts::UUID::Process::_get(context); + scratch = cripts::UUID::Process::_get(context); break; case Type::UNIQUE: - _value = cripts::UUID::Unique::_get(context); + scratch = cripts::UUID::Unique::_get(context); break; default: - _value = ""; + scratch.clear(); break; } - return _value; + return scratch; } ///////////////////////////////////////////////////////////////////////////// @@ -95,7 +95,7 @@ class IP : public detail::HRWBridge IP(const cripts::string_view &ip); ~IP() override = default; - cripts::string_view value(cripts::Context *context) override; + cripts::string_view value(cripts::Context *context, cripts::string &scratch) override; private: Type _type = Type::none; @@ -110,38 +110,38 @@ IP::IP(const cripts::string_view &type) : super_type(type) } else if (type == "SERVER") { _type = Type::SERVER; } else if (type == "OUTBOUND") { - _type = Type::INBOUND; + _type = Type::OUTBOUND; } else { CFatal("[Cripts::Headers] Unknown HRWBridge IP type: %s.", type.data()); } } cripts::string_view -IP::value(cripts::Context *context) +IP::value(cripts::Context *context, cripts::string &scratch) { switch (_type) { case Type::CLIENT: { auto ip = cripts::Client::Connection::Get().IP(); - _value = ip.string(); + scratch = ip.string(); } break; case Type::INBOUND: { auto ip = cripts::Client::Connection::Get().LocalIP(); - _value = ip.string(); + scratch = ip.string(); } break; case Type::SERVER: { auto ip = cripts::Server::Connection::Get().IP(); - _value = ip.string(); + scratch = ip.string(); } break; case Type::OUTBOUND: { auto ip = cripts::Server::Connection::Get().LocalIP(); - _value = ip.string(); + scratch = ip.string(); } break; default: - _value = ""; + scratch.clear(); break; } - return _value; + return scratch; } ///////////////////////////////////////////////////////////////////////////// @@ -158,7 +158,7 @@ class CIDR : public detail::HRWBridge CIDR(cripts::string_view &cidr); ~CIDR() override = default; - cripts::string_view value(cripts::Context *context) override; + cripts::string_view value(cripts::Context *context, cripts::string &scratch) override; private: unsigned int _ipv4_cidr = 32; @@ -184,13 +184,13 @@ CIDR::CIDR(cripts::string_view &cidr) : super_type(cidr) } cripts::string_view -CIDR::value(cripts::Context *context) +CIDR::value(cripts::Context *context, cripts::string &scratch) { auto ip = cripts::Client::Connection::Get().IP(); - _value = ip.string(_ipv4_cidr, _ipv6_cidr); + scratch = ip.string(_ipv4_cidr, _ipv6_cidr); - return _value; + return scratch; } ///////////////////////////////////////////////////////////////////////////// @@ -212,50 +212,45 @@ class URL : public detail::HRWBridge URL(Type utype, const cripts::string_view &comp); ~URL() override = default; - cripts::string_view value(cripts::Context *context) override; + cripts::string_view value(cripts::Context *context, cripts::string &scratch) override; private: - cripts::string_view _getComponent(cripts::Url &url); + cripts::string_view _getComponent(cripts::Url &url, cripts::string &scratch); Type _type = Type::none; Component _comp = Component::none; }; cripts::string_view -URL::_getComponent(cripts::Url &url) +URL::_getComponent(cripts::Url &url, cripts::string &scratch) { switch (_comp) { case Component::HOST: return url.host.GetSV(); - break; case Component::PATH: return url.path; - break; case Component::PORT: - _value = cripts::string(std::to_string(url.port)); - break; + scratch = cripts::string(std::to_string(url.port)); + return scratch; case Component::QUERY: return url.query; - break; case Component::SCHEME: return url.scheme; - break; case Component::URL: return ""; // return url.url; - break; default: CFatal("[Cripts::Headers] Invalid URL component in HRWBridge."); break; } - return ""; // Should never happen + return ""; // unreachable } URL::URL(Type utype, const cripts::string_view &comp) : super_type("") @@ -280,51 +275,51 @@ URL::URL(Type utype, const cripts::string_view &comp) : super_type("") } cripts::string_view -URL::value(cripts::Context *context) +URL::value(cripts::Context *context, cripts::string &scratch) { switch (_type) { case Type::CLIENT: { borrow url = cripts::Client::URL::Get(); - return _getComponent(url); - } break; + return _getComponent(url, scratch); + } case Type::REMAP_FROM: { borrow url = cripts::Remap::From::URL::Get(); - return _getComponent(url); - } break; + return _getComponent(url, scratch); + } case Type::REMAP_TO: { borrow url = cripts::Remap::To::URL::Get(); - return _getComponent(url); - } break; + return _getComponent(url, scratch); + } case Type::PRISTINE: { borrow url = cripts::Pristine::URL::Get(); - return _getComponent(url); - } break; + return _getComponent(url, scratch); + } case Type::CACHE: { borrow url = cripts::Cache::URL::Get(); - return _getComponent(url); - } break; + return _getComponent(url, scratch); + } case Type::PARENT: { borrow url = cripts::Parent::URL::Get(); - return _getComponent(url); - } break; + return _getComponent(url, scratch); + } default: CFatal("[Cripts::Headers] Invalid URL type in HRWBridge."); break; } - return _value; + return {}; // unreachable } } // namespace detail diff --git a/src/cripts/Bundles/Headers.cc b/src/cripts/Bundles/Headers.cc index 553a96ba061..e42da5dfb31 100644 --- a/src/cripts/Bundles/Headers.cc +++ b/src/cripts/Bundles/Headers.cc @@ -65,11 +65,11 @@ Headers::rm_headers(const cripts::string_view target, const HeaderList &headers) NeedCallback(cripts::Callbacks::DO_SEND_RESPONSE); break; case SERVER_REQUEST: - _client_response.rm_headers.insert(_client_response.rm_headers.end(), headers.begin(), headers.end()); + _server_request.rm_headers.insert(_server_request.rm_headers.end(), headers.begin(), headers.end()); NeedCallback(cripts::Callbacks::DO_SEND_REQUEST); break; case SERVER_RESPONSE: - _client_response.rm_headers.insert(_client_response.rm_headers.end(), headers.begin(), headers.end()); + _server_response.rm_headers.insert(_server_response.rm_headers.end(), headers.begin(), headers.end()); NeedCallback(cripts::Callbacks::DO_READ_RESPONSE); break; default: @@ -122,8 +122,9 @@ Headers::doRemap(cripts::Context *context) req[header] = ""; } + cripts::string scratch; for (auto &header : _client_request.set_headers) { - req[header.first] = header.second->value(context); + req[header.first] = header.second->value(context, scratch); } } @@ -136,8 +137,9 @@ Headers::doSendResponse(cripts::Context *context) resp[header] = ""; } + cripts::string scratch; for (auto &header : _client_response.set_headers) { - resp[header.first] = header.second->value(context); + resp[header.first] = header.second->value(context, scratch); } } @@ -150,8 +152,9 @@ Headers::doSendRequest(cripts::Context *context) req[header] = ""; } + cripts::string scratch; for (auto &header : _server_request.set_headers) { - req[header.first] = header.second->value(context); + req[header.first] = header.second->value(context, scratch); } } @@ -164,8 +167,9 @@ Headers::doReadResponse(cripts::Context *context) resp[header] = ""; } + cripts::string scratch; for (auto &header : _server_response.set_headers) { - resp[header.first] = header.second->value(context); + resp[header.first] = header.second->value(context, scratch); } } diff --git a/src/cripts/Connections.cc b/src/cripts/Connections.cc index 8355274dc1c..ade5d1a64c5 100644 --- a/src/cripts/Connections.cc +++ b/src/cripts/Connections.cc @@ -142,13 +142,10 @@ IP::GetSV(unsigned ipv4_cidr, unsigned ipv6_cidr) return ""; } -sockaddr +swoc::IPEndpoint IP::Socket() const { - sockaddr addr = {}; - - this->copy_to(&addr); - return addr; + return swoc::IPEndpoint{*this}; } uint64_t diff --git a/src/cripts/Context.cc b/src/cripts/Context.cc index fe1a2ce389a..1a5c7bfcb40 100644 --- a/src/cripts/Context.cc +++ b/src/cripts/Context.cc @@ -34,13 +34,17 @@ Context::reset() { // Clear the initialized headers before calling next hook // Note: we don't clear the pristine URL, nor the Remap From/To URLs, they are static. - // We also don't clear the client URL, since it's from the RRI. if (_client.response.Initialized()) { _client.response.Reset(); } if (_server.response.Initialized()) { _server.response.Reset(); } + + // Reset the client URL (releases its handle only when we own it). + if (_urls.request.Initialized()) { + _urls.request.Reset(); + } if (_client.request.Initialized()) { _client.request.Reset(); } diff --git a/src/cripts/Files.cc b/src/cripts/Files.cc index aa5c018fe59..5b96325eaae 100644 --- a/src/cripts/Files.cc +++ b/src/cripts/Files.cc @@ -35,7 +35,16 @@ File::Path & File::Path::Rebase() { if (std::filesystem::status(*this).type() != std::filesystem::file_type::regular) { - *this = RecConfigReadConfigDir() + "/" + this->string(); + auto config_dir = std::filesystem::canonical(RecConfigReadConfigDir()); + auto rebased = std::filesystem::weakly_canonical(config_dir / *this); + + if (auto mm = std::mismatch(config_dir.begin(), config_dir.end(), rebased.begin(), rebased.end()); + mm.first == config_dir.end()) { + static_cast(*this) = rebased; + } else { + TSError("[Cripts] File::Path::Rebase: '%s' escapes config directory, clearing path", this->c_str()); + static_cast(*this).clear(); + } } return *this; diff --git a/src/cripts/Geo.cc b/src/cripts/Geo.cc index 3b9e9907e34..9a9bf55242c 100644 --- a/src/cripts/Geo.cc +++ b/src/cripts/Geo.cc @@ -93,7 +93,7 @@ cripts::string get_geo_string_from_ip(const cripts::IP &ip, Qualifiers q) { auto addr = ip.Socket(); - return get_geo_string(&addr, q); + return get_geo_string(addr, q); } // IP class Geo methods - can be used with any IP address diff --git a/src/cripts/Instance.cc b/src/cripts/Instance.cc index ebc1f3a34b6..04afa332e08 100644 --- a/src/cripts/Instance.cc +++ b/src/cripts/Instance.cc @@ -30,7 +30,7 @@ Instance::_initialize(int argc, const char *argv[], const char *filename, bool r if (remap) { from_url = argv[0]; to_url = argv[1]; - for (int i = 2; i < argc; i++) { + for (int i = 2; i < argc && (i - 2) < static_cast(data.size()); i++) { auto s = cripts::string(argv[i]); s.trim("\"\'"); diff --git a/src/cripts/Urls.cc b/src/cripts/Urls.cc index 044b0c551a0..eda2a3a3202 100644 --- a/src/cripts/Urls.cc +++ b/src/cripts/Urls.cc @@ -386,14 +386,15 @@ Url::Query::Erase(std::initializer_list list, bool keep) for (auto viter = s.ordered.begin(); viter != s.ordered.end();) { if (list.end() == std::ranges::find(list, *viter)) { - auto iter = s.hashed.find(*viter); - - CAssert(iter != s.hashed.end()); - s.size -= iter->second.size(); // Size of the erased value - s.size -= viter->size(); // Length of the erased key - s.hashed.erase(iter); - viter = s.ordered.erase(viter); - s.modified = true; + // Duplicate keys (?a=1&a=2) share one hashed entry, so a later occurrence may + // already be gone -- guard the lookup; attacker input must not abort. + if (auto iter = s.hashed.find(*viter); iter != s.hashed.end()) { + s.size -= iter->second.size(); // Size of the erased value + s.hashed.erase(iter); + } + s.size -= viter->size(); // Length of the erased key + viter = s.ordered.erase(viter); + s.modified = true; } else { ++viter; } @@ -484,9 +485,10 @@ Client::URL::_initialize() { if (_context->rriValid()) { super_type::_initialize(); - _bufp = _context->rri->requestBufp; - _hdr_loc = _context->rri->requestHdrp; - _urlp = _context->rri->requestUrl; + _bufp = _context->rri->requestBufp; + _hdr_loc = _context->rri->requestHdrp; + _urlp = _context->rri->requestUrl; + _owns_urlp = false; } else { Client::Request &req = Client::Request::_get(_context); // Repurpose / create the shared request object @@ -497,6 +499,7 @@ Client::URL::_initialize() _context->state.error.Fail(); } else { super_type::_initialize(); + _owns_urlp = true; } } } diff --git a/src/iocore/aio/AIO.cc b/src/iocore/aio/AIO.cc index 1a4fb4fe9bf..67fba1b29fd 100644 --- a/src/iocore/aio/AIO.cc +++ b/src/iocore/aio/AIO.cc @@ -310,7 +310,8 @@ aio_init_fildes(int fildes, int fromAPI = 0) thread_is_created = 1; thread_num = api_config_threads_per_disk; } else { - request->filedes = fildes; + request->filedes = fildes; + ink_release_assert(num_filedes < MAX_DISKS_POSSIBLE); aio_reqs[num_filedes] = request; thread_num = cache_config_threads_per_disk; } diff --git a/src/iocore/cache/CacheRead.cc b/src/iocore/cache/CacheRead.cc index d0456caf187..5c60a68d700 100644 --- a/src/iocore/cache/CacheRead.cc +++ b/src/iocore/cache/CacheRead.cc @@ -29,6 +29,8 @@ #include "tscore/InkErrno.h" #include "ts/ats_probe.h" +#include + #ifdef DEBUG #include "iocore/eventsystem/EThread.h" #endif @@ -48,6 +50,24 @@ DbgCtl dbg_ctl_cache_hit_evac{"cache_hit_evac"}; constexpr int MAX_READ_RECURSION_DEPTH = 10; +// Per-thread recursion counter for openReadStartEarliest. The counter must +// outlive the CacheVC because the recursive handleEvent below can free `this`, +// after which any access through a CacheVC member would be a use-after-free. +thread_local int t_read_recursive = 0; + +// Test hook: when the env variable ATS_TEST_FORCE_CORRUPT_DOC is set at startup +// every doc read is treated as having a bad magic, which drives +// openReadStartEarliest into the recursive Lread path. When an inner recursion +// level falls into free_CacheVC the outer frame then touches the recursion +// counter through the freed `this`; reaching the depth limit is not required to +// trigger it. Used by the autest reproducer to observe the recursive-read +// use-after-free (ASan build) on the unfixed sources. +#if TS_HAS_TESTS +static bool const test_force_corrupt_doc = std::getenv("ATS_TEST_FORCE_CORRUPT_DOC") != nullptr; +#else +static constexpr bool test_force_corrupt_doc = false; +#endif + } // end anonymous namespace uint32_t @@ -786,7 +806,7 @@ CacheVC::openReadStartEarliest(int /* event ATS_UNUSED */, Event * /* e ATS_UNUS goto Lread; } doc = reinterpret_cast(buf->data()); - if (doc->magic != DOC_MAGIC) { + if (doc->magic != DOC_MAGIC || test_force_corrupt_doc) { char tmpstring[CRYPTO_HEX_SIZE]; if (is_action_tag_set("cache")) { ink_release_assert(false); @@ -827,12 +847,12 @@ CacheVC::openReadStartEarliest(int /* event ATS_UNUSED */, Event * /* e ATS_UNUS if ((call_result = do_read_call(&key)) == EVENT_RETURN) { if (this->handler == reinterpret_cast(&CacheVC::openReadStartEarliest)) { is_recursive_call = true; - if (read_recursive > MAX_READ_RECURSION_DEPTH) { + if (t_read_recursive > MAX_READ_RECURSION_DEPTH) { char tmpstring[CRYPTO_HEX_SIZE]; Error("Too many recursive calls with %s", key.toHexStr(tmpstring)); goto Ldone; } - ++read_recursive; + ++t_read_recursive; } goto Lcallreturn; @@ -906,8 +926,11 @@ CacheVC::openReadStartEarliest(int /* event ATS_UNUSED */, Event * /* e ATS_UNUS return free_CacheVC(this); Lcallreturn: event_result = handleEvent(AIO_EVENT_DONE, nullptr); // hopefully a tail call + // handleEvent above may free `this` on the recursive failure path, so the + // depth counter cannot be a CacheVC member. It is a thread-local + // (t_read_recursive) and remains valid after `this` is gone. if (is_recursive_call) { - --read_recursive; + --t_read_recursive; } return event_result; Lsuccess: diff --git a/src/iocore/cache/CacheVC.h b/src/iocore/cache/CacheVC.h index bfd2d39b225..601c3f9ea04 100644 --- a/src/iocore/cache/CacheVC.h +++ b/src/iocore/cache/CacheVC.h @@ -268,7 +268,6 @@ struct CacheVC : public CacheVConnection { ink_hrtime start_time; int op_type; // Index into the metrics array for this operation, rather than a CacheOpType (fewer casts) int recursive; - int read_recursive; int closed; uint64_t seek_to; // pread offset int64_t offset; // offset into 'blocks' of data to write diff --git a/src/iocore/cache/StripeSM.h b/src/iocore/cache/StripeSM.h index ca6d0a6334c..a46b0ab5df2 100644 --- a/src/iocore/cache/StripeSM.h +++ b/src/iocore/cache/StripeSM.h @@ -270,8 +270,8 @@ class StripeSM : public Continuation, public Stripe return this->_preserved_dirs.evac_bucket_valid(bucket); } - DLL - get_evac_bucket(off_t bucket) const + DLL & + get_evac_bucket(off_t bucket) { return this->_preserved_dirs.evacuate[bucket]; } diff --git a/src/iocore/cache/unit_tests/test_Stripe.cc b/src/iocore/cache/unit_tests/test_Stripe.cc index 164936f02f1..5306ac6ee4d 100644 --- a/src/iocore/cache/unit_tests/test_Stripe.cc +++ b/src/iocore/cache/unit_tests/test_Stripe.cc @@ -397,3 +397,33 @@ TEST_CASE("aggWrite behavior with f.evacuator set") delete[] source; } + +TEST_CASE("get_evac_bucket returns a mutable reference") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + + EvacuationBlock b1; + b1.init = 0; + b1.readers = 0; + b1.earliest_evacuator = nullptr; + b1.evac_frags.link.next = nullptr; + + EvacuationBlock b2; + b2.init = 0; + b2.readers = 0; + b2.earliest_evacuator = nullptr; + b2.evac_frags.link.next = nullptr; + + REQUIRE(stripe.evac_bucket_valid(0)); + REQUIRE(stripe.get_evac_bucket(0).empty()); + + stripe.get_evac_bucket(0).push(&b1); + CHECK_FALSE(stripe.get_evac_bucket(0).empty()); + CHECK(stripe.get_evac_bucket(0).head == &b1); + + stripe.get_evac_bucket(0).push(&b2); + CHECK(stripe.get_evac_bucket(0).head == &b2); + CHECK(stripe.get_evac_bucket(0).head->link.next == &b1); +} diff --git a/src/iocore/dns/CMakeLists.txt b/src/iocore/dns/CMakeLists.txt index f22cad29c53..2a924c431f6 100644 --- a/src/iocore/dns/CMakeLists.txt +++ b/src/iocore/dns/CMakeLists.txt @@ -15,7 +15,7 @@ # ####################### -add_library(inkdns STATIC DNS.cc DNSConnection.cc DNSEventIO.cc SplitDNS.cc) +add_library(inkdns STATIC DNS.cc DNSConnection.cc DNSEventIO.cc HostEnt.cc SplitDNS.cc) add_library(ts::inkdns ALIAS inkdns) target_include_directories( @@ -32,3 +32,7 @@ target_link_libraries( ) clang_tidy_check(inkdns) + +if(BUILD_TESTING) + add_subdirectory(unit_tests) +endif() diff --git a/src/iocore/dns/DNS.cc b/src/iocore/dns/DNS.cc index 09c0e0f8cf7..4c6cecdcca3 100644 --- a/src/iocore/dns/DNS.cc +++ b/src/iocore/dns/DNS.cc @@ -92,9 +92,7 @@ is_addr_query(int qtype) DNSProcessor dnsProcessor; ClassAllocator dnsEntryAllocator("dnsEntryAllocator"); -// Users are expected to free these entries in short order! -// We could page align this buffer to enable page flipping for recv... -ClassAllocator dnsBufAllocator("dnsBufAllocator", 2); +extern ClassAllocator dnsBufAllocator; // // Function Prototypes @@ -151,12 +149,6 @@ HostEnt::isNameError() return get_rcode(this) == NXDOMAIN; } -void -HostEnt::free() -{ - dnsBufAllocator.free(this); -} - size_t make_ipv4_ptr(in_addr_t addr, char *buffer) { @@ -883,25 +875,23 @@ DNSHandler::recv_dns(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) dnsc->tcp_data.buf_ptr = make_ptr(dnsBufAllocator.alloc()); } if (dnsc->tcp_data.total_length == 0) { - // see if TS gets a two-byte size - uint16_t tmp = 0; - res = dnsc->sock.recv(&tmp, sizeof(tmp), MSG_PEEK); - if (res == -EAGAIN || res == 1) { - break; - } - if (res <= 0) { - goto Lerror; - } - // reading total size - res = dnsc->sock.recv(&(dnsc->tcp_data.total_length), sizeof(dnsc->tcp_data.total_length), 0); + // Read the 2-byte length prefix incrementally + res = dnsc->sock.recv(dnsc->tcp_data.length_buf + dnsc->tcp_data.length_read, + sizeof(dnsc->tcp_data.length_buf) - dnsc->tcp_data.length_read, 0); if (res == -EAGAIN) { break; } if (res <= 0) { goto Lerror; } - dnsc->tcp_data.total_length = ntohs(dnsc->tcp_data.total_length); - if (res != sizeof(dnsc->tcp_data.total_length)) { + dnsc->tcp_data.length_read += res; + if (dnsc->tcp_data.length_read < sizeof(dnsc->tcp_data.length_buf)) { + continue; + } + uint16_t net_length; + memcpy(&net_length, dnsc->tcp_data.length_buf, sizeof(net_length)); + dnsc->tcp_data.total_length = ntohs(net_length); + if (dnsc->tcp_data.total_length == 0) { goto Lerror; } } @@ -1721,8 +1711,11 @@ dns_process(DNSHandler *handler, HostEnt *buf, int len) unsigned char *here = reinterpret_cast(buf->buf) + HFIXEDSZ; if (e->qtype == T_SRV) { for (int ctr = ntohs(h->qdcount); ctr > 0; ctr--) { - int strlen = dn_skipname(here, eom); - here += strlen + QFIXEDSZ; + int strlen = dn_skipname(here, eom); + if (strlen < 0 || static_cast(eom - here) < static_cast(strlen) + QFIXEDSZ) { + goto Lerror; + } + here += strlen + QFIXEDSZ; } } // @@ -1737,6 +1730,10 @@ dns_process(DNSHandler *handler, HostEnt *buf, int len) break; } cp += n; + if (static_cast(eom - cp) < RRFIXEDSZ) { + ++error; + break; + } short int type; NS_GET16(type, cp); cp += NS_INT16SZ; // NS_GET16(cls, cp); @@ -1745,6 +1742,10 @@ dns_process(DNSHandler *handler, HostEnt *buf, int len) buf->ttl = temp_ttl; } NS_GET16(n, cp); + if (n > eom - cp) { + ++error; + break; + } // // Decode cname @@ -1814,13 +1815,26 @@ dns_process(DNSHandler *handler, HostEnt *buf, int len) if (buf->srv_hosts.hosts.size() >= hostdb_round_robin_max_count) { break; } - cp = here; /* hack */ - int strlen = dn_skipname(cp, eom); - cp += strlen; + cp = here; /* hack */ + int strlen = dn_skipname(cp, eom); + if (strlen < 0) { + ++error; + break; + } + cp += strlen; + if (static_cast(eom - cp) < SRV_FIXEDSZ) { + ++error; + break; + } const unsigned char *srv_off = cp; cp += SRV_FIXEDSZ; - cp += dn_skipname(cp, eom); - here = cp; /* hack */ + int srv_namelen = dn_skipname(cp, eom); + if (srv_namelen < 0) { + ++error; + break; + } + cp += srv_namelen; + here = cp; /* hack */ SRV srv; diff --git a/src/iocore/dns/HostEnt.cc b/src/iocore/dns/HostEnt.cc new file mode 100644 index 00000000000..c799369e1d2 --- /dev/null +++ b/src/iocore/dns/HostEnt.cc @@ -0,0 +1,33 @@ +/** @file + + HostEnt allocator and free hook. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "iocore/dns/DNSProcessor.h" +#include "tscore/Allocator.h" + +ClassAllocator dnsBufAllocator("dnsBufAllocator", 2); + +void +HostEnt::free() +{ + dnsBufAllocator.free(this); +} diff --git a/src/iocore/dns/P_DNSConnection.h b/src/iocore/dns/P_DNSConnection.h index f2a88c1aad0..14473797221 100644 --- a/src/iocore/dns/P_DNSConnection.h +++ b/src/iocore/dns/P_DNSConnection.h @@ -92,15 +92,21 @@ struct DNSConnection { /// TCPData structure is to track the reading progress of a TCP connection struct TCPData { - Ptr buf_ptr; - unsigned short total_length = 0; - unsigned short done_reading = 0; + Ptr buf_ptr; + // Staging for the 2-byte TCP length prefix + unsigned char length_buf[2] = {0, 0}; + unsigned short length_read = 0; + unsigned short total_length = 0; + unsigned short done_reading = 0; void reset() { buf_ptr.clear(); - total_length = 0; - done_reading = 0; + length_buf[0] = 0; + length_buf[1] = 0; + length_read = 0; + total_length = 0; + done_reading = 0; } } tcp_data; diff --git a/src/iocore/dns/unit_tests/CMakeLists.txt b/src/iocore/dns/unit_tests/CMakeLists.txt new file mode 100644 index 00000000000..602eb638dd0 --- /dev/null +++ b/src/iocore/dns/unit_tests/CMakeLists.txt @@ -0,0 +1,23 @@ +###################### +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor license +# agreements. See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# +###################### + +# Compile HostEnt.cc directly into the test rather than linking libinkdns, +# which would drag in libproxy/libhttp via inkdns's PUBLIC link deps. +add_executable(test_HostEnt test_HostEnt.cc ../HostEnt.cc) +target_include_directories(test_HostEnt PRIVATE "${PROJECT_SOURCE_DIR}/include") +target_link_libraries(test_HostEnt PRIVATE Catch2::Catch2WithMain ts::tscore ts::tsutil ts::inkevent) +add_catch2_test(NAME test_dns_HostEnt COMMAND $) diff --git a/src/iocore/dns/unit_tests/test_HostEnt.cc b/src/iocore/dns/unit_tests/test_HostEnt.cc new file mode 100644 index 00000000000..e6923e1e9d3 --- /dev/null +++ b/src/iocore/dns/unit_tests/test_HostEnt.cc @@ -0,0 +1,102 @@ +/** @file + + Unit tests for HostEnt allocator lifecycle. + + Exercises the same dnsBufAllocator that DNS.cc uses in production. With the + std::vector in HostEnt::srv_hosts.hosts, any path that returns a HostEnt + to the freelist without running ~vector() leaks the vector's heap + storage. Run under ASan/LSan: the leak shows up as a sanitizer report and + fails the test. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include "iocore/dns/DNSProcessor.h" +#include "iocore/dns/SRV.h" +#include "tscore/Allocator.h" + +extern ClassAllocator dnsBufAllocator; + +namespace +{ +void +fill_srv_hosts(HostEnt *e, int count) +{ + for (int i = 0; i < count; ++i) { + SRV srv{}; + srv.priority = static_cast(i); + srv.host_len = 1; + e->srv_hosts.hosts.push_back(srv); + e->srv_hosts.srv_hosts_length += srv.host_len; + } +} +} // namespace + +TEST_CASE("HostEnt SRV vector is released when HostEnt is freed", "[dns][hostent]") +{ + SECTION("single allocation, push then free") + { + HostEnt *e = dnsBufAllocator.alloc(); + REQUIRE(e != nullptr); + REQUIRE(e->srv_hosts.hosts.empty()); + REQUIRE(e->srv_hosts.srv_hosts_length == 0); + + fill_srv_hosts(e, 16); + REQUIRE(e->srv_hosts.hosts.size() == 16); + REQUIRE(e->srv_hosts.hosts.capacity() >= 16); + + e->free(); + // LSan flags any leaked vector heap from the slot above. + } + + SECTION("repeated alloc/fill/free cycle exercises freelist reuse") + { + for (int iter = 0; iter < 8; ++iter) { + HostEnt *e = dnsBufAllocator.alloc(); + REQUIRE(e != nullptr); + REQUIRE(e->srv_hosts.hosts.empty()); + REQUIRE(e->srv_hosts.srv_hosts_length == 0); + + fill_srv_hosts(e, 32); + REQUIRE(e->srv_hosts.hosts.size() == 32); + + e->free(); + } + } + + SECTION("multiple live allocations freed in reverse order") + { + constexpr int kCount = 4; + std::vector live; + live.reserve(kCount); + + for (int i = 0; i < kCount; ++i) { + HostEnt *e = dnsBufAllocator.alloc(); + REQUIRE(e != nullptr); + fill_srv_hosts(e, 8 + i); + live.push_back(e); + } + + for (auto it = live.rbegin(); it != live.rend(); ++it) { + (*it)->free(); + } + } +} diff --git a/src/iocore/eventsystem/IOBuffer.cc b/src/iocore/eventsystem/IOBuffer.cc index 246d98b5d42..76357c0bf61 100644 --- a/src/iocore/eventsystem/IOBuffer.cc +++ b/src/iocore/eventsystem/IOBuffer.cc @@ -380,10 +380,13 @@ IOBufferBlock::set_internal(void *b, int64_t len, int64_t asize_index) void IOBufferBlock::set(IOBufferData *d, int64_t len, int64_t offset) { - data = d; - _start = buf() + offset; - _end = _start + len; - _buf_end = buf() + d->block_size(); + data = d; + const int64_t block_sz = d->block_size(); + const int64_t safe_offset = std::clamp(offset, 0, block_sz); + const int64_t safe_len = std::clamp(len, 0, block_sz - safe_offset); + _start = buf() + safe_offset; + _buf_end = buf() + block_sz; + _end = _start + safe_len; } ////////////////////////////////////////////////////////////////// @@ -517,7 +520,7 @@ IOBufferReader::is_read_avail_more_than(int64_t size) void IOBufferReader::consume(int64_t n) { - ink_assert(read_avail() >= n); + ink_release_assert(n == 0 || is_read_avail_more_than(n - 1)); start_offset += n; if (size_limit != INT64_MAX) { size_limit -= n; @@ -851,6 +854,7 @@ MIOBuffer::fill(int64_t len) len -= f; if (len > 0) { _writer = _writer->next; + ink_release_assert(_writer); } f = _writer->write_avail(); } diff --git a/src/iocore/eventsystem/unit_tests/test_IOBuffer.cc b/src/iocore/eventsystem/unit_tests/test_IOBuffer.cc index 2c25bef6585..3f66069eb7d 100644 --- a/src/iocore/eventsystem/unit_tests/test_IOBuffer.cc +++ b/src/iocore/eventsystem/unit_tests/test_IOBuffer.cc @@ -332,6 +332,96 @@ TEST_CASE("MIOBuffer", "[iocore]") } } +TEST_CASE("MIOBuffer accessors on a writerless buffer", "[iocore]") +{ + // After dealloc(), MIOBuffer::_writer is null and first_write_block() + // returns nullptr. The chain accessors must mirror the null-safe + // behaviour that buf() already provides instead of dereferencing the + // missing block. + MIOBuffer *miob = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + miob->dealloc(); + + REQUIRE(miob->first_write_block() == nullptr); + CHECK(miob->buf() == nullptr); + CHECK(miob->buf_end() == nullptr); + CHECK(miob->start() == nullptr); + CHECK(miob->end() == nullptr); + + free_MIOBuffer(miob); +} + +TEST_CASE("IOBufferBlock::set clamps _end to block storage", "[iocore]") +{ + // IOBufferBlock::set must not let _end land beyond _buf_end when the + // caller passes a length larger than the underlying block. Otherwise + // read_avail() reports more bytes than the block actually holds and + // any subsequent copy reads out-of-bounds memory. + Ptr data{new_IOBufferData(BUFFER_SIZE_INDEX_512)}; + Ptr block{new_IOBufferBlock()}; + + const int64_t block_sz = data->block_size(); + + SECTION("len within the block") + { + block->set(data.get(), block_sz / 2, 0); + CHECK(block->read_avail() == block_sz / 2); + CHECK(block->end() <= block->buf_end()); + } + + SECTION("len exactly equal to the block") + { + block->set(data.get(), block_sz, 0); + CHECK(block->read_avail() == block_sz); + CHECK(block->end() == block->buf_end()); + } + + SECTION("len larger than the block is clamped") + { + block->set(data.get(), block_sz * 4, 0); + CHECK(block->read_avail() == block_sz); + CHECK(block->end() == block->buf_end()); + } + + SECTION("non-zero offset is honoured by the clamp") + { + const int64_t offset = block_sz / 4; + block->set(data.get(), block_sz, offset); + CHECK(block->read_avail() == block_sz - offset); + CHECK(block->end() == block->buf_end()); + } + + SECTION("offset beyond the block produces an empty slice") + { + block->set(data.get(), block_sz, block_sz * 2); + CHECK(block->read_avail() == 0); + CHECK(block->start() == block->buf_end()); + CHECK(block->end() == block->buf_end()); + } + + SECTION("offset exactly at the block end produces an empty slice") + { + block->set(data.get(), block_sz, block_sz); + CHECK(block->read_avail() == 0); + CHECK(block->start() == block->buf_end()); + CHECK(block->end() == block->buf_end()); + } + + SECTION("negative offset is treated as zero") + { + block->set(data.get(), block_sz, -32); + CHECK(block->read_avail() == block_sz); + CHECK(block->start() == block->buf()); + CHECK(block->end() == block->buf_end()); + } + + SECTION("negative len is treated as zero") + { + block->set(data.get(), -10, 0); + CHECK(block->read_avail() == 0); + CHECK(block->start() == block->end()); + } +} + TEST_CASE("block size parser", "[iocore]") { int chunk_sizes[DEFAULT_BUFFER_SIZES] = {0}; diff --git a/src/iocore/hostdb/HostDB.cc b/src/iocore/hostdb/HostDB.cc index 780de2a5d60..6abda699462 100644 --- a/src/iocore/hostdb/HostDB.cc +++ b/src/iocore/hostdb/HostDB.cc @@ -25,6 +25,7 @@ #include "swoc/swoc_file.h" #include "tscore/Regression.h" #include "tsutil/ts_bw_format.h" +#include "tsutil/LocalBuffer.h" #include "P_HostDB.h" // Gross @@ -948,8 +949,9 @@ HostDBContinuation::dnsEvent(int event, HostEnt *e) auto rr_info = r->rr_info(); // Fill in record type specific data. if (hash.is_srv()) { - char *pos = rr_info.rebind().end(); - SRV *q[valid_records]; + char *pos = rr_info.rebind().end(); + ts::LocalBuffer q_buf(valid_records); + SRV **q = q_buf.data(); ink_assert(valid_records <= static_cast(hostdb_round_robin_max_count)); for (int i = 0; i < valid_records; ++i) { q[i] = &e->srv_hosts.hosts[i]; @@ -1607,7 +1609,8 @@ HostDBRecord::select_best_srv(char *target, InkRand *rand, ts_time now, ts_secon HostDBInfo *result = nullptr; auto rr = this->rr_info(); // Array of live targets, sized by @a live_n - HostDBInfo *live[rr.count()]; + ts::LocalBuffer live_buf(rr.count()); + HostDBInfo **live = live_buf.data(); for (auto &rr_target : rr) { // skip down targets. if (rr_target.is_down(now, fail_window)) { diff --git a/src/iocore/hostdb/HostDBInfo.cc b/src/iocore/hostdb/HostDBInfo.cc index 1bb4126359b..aab25efd088 100644 --- a/src/iocore/hostdb/HostDBInfo.cc +++ b/src/iocore/hostdb/HostDBInfo.cc @@ -88,7 +88,9 @@ HostDBInfo::assign(SRV const *srv, char const *name) -> self_type & data.srv.key = srv->key; // Danger! This offset calculation assumes that name and this are with 16-bits of each // other. This invariant must be upheld for every caller of this function. - data.srv.srv_offset = name - reinterpret_cast(this); + auto offset = name - reinterpret_cast(this); + ink_release_assert(offset >= 0 && offset <= UINT16_MAX); + data.srv.srv_offset = static_cast(offset); return *this; } diff --git a/src/iocore/net/CMakeLists.txt b/src/iocore/net/CMakeLists.txt index cb0d8e7957e..6dc65f024c3 100644 --- a/src/iocore/net/CMakeLists.txt +++ b/src/iocore/net/CMakeLists.txt @@ -135,8 +135,16 @@ endif() if(BUILD_TESTING) # libinknet_stub.cc is need because GNU ld is sensitive to the order of static libraries on the command line, and we have a cyclic dependency between inknet and proxy add_executable( - test_net libinknet_stub.cc NetVCTest.cc unit_tests/test_ProxyProtocol.cc unit_tests/test_SSLSNIConfig.cc - unit_tests/test_YamlSNIConfig.cc unit_tests/unit_test_main.cc + test_net + libinknet_stub.cc + NetVCTest.cc + unit_tests/test_ProxyProtocol.cc + unit_tests/test_SSLCertLookup.cc + unit_tests/test_SSLNetVConnectionAsyncEp.cc + unit_tests/test_SSLSNIConfig.cc + unit_tests/test_YamlSNIConfig.cc + unit_tests/test_OCSPStapling.cc + unit_tests/unit_test_main.cc ) # Use link groups to solve circular dependency set(LINK_GROUP_LIBS diff --git a/src/iocore/net/NetVConnection.cc b/src/iocore/net/NetVConnection.cc index bacfc0e354d..b42868cc8aa 100644 --- a/src/iocore/net/NetVConnection.cc +++ b/src/iocore/net/NetVConnection.cc @@ -33,10 +33,19 @@ #include "iocore/net/NetVConnection.h" #include "iocore/eventsystem/IOBuffer.h" #include "tsutil/DbgCtl.h" +#include "tsutil/LocalBuffer.h" #include +#include namespace { + +// 1024 bytes should be more than enough in the vast majority of +// circumstances. proxy.config.proxy_protocol.max_header_size defaults +// to 109 bytes. If somehow this is exceeded, LocalBuffer will allocate +// space on the heap. +static constexpr size_t PROXY_PROTOCOL_LOCAL_BUFFER_SIZE = 1024; + DbgCtl dbg_ctl_ssl{"ssl"}; } // end anonymous namespace @@ -61,9 +70,9 @@ NetVConnection::has_proxy_protocol(IOBufferReader *reader, int max_header_size) return false; } - int bufsize = max_header_size; - char buf[bufsize]; - tv.assign(buf, reader->memcpy(buf, bufsize, 0)); + auto const bufsize = std::min(max_header_size, reader->read_avail()); + ts::LocalBuffer buf(static_cast(bufsize)); + tv.assign(buf.data(), reader->memcpy(buf.data(), bufsize, 0)); size_t len = proxy_protocol_parse(&this->pp_info, tv); diff --git a/src/iocore/net/P_SSLClientUtils.h b/src/iocore/net/P_SSLClientUtils.h index daf44b90c30..6be333ccac2 100644 --- a/src/iocore/net/P_SSLClientUtils.h +++ b/src/iocore/net/P_SSLClientUtils.h @@ -24,15 +24,19 @@ #include "P_SSLConfig.h" #include +#include // BoringSSL does not have this include file #if __has_include() #include #endif +class NetVConnection; + // Create and initialize a SSL client context. SSL_CTX *SSLInitClientContext(const struct SSLConfigParams *param); SSL_CTX *SSLCreateClientContext(const struct SSLConfigParams *params, const char *ca_bundle_file, const char *ca_bundle_path, const char *cert_path, const char *key_path); -int verify_callback(int preverify_ok, X509_STORE_CTX *ctx); +int verify_callback(int preverify_ok, X509_STORE_CTX *ctx); +bool validate_server_certificate_hostname(NetVConnection *netvc, std::string_view hostname); diff --git a/src/iocore/net/P_SSLNetVConnection.h b/src/iocore/net/P_SSLNetVConnection.h index ee1a04d9e26..28e0e99eb3b 100644 --- a/src/iocore/net/P_SSLNetVConnection.h +++ b/src/iocore/net/P_SSLNetVConnection.h @@ -106,6 +106,8 @@ class SSLNetVConnection : public UnixNetVConnection, { using super = UnixNetVConnection; ///< Parent type. + friend struct SSLNetVConnectionAsyncEpTestAccess; + public: void clear() override; void free_thread(EThread *t) override; diff --git a/src/iocore/net/ProxyProtocol.cc b/src/iocore/net/ProxyProtocol.cc index 1b475c96c58..51e67270d22 100644 --- a/src/iocore/net/ProxyProtocol.cc +++ b/src/iocore/net/ProxyProtocol.cc @@ -29,6 +29,7 @@ #include "swoc/TextView.h" #include "swoc/bwf_base.h" #include "tsutil/DbgCtl.h" +#include #include #include @@ -64,6 +65,19 @@ constexpr uint16_t PPv2_ADDR_LEN_UNIX = 108 + 108; const swoc::bwf::Spec ADDR_ONLY_FMT{"::a"}; +std::optional +proxy_protocol_v1_parse_port(swoc::TextView token) +{ + swoc::TextView parsed; + auto port = swoc::svtoi(token, &parsed, 10); + + if (parsed != token || port <= 0 || port > std::numeric_limits::max()) { + return std::nullopt; + } + + return static_cast(port); +} + DbgCtl dbg_ctl_proxyprotocol_v1{"proxyprotocol_v1"}; DbgCtl dbg_ctl_proxyprotocol_v2{"proxyprotocol_v2"}; DbgCtl dbg_ctl_proxyprotocol{"proxyprotocol"}; @@ -182,22 +196,22 @@ proxy_protocol_v1_parse(ProxyProtocol *pp_info, swoc::TextView hdr) return 0; } - // Next is the TCP source port represented as a decimal number in the range of [0..65535] inclusive. + // Next is the TCP source port represented as a decimal number in the range of [1..65535] inclusive. token = hdr.split_prefix_at(' '); if (0 == token.size()) { return 0; } Dbg(dbg_ctl_proxyprotocol_v1, "proxy_protov1_parse: [%.*s] = Source Port", static_cast(token.size()), token.data()); - in_port_t src_port = swoc::svtoi(token); - if (src_port == 0) { - Dbg(dbg_ctl_proxyprotocol_v1, "proxy_protov1_parse: src port [%d] token [%.*s] failed to parse", src_port, - static_cast(token.size()), token.data()); + auto src_port = proxy_protocol_v1_parse_port(token); + if (!src_port) { + Dbg(dbg_ctl_proxyprotocol_v1, "proxy_protov1_parse: src port token [%.*s] failed to parse", static_cast(token.size()), + token.data()); return 0; } - pp_info->src_addr.network_order_port() = htons(src_port); + pp_info->src_addr.network_order_port() = htons(*src_port); - // Next is the TCP destination port represented as a decimal number in the range of [0..65535] inclusive. + // Next is the TCP destination port represented as a decimal number in the range of [1..65535] inclusive. // Final trailer is CR LF so split at CR. token = hdr.split_prefix_at('\r'); if (0 == token.size() || token.find(0x20) != token.npos) { @@ -205,13 +219,13 @@ proxy_protocol_v1_parse(ProxyProtocol *pp_info, swoc::TextView hdr) } Dbg(dbg_ctl_proxyprotocol_v1, "proxy_protov1_parse: [%.*s] = Destination Port", static_cast(token.size()), token.data()); - in_port_t dst_port = swoc::svtoi(token); - if (dst_port == 0) { - Dbg(dbg_ctl_proxyprotocol_v1, "proxy_protov1_parse: dst port [%d] token [%.*s] failed to parse", dst_port, - static_cast(token.size()), token.data()); + auto dst_port = proxy_protocol_v1_parse_port(token); + if (!dst_port) { + Dbg(dbg_ctl_proxyprotocol_v1, "proxy_protov1_parse: dst port token [%.*s] failed to parse", static_cast(token.size()), + token.data()); return 0; } - pp_info->dst_addr.network_order_port() = htons(dst_port); + pp_info->dst_addr.network_order_port() = htons(*dst_port); pp_info->version = ProxyProtocolVersion::V1; diff --git a/src/iocore/net/SNIActionPerformer.cc b/src/iocore/net/SNIActionPerformer.cc index a5c04710eb1..1e12ecb6f01 100644 --- a/src/iocore/net/SNIActionPerformer.cc +++ b/src/iocore/net/SNIActionPerformer.cc @@ -200,8 +200,10 @@ TunnelDestination::SNIAction(SSL &ssl, const Context &ctx) const auto fixed_dst{destination}; // Apply mapping functions to get the final destination. for (auto fnArrIndex : fnArrIndexes) { + bool has_dynamic_port = false; // Dispatch to the correct tunnel destination port function. - fixed_dst = fix_destination[fnArrIndex](fixed_dst, var_start_pos, ctx, ssl_netvc, port_is_dynamic); + fixed_dst = fix_destination[fnArrIndex](fixed_dst, var_start_pos, ctx, ssl_netvc, has_dynamic_port); + port_is_dynamic |= has_dynamic_port; } tuns->set_tunnel_destination(fixed_dst, type, port_is_dynamic, tunnel_prewarm); Dbg(dbg_ctl_ssl_sni, "Destination now is [%s], configured [%s], fqdn [%s]", fixed_dst.c_str(), destination.c_str(), servername); @@ -421,10 +423,15 @@ SNI_IpAllow::SNIAction(SSL &ssl, ActionItem::Context const & /* ctx ATS_UNUSED * break; } else if (IpAllow::Subject::PROXY == IpAllow::subjects[i] && ssl_vc->get_proxy_protocol_version() != ProxyProtocolVersion::UNDEFINED) { - client_ip = ssl_vc->get_proxy_protocol_src_addr(); - break; + if (sockaddr const *proxy_ip = ssl_vc->get_proxy_protocol_src_addr(); proxy_ip != nullptr) { + client_ip = proxy_ip; + break; + } } } + if (client_ip == nullptr) { + client_ip = ssl_vc->get_remote_addr(); + } swoc::IPAddr ip = swoc::IPAddr(client_ip); // check the allowed ips @@ -439,10 +446,10 @@ SNI_IpAllow::SNIAction(SSL &ssl, ActionItem::Context const & /* ctx ATS_UNUSED * } bool -SNI_IpAllow::TestClientSNIAction(char const * /* servrername ATS_UNUSED */, IpEndpoint const &ep, +SNI_IpAllow::TestClientSNIAction(char const * /* servrername ATS_UNUSED */, IpEndpoint const & /* ep ATS_UNUSED */, int & /* policy ATS_UNUSED */) const { - return ip_addrs.contains(swoc::IPAddr(ep)); + return !ip_addrs.empty(); } int @@ -450,7 +457,7 @@ OutboundSNIPolicy::SNIAction(SSL &ssl, const Context & /* ctx ATS_UNUSED */) con { if (!policy.empty()) { if (auto snis = TLSSNISupport::getInstance(&ssl)) { - snis->hints_from_sni.outbound_sni_policy = policy; + snis->hints_from_sni.outbound_sni_policy.emplace(policy); } } return SSL_TLSEXT_ERR_OK; diff --git a/src/iocore/net/SSLClientUtils.cc b/src/iocore/net/SSLClientUtils.cc index 453b971dd79..b675f8c560e 100644 --- a/src/iocore/net/SSLClientUtils.cc +++ b/src/iocore/net/SSLClientUtils.cc @@ -25,6 +25,7 @@ #include "P_TLSKeyLogger.h" #include "SSLSessionCache.h" #include "TLSCertCompression.h" +#include "iocore/net/TLSBasicSupport.h" #include "iocore/net/YamlSNIConfig.h" #include "iocore/net/SSLDiags.h" #include "tscore/ink_config.h" @@ -105,17 +106,18 @@ verify_callback(int signature_ok, X509_STORE_CTX *ctx) bool check_name = static_cast(netvc->options.verifyServerProperties) & static_cast(YamlSNIConfig::Property::NAME_MASK); if (check_name) { - char *matched_name = nullptr; - unsigned char *sni_name; - char buff[INET6_ADDRSTRLEN]; + char *matched_name = nullptr; + std::string_view sni_name; + char buff[INET6_ADDRSTRLEN]; if (netvc->options.sni_servername) { - sni_name = reinterpret_cast(netvc->options.sni_servername.get()); + sni_name = netvc->options.sni_servername.get(); } else { - sni_name = reinterpret_cast(buff); ats_ip_ntop(netvc->get_effective_remote_addr(), buff, INET6_ADDRSTRLEN); + sni_name = buff; } if (validate_hostname(cert, sni_name, false, &matched_name)) { - Dbg(dbg_ctl_ssl_verify, "Hostname %s verified OK, matched %s", sni_name, matched_name); + Dbg(dbg_ctl_ssl_verify, "Hostname %.*s verified OK, matched %s", static_cast(sni_name.length()), sni_name.data(), + matched_name); ats_free(matched_name); } else { // Name validation failed // Get the server address if we did't already compute it @@ -123,8 +125,8 @@ verify_callback(int signature_ok, X509_STORE_CTX *ctx) ats_ip_ntop(netvc->get_effective_remote_addr(), buff, INET6_ADDRSTRLEN); } // If we got here the verification failed - Warning("SNI (%s) not in certificate. Action=%s server=%s(%s)", sni_name, enforce_mode ? "Terminate" : "Continue", - netvc->options.ssl_servername.get(), buff); + Warning("SNI (%.*s) not in certificate. Action=%s server=%s(%s)", static_cast(sni_name.length()), sni_name.data(), + enforce_mode ? "Terminate" : "Continue", netvc->options.ssl_servername.get(), buff); return !enforce_mode; } } @@ -153,6 +155,55 @@ verify_callback(int signature_ok, X509_STORE_CTX *ctx) return true; } +bool +validate_server_certificate_hostname(NetVConnection *netvc, std::string_view hostname) +{ + if (netvc == nullptr || hostname.empty() || netvc->options.verifyServerPolicy == YamlSNIConfig::Policy::DISABLED) { + return true; + } + + auto *tls = netvc->get_service(); + auto *ssl = tls != nullptr ? tls->get_tls_handle() : nullptr; + if (ssl == nullptr) { + return true; + } + + bool check_name = + static_cast(netvc->options.verifyServerProperties) & static_cast(YamlSNIConfig::Property::NAME_MASK); + if (!check_name) { + return true; + } + + char *matched_name = nullptr; + bool const enforce_mode = netvc->options.verifyServerPolicy == YamlSNIConfig::Policy::ENFORCED; + bool verified = false; +#ifdef OPENSSL_IS_OPENSSL3 + X509 *cert = SSL_get1_peer_certificate(ssl); +#else + X509 *cert = SSL_get_peer_certificate(ssl); +#endif + + if (cert != nullptr) { + verified = validate_hostname(cert, hostname, false, &matched_name); + X509_free(cert); + } + + if (verified) { + Dbg(dbg_ctl_ssl_verify, "Hostname %.*s verified OK for session reuse, matched %s", static_cast(hostname.length()), + hostname.data(), matched_name != nullptr ? matched_name : ""); + ats_free(matched_name); + return true; + } + + char buff[INET6_ADDRSTRLEN]; + const char *server_name = netvc->options.ssl_servername ? netvc->options.ssl_servername.get() : ""; + ats_ip_ntop(netvc->get_effective_remote_addr(), buff, INET6_ADDRSTRLEN); + Warning("Origin hostname (%.*s) not in certificate. Action=%s server=%s(%s)", static_cast(hostname.length()), + hostname.data(), enforce_mode ? "Terminate" : "Continue", server_name, buff); + + return !enforce_mode; +} + static int ssl_client_cert_callback(SSL *ssl, void * /*arg*/) { diff --git a/src/iocore/net/SSLNetVConnection.cc b/src/iocore/net/SSLNetVConnection.cc index fcee0975947..4165cf23dfb 100644 --- a/src/iocore/net/SSLNetVConnection.cc +++ b/src/iocore/net/SSLNetVConnection.cc @@ -389,15 +389,22 @@ SSLNetVConnection::read_raw_data() if (this->has_proxy_protocol(buffer, &r)) { Dbg(dbg_ctl_proxyprotocol, "ssl has proxy protocol header"); if (dbg_ctl_proxyprotocol.on()) { - IpEndpoint src; - src.sa = *(this->get_proxy_protocol_src_addr()); - IpEndpoint dst; - dst.sa = *(this->get_proxy_protocol_dst_addr()); - ip_port_text_buffer src_ipb, dst_ipb; - ats_ip_nptop(&src, src_ipb, sizeof(src_ipb)); - ats_ip_nptop(&dst, dst_ipb, sizeof(dst_ipb)); - DbgPrint(dbg_ctl_proxyprotocol, "ssl proxy protocol v%d header parsed: src=[%s] dst=[%s]", - static_cast(this->get_proxy_protocol_version()), src_ipb, dst_ipb); + sockaddr const *src_addr = this->get_proxy_protocol_src_addr(); + sockaddr const *dst_addr = this->get_proxy_protocol_dst_addr(); + if (src_addr != nullptr && dst_addr != nullptr) { + IpEndpoint src; + src.sa = *src_addr; + IpEndpoint dst; + dst.sa = *dst_addr; + ip_port_text_buffer src_ipb, dst_ipb; + ats_ip_nptop(&src, src_ipb, sizeof(src_ipb)); + ats_ip_nptop(&dst, dst_ipb, sizeof(dst_ipb)); + DbgPrint(dbg_ctl_proxyprotocol, "ssl proxy protocol v%d header parsed: src=[%s] dst=[%s]", + static_cast(this->get_proxy_protocol_version()), src_ipb, dst_ipb); + } else { + DbgPrint(dbg_ctl_proxyprotocol, "ssl proxy protocol v%d header parsed without address information", + static_cast(this->get_proxy_protocol_version())); + } } } else { Dbg(dbg_ctl_proxyprotocol, "proxy protocol was enabled, but Proxy Protocol header was not present"); @@ -884,6 +891,14 @@ SSLNetVConnection::SSLNetVConnection() void SSLNetVConnection::do_io_close(int lerrno) { + // Stop any async-handshake eventfd before the VC is closed. + // If WANT_ASYNC was registered the eventfd is wired into the poller with + // `this` as the EventIO target. Without this stop() the SSLNetVConnection + // can be freed while the eventfd still has a live epoll registration; when + // the OpenSSL async job completes the poller wakes on freed memory. + if (async_ep.fd >= 0) { + async_ep.stop(); + } if (this->ssl != nullptr) { if (get_context() == NET_VCONNECTION_OUT) { callHooks(TS_EVENT_VCONN_OUTBOUND_CLOSE); @@ -934,7 +949,7 @@ SSLNetVConnection::do_io_close(int lerrno) void SSLNetVConnection::do_io_shutdown(ShutdownHowTo_t howto) { - if (get_tunnel_type() == SNIRoutingType::BLIND) { + if (ssl == nullptr || get_tunnel_type() == SNIRoutingType::BLIND) { // we don't have TLS layer control of blind tunnel UnixNetVConnection::do_io_shutdown(howto); return; @@ -993,6 +1008,15 @@ SSLNetVConnection::clear() // resetting here will decrement the ref-counter. client_sess.reset(); + // Stop the async-handshake eventfd before SSL_free. The + // eventfd is owned by the SSL object, so SSL_free closes it; deregistering + // first keeps the EPOLL_CTL_DEL operating on a valid, owned fd. clear() runs + // on every free path through free_thread(), so it is the backstop in case + // do_io_close() did not already stop the eventfd. + if (async_ep.fd >= 0) { + async_ep.stop(); + } + if (ssl != nullptr) { // clear() runs from free() once per VC recycle, so this is the single chokepoint where a TLS // connection's SSL object is torn down -- count it as one connection close here. Blind-tunnel @@ -2001,8 +2025,12 @@ SSLNetVConnection::_lookupContextByIP() } SSLCertContext *cc = nullptr; - if (this->get_is_proxy_protocol() && this->get_proxy_protocol_version() != ProxyProtocolVersion::UNDEFINED) { - ip.sa = *(this->get_proxy_protocol_dst_addr()); + sockaddr const *proxy_protocol_dst_addr = + this->get_is_proxy_protocol() && this->get_proxy_protocol_version() != ProxyProtocolVersion::UNDEFINED ? + this->get_proxy_protocol_dst_addr() : + nullptr; + if (proxy_protocol_dst_addr != nullptr) { + ip.sa = *proxy_protocol_dst_addr; ip_port_text_buffer ipb1; ats_ip_nptop(&ip, ipb1, sizeof(ipb1)); cc = lookup->find(ip); diff --git a/src/iocore/net/SSLNextProtocolAccept.cc b/src/iocore/net/SSLNextProtocolAccept.cc index b66b60dcd6b..e49f9935a2d 100644 --- a/src/iocore/net/SSLNextProtocolAccept.cc +++ b/src/iocore/net/SSLNextProtocolAccept.cc @@ -118,6 +118,8 @@ struct SSLNextProtocolTrampoline : public Continuation { } if (endpoint_cont) { + netvc->cancel_inactivity_timeout(); + // disable read io, send events to endpoint netvc->do_io_read(endpoint_cont, 0, nullptr); diff --git a/src/iocore/net/SSLSNIConfig.cc b/src/iocore/net/SSLSNIConfig.cc index ce72e9d4bac..12ab464d483 100644 --- a/src/iocore/net/SSLSNIConfig.cc +++ b/src/iocore/net/SSLSNIConfig.cc @@ -111,9 +111,9 @@ const NextHopProperty * SNIConfigParams::get_property_config(const std::string &servername) const { const NextHopProperty *nps = nullptr; + RegexMatches matches; for (auto &&item : next_hop_list) { - if (item.match.exec(servername)) { - // Found a match + if (item.match.exec(servername, matches, RE_FULL_MATCH) >= 0) { nps = &item.prop; break; } @@ -237,12 +237,11 @@ SNIConfigParams::get(std::string_view servername, in_port_t dest_incoming_port) if (retval.match.empty() && servername.length() == 0) { return {&retval.actions, {}}; - } else if (retval.match.exec(servername, matches) >= 0) { + } else if (retval.match.exec(servername, matches, RE_FULL_MATCH) >= 0) { if (!is_port_in_the_ranges(retval.inbound_port_ranges, dest_incoming_port)) { continue; } if (matches.size() == 1) { - // full match return {&retval.actions, {}}; } diff --git a/src/iocore/net/Server.cc b/src/iocore/net/Server.cc index fff587df2b3..32a8c0f7ae3 100644 --- a/src/iocore/net/Server.cc +++ b/src/iocore/net/Server.cc @@ -126,9 +126,14 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) } if (ats_is_unix(&accept_addr)) { - if (chmod(accept_addr.sun.sun_path, 0777) < 0) { + if (chmod(accept_addr.sun.sun_path, opt.unix_perm) < 0) { goto Lerror; } + if (opt.unix_uid != static_cast(-1) || opt.unix_gid != static_cast(-1)) { + if (chown(accept_addr.sun.sun_path, opt.unix_uid, opt.unix_gid) < 0) { + goto Lerror; + } + } } if ((res = safe_listen(sock.get_fd(), get_listen_backlog())) < 0) { diff --git a/src/iocore/net/unit_tests/sni_conf_test.yaml b/src/iocore/net/unit_tests/sni_conf_test.yaml index 487daf8070c..f0c0c789a47 100644 --- a/src/iocore/net/unit_tests/sni_conf_test.yaml +++ b/src/iocore/net/unit_tests/sni_conf_test.yaml @@ -54,3 +54,11 @@ sni: - fqdn: tickets.com ssl_ticket_enabled: 1 ssl_ticket_number: 3 + +# test ip_allow filtering +- fqdn: ipallow.example.com + ip_allow: 192.168.1.0/24,10.0.0.1 + +# test entry with no ip_allow (only protocol settings) +- fqdn: noipallow.example.com + http2: off diff --git a/src/iocore/net/unit_tests/test_OCSPStapling.cc b/src/iocore/net/unit_tests/test_OCSPStapling.cc new file mode 100644 index 00000000000..dbeb71e6cb1 --- /dev/null +++ b/src/iocore/net/unit_tests/test_OCSPStapling.cc @@ -0,0 +1,105 @@ +/** @file + + Catch based unit tests for OCSP stapling + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#ifndef LIBINKNET_UNIT_TEST_DIR +#error please set LIBINKNET_UNIT_TEST_DIR +#endif + +#define _STR(s) #s +#define _XSTR(s) _STR(s) + +#include "../P_OCSPStapling.h" + +#include + +#include +#include +#include + +#include +#include + +namespace +{ +constexpr char OCSP_TEST_SSL_DIR[] = _XSTR(LIBINKNET_UNIT_TEST_DIR) "/../../../../tests/gold_tests/tls/ssl"; + +struct BioDeleter { + void + operator()(BIO *bio) const + { + BIO_free(bio); + } +}; + +struct SslCtxDeleter { + void + operator()(SSL_CTX *ctx) const + { + SSL_CTX_free(ctx); + } +}; + +struct X509Deleter { + void + operator()(X509 *cert) const + { + X509_free(cert); + } +}; + +using BioPtr = std::unique_ptr; +using SslCtxPtr = std::unique_ptr; +using X509Ptr = std::unique_ptr; + +X509Ptr +load_cert(std::string const &path) +{ + BioPtr bio{BIO_new_file(path.c_str(), "r")}; + REQUIRE(bio != nullptr); + + X509Ptr cert{PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)}; + REQUIRE(cert != nullptr); + return cert; +} + +} // end anonymous namespace + +TEST_CASE("OCSP stapling keeps SSL_CTX certificate map after later init failure", "[ssl][ocsp]") +{ + ssl_stapling_ex_init(); + + SslCtxPtr ctx{SSL_CTX_new(TLS_method())}; + REQUIRE(ctx != nullptr); + + auto issuer = load_cert(std::string{OCSP_TEST_SSL_DIR} + "/ca.ocsp.pem"); + auto good = load_cert(std::string{OCSP_TEST_SSL_DIR} + "/server.ocsp.pem"); + auto bad = load_cert(std::string{OCSP_TEST_SSL_DIR} + "/signed-foo.pem"); + + REQUIRE(SSL_CTX_use_certificate(ctx.get(), good.get()) == 1); + + REQUIRE(SSL_CTX_add_extra_chain_cert(ctx.get(), issuer.get()) == 1); + issuer.release(); + + REQUIRE(ssl_stapling_init_cert(ctx.get(), good.get(), "server.ocsp.pem", nullptr)); + CHECK_FALSE(ssl_stapling_init_cert(ctx.get(), bad.get(), "signed-foo.pem", nullptr)); +} diff --git a/src/iocore/net/unit_tests/test_ProxyProtocol.cc b/src/iocore/net/unit_tests/test_ProxyProtocol.cc index cc62e63e82f..8815845ab64 100644 --- a/src/iocore/net/unit_tests/test_ProxyProtocol.cc +++ b/src/iocore/net/unit_tests/test_ProxyProtocol.cc @@ -64,6 +64,30 @@ TEST_CASE("PROXY Protocol v1 Parser", "[ProxyProtocol][ProxyProtocolv1]") CHECK(pp_info.dst_addr == dst_addr); } + SECTION("TCP port boundaries") + { + swoc::TextView raw_data = "PROXY TCP4 192.0.2.1 198.51.100.1 65535 65535\r\n"sv; + + ProxyProtocol pp_info; + REQUIRE(proxy_protocol_parse(&pp_info, raw_data) == raw_data.size()); + + REQUIRE(ats_ip_pton("192.0.2.1:65535", src_addr) == 0); + REQUIRE(ats_ip_pton("198.51.100.1:65535", dst_addr) == 0); + + CHECK(pp_info.version == ProxyProtocolVersion::V1); + CHECK(pp_info.src_addr == src_addr); + CHECK(pp_info.dst_addr == dst_addr); + + CHECK(proxy_protocol_parse(&pp_info, "PROXY TCP4 192.0.2.1 198.51.100.1 0 443\r\n"sv) == 0); + CHECK(proxy_protocol_parse(&pp_info, "PROXY TCP4 192.0.2.1 198.51.100.1 50000 0\r\n"sv) == 0); + CHECK(proxy_protocol_parse(&pp_info, "PROXY TCP4 192.0.2.1 198.51.100.1 65536 443\r\n"sv) == 0); + CHECK(proxy_protocol_parse(&pp_info, "PROXY TCP4 192.0.2.1 198.51.100.1 50000 65536\r\n"sv) == 0); + CHECK(proxy_protocol_parse(&pp_info, "PROXY TCP4 192.0.2.1 198.51.100.1 65537 443\r\n"sv) == 0); + CHECK(proxy_protocol_parse(&pp_info, "PROXY TCP4 192.0.2.1 198.51.100.1 50000 65537\r\n"sv) == 0); + CHECK(proxy_protocol_parse(&pp_info, "PROXY TCP4 192.0.2.1 198.51.100.1 131152 443\r\n"sv) == 0); + CHECK(proxy_protocol_parse(&pp_info, "PROXY TCP4 192.0.2.1 198.51.100.1 50000 131152\r\n"sv) == 0); + } + SECTION("UNKNOWN connection (short form)") { swoc::TextView raw_data = "PROXY UNKNOWN\r\n"sv; diff --git a/src/iocore/net/unit_tests/test_SSLCertLookup.cc b/src/iocore/net/unit_tests/test_SSLCertLookup.cc new file mode 100644 index 00000000000..42fddf0867e --- /dev/null +++ b/src/iocore/net/unit_tests/test_SSLCertLookup.cc @@ -0,0 +1,53 @@ +/** @file + + Catch based unit tests for SSLCertLookup + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "../P_SSLCertLookup.h" + +#include + +#include + +TEST_CASE("SSLCertLookup handles high-bit bytes while normalizing hostnames") +{ + constexpr auto HIGH_ORDER_BIT = static_cast(0x80); + + SSLCertLookup lookup; + auto *ctx = SSL_CTX_new(SSLv23_server_method()); + REQUIRE(ctx != nullptr); + + SSLCertContext context(ctx); + + std::string cert_name{"High"}; + cert_name.push_back(HIGH_ORDER_BIT); + cert_name.append(".Example.Com"); + + std::string lookup_name{"hIGH"}; + lookup_name.push_back(HIGH_ORDER_BIT); + lookup_name.append(".eXAMPLE.cOM"); + + REQUIRE(lookup.insert(cert_name.c_str(), context) >= 0); + + SSLCertContext *matched = lookup.find(lookup_name); + REQUIRE(matched != nullptr); + CHECK(matched->getCtx().get() == ctx); +} diff --git a/src/iocore/net/unit_tests/test_SSLNetVConnectionAsyncEp.cc b/src/iocore/net/unit_tests/test_SSLNetVConnectionAsyncEp.cc new file mode 100644 index 00000000000..3f5420d9b52 --- /dev/null +++ b/src/iocore/net/unit_tests/test_SSLNetVConnectionAsyncEp.cc @@ -0,0 +1,132 @@ +/** @file + + Catch based unit test for the async-handshake eventfd teardown invariant + in SSLNetVConnection. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "../P_SSLNetVConnection.h" +#include "../P_UnixPollDescriptor.h" +#include "iocore/net/EventIO.h" + +#include + +#include + +// Grants the test access to the private async_ep member. A friend struct adds +// no callable surface to the production class, so it is safe in shipped builds. +struct SSLNetVConnectionAsyncEpTestAccess { + static ReadWriteEventIO & + async_ep(SSLNetVConnection *vc) + { + return vc->async_ep; + } +}; + +// When OpenSSL returns SSL_ERROR_WANT_ASYNC during the TLS handshake, +// SSLNetVConnection registers async_ep on the poller with `this` as the +// EventIO target. free_thread() calls clear() immediately before the +// connection is returned to the allocator, so clear() must deregister the +// eventfd; otherwise the poller keeps a live registration pointing at memory +// that is about to be reused, which is the use-after-free this fix prevents. +TEST_CASE("SSLNetVConnection::clear stops a registered async-handshake eventfd") +{ + PollDescriptor pd; + + // Any pollable fd stands in for the OpenSSL async eventfd; a pipe read end + // is a portable pollable fd. The deregistration the fix relies on is the + // EPOLL_CTL_DEL in EventIO::stop(), which CI exercises on Linux. + int fds[2] = {-1, -1}; + REQUIRE(pipe(fds) == 0); + + auto *vc = new SSLNetVConnection(); + auto &ep = SSLNetVConnectionAsyncEpTestAccess::async_ep(vc); + + // Arm the eventfd the same way the WANT_ASYNC path does. The NetEvent and + // NetHandler pointers are only dereferenced when the poller dispatches an + // event, which this test never triggers, so nullptr is sufficient here. + ep.start(&pd, fds[0], nullptr, nullptr, EVENTIO_READ); + REQUIRE(ep.event_loop == &pd); + REQUIRE(ep.fd == fds[0]); + + vc->clear(); + + // The fix: clear() deregisters the eventfd. EventIO::stop() nulls event_loop + // after removing the registration from the poller. + CHECK(ep.event_loop == nullptr); + + delete vc; + close(fds[0]); + close(fds[1]); +} + +// The fix stops the eventfd in both do_io_close() and clear(), and on the +// normal close path both run. That double stop must be safe: free_thread() +// calls clear() after do_io_close() has already deregistered the eventfd. +// EventIO::stop() guards on event_loop, so the second call removes nothing. +TEST_CASE("SSLNetVConnection async-handshake eventfd teardown is idempotent") +{ + PollDescriptor pd; + + int fds[2] = {-1, -1}; + REQUIRE(pipe(fds) == 0); + + auto *vc = new SSLNetVConnection(); + auto &ep = SSLNetVConnectionAsyncEpTestAccess::async_ep(vc); + + ep.start(&pd, fds[0], nullptr, nullptr, EVENTIO_READ); + REQUIRE(ep.event_loop == &pd); + + // do_io_close() frees the VC (via free_thread), so it cannot be driven + // directly here; call stop() to mirror the deregistration it performs. + ep.stop(); + REQUIRE(ep.event_loop == nullptr); + + // Second teardown via clear() (the free path) must be a no-op, not a second + // deregistration of a now-stale fd. + vc->clear(); + CHECK(ep.event_loop == nullptr); + CHECK(ep.stop() == 0); + + delete vc; + close(fds[0]); + close(fds[1]); +} + +// The common case: a connection that never returned SSL_ERROR_WANT_ASYNC never +// armed async_ep, so its fd stays -1. The `if (async_ep.fd >= 0)` guard in the +// fix must make clear() a no-op there -- no spurious deregistration, no crash. +TEST_CASE("SSLNetVConnection clear is a no-op when no async-handshake eventfd was armed") +{ + auto *vc = new SSLNetVConnection(); + auto &ep = SSLNetVConnectionAsyncEpTestAccess::async_ep(vc); + + // Default-constructed EventIO: never registered with a poller. + REQUIRE(ep.fd < 0); + REQUIRE(ep.event_loop == nullptr); + + vc->clear(); + + // Nothing was armed, so nothing is deregistered and the guard is not entered. + CHECK(ep.fd < 0); + CHECK(ep.event_loop == nullptr); + + delete vc; +} diff --git a/src/iocore/net/unit_tests/test_SSLSNIConfig.cc b/src/iocore/net/unit_tests/test_SSLSNIConfig.cc index 382c6aab7f7..b9aa20f2668 100644 --- a/src/iocore/net/unit_tests/test_SSLSNIConfig.cc +++ b/src/iocore/net/unit_tests/test_SSLSNIConfig.cc @@ -34,6 +34,7 @@ #include #include +#include "tscore/ink_inet.h" TEST_CASE("Test SSLSNIConfig") { @@ -177,6 +178,67 @@ TEST_CASE("Test SSLSNIConfig") REQUIRE(actions.first); REQUIRE(actions.first->size() == 2); } + + SECTION("Wildcard fqdn does not match when the input has trailing content past the wildcarded suffix") + { + // *.bar.com must not match foo.bar.com.extra.com -- the regex must + // consume the entire SNI, not just a prefix of it. + auto const &actions{params.get("foo.bar.com.extra.com", 443)}; + CHECK(!actions.first); + } + + SECTION("get_property_config matches an exact fqdn") + { + CHECK(params.get_property_config("foo.bar.com") != nullptr); + } + + SECTION("get_property_config matches a wildcard fqdn") + { + CHECK(params.get_property_config("baz.bar.com") != nullptr); + } + + SECTION("get_property_config does not match when the input has trailing content past an exact fqdn") + { + // foo.bar.com is stored as a regex in next_hop_list; the lookup must + // not accept foo.bar.com.extra.com as a match. + CHECK(params.get_property_config("foo.bar.com.extra.com") == nullptr); + } + + SECTION("get_property_config does not match when the input has trailing content past a wildcard fqdn") + { + CHECK(params.get_property_config("baz.bar.com.extra.com") == nullptr); + } + + SECTION("get_property_config does not match when the input has leading content before an exact fqdn") + { + // allports.com is stored as a regex in next_hop_list; RE_ANCHORED at + // compile time should keep the lookup from matching prefix.allports.com. + // Use allports.com (which has no wildcard sibling in the test config) + // so the lookup cannot legitimately match via *.something. + CHECK(params.get_property_config("prefix.allports.com") == nullptr); + } + + SECTION("get_property_config does not match when the input has trailing content past an exact-only fqdn") + { + // Mirror of the prefix case: allports.com has no wildcard sibling, so + // this isolates the exact-entry path in next_hop_list. + CHECK(params.get_property_config("allports.com.extra.com") == nullptr); + } + + SECTION("get for an exact fqdn does not match when the input has leading content") + { + // Exact fqdns live in sni_action_map (hash-keyed), so a prefixed + // input must not produce a hit. + auto const &actions{params.get("prefix.allports.com", 1)}; + CHECK(!actions.first); + } + + SECTION("get for an exact-only fqdn does not match when the input has trailing content") + { + // Mirror of the prefix case for the hash-keyed path. + auto const &actions{params.get("allports.com.extra.com", 1)}; + CHECK(!actions.first); + } } TEST_CASE("SNIConfig reconfigure callback is invoked") @@ -192,3 +254,75 @@ TEST_CASE("SNIConfig reconfigure callback is invoked") SNIConfig::reconfigure(); CHECK(result == 42); } + +TEST_CASE("SNIConfig handles high-bit bytes while normalizing server names") +{ + constexpr auto HIGH_ORDER_BIT = static_cast(0x80); + + YamlSNIConfig::Item item; + item.fqdn = "High"; + item.fqdn.push_back(HIGH_ORDER_BIT); + item.fqdn.append(".Example.Com"); + item.inbound_port_ranges.emplace_back(1, ts::MAX_PORT_VALUE); + + SNIConfigParams params; + params.yaml_sni.items.push_back(item); + REQUIRE(params.load_sni_config()); + + std::string servername{"hIGH"}; + servername.push_back(HIGH_ORDER_BIT); + servername.append(".eXAMPLE.cOM"); + + auto const &actions{params.get(servername, 443)}; + REQUIRE(actions.first); + CHECK(actions.first->size() == 2); +} + +static IpEndpoint +make_endpoint(const char *ip_str) +{ + IpEndpoint ep; + ats_ip_pton(ip_str, &ep.sa); + return ep; +} + +TEST_CASE("SNI_IpAllow TestClientSNIAction") +{ + SNIConfigParams params; + REQUIRE(params.initialize(_XSTR(LIBINKNET_UNIT_TEST_DIR) "/sni_conf_test.yaml")); + + SECTION("Entry with ip_allow always triggers regardless of client IP") + { + auto const &actions{params.get("ipallow.example.com", 443)}; + REQUIRE(actions.first); + + auto blocked_ep = make_endpoint("172.16.0.1"); + int policy = 2; + bool triggered = false; + for (auto &&item : *actions.first) { + triggered |= item->TestClientSNIAction("ipallow.example.com", blocked_ep, policy); + } + CHECK(triggered); + + auto allowed_ep = make_endpoint("192.168.1.50"); + triggered = false; + for (auto &&item : *actions.first) { + triggered |= item->TestClientSNIAction("ipallow.example.com", allowed_ep, policy); + } + CHECK(triggered); + } + + SECTION("Entry without ip_allow does not trigger TestClientSNIAction for any IP") + { + auto const &actions{params.get("noipallow.example.com", 443)}; + REQUIRE(actions.first); + + auto any_ep = make_endpoint("203.0.113.1"); + int policy = 2; + bool triggered = false; + for (auto &&item : *actions.first) { + triggered |= item->TestClientSNIAction("noipallow.example.com", any_ep, policy); + } + CHECK_FALSE(triggered); + } +} diff --git a/src/iocore/net/unit_tests/test_YamlSNIConfig.cc b/src/iocore/net/unit_tests/test_YamlSNIConfig.cc index 7f22739e6b6..bb67951c4c3 100644 --- a/src/iocore/net/unit_tests/test_YamlSNIConfig.cc +++ b/src/iocore/net/unit_tests/test_YamlSNIConfig.cc @@ -56,7 +56,7 @@ TEST_CASE("YamlSNIConfig sets port ranges appropriately") FAIL(errorstream.str()); } REQUIRE(zret.is_ok()); - REQUIRE(conf.items.size() == 11); + REQUIRE(conf.items.size() == 13); SECTION("If no ports were specified, port range should contain all ports.") { diff --git a/src/mgmt/rpc/server/IPCSocketServer.cc b/src/mgmt/rpc/server/IPCSocketServer.cc index 6b6ebbe7c22..5063f90a593 100644 --- a/src/mgmt/rpc/server/IPCSocketServer.cc +++ b/src/mgmt/rpc/server/IPCSocketServer.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -304,12 +305,6 @@ IPCSocketServer::bind(std::error_code &ec) // remove socket file unlink(_conf.sockPathName.c_str()); - ret = ::bind(_socket, (struct sockaddr *)&_serverAddr, sizeof(struct sockaddr_un)); - if (ret < 0) { - ec = std::make_error_code(static_cast(errno)); - return; - } - // If the socket is not administratively restricted, check whether we have platform // support. Otherwise, default to making it restricted. bool restricted{true}; @@ -317,10 +312,26 @@ IPCSocketServer::bind(std::error_code &ec) restricted = !has_peereid(); } - mode_t mode = restricted ? 00700 : 00777; + const mode_t mode = restricted ? 00700 : 00777; + + // Narrow umask for the restricted socket so bind() creates the inode at the + // final mode. Safe: bind() runs single-threaded at startup before the thread pools. + const bool narrow_umask = restricted; + const mode_t old_umask = narrow_umask ? umask(0777 & ~mode) : 0; + + ret = ::bind(_socket, (struct sockaddr *)&_serverAddr, sizeof(struct sockaddr_un)); + const int bind_errno = errno; + if (narrow_umask) { + umask(old_umask); + } + if (ret < 0) { + ec = std::make_error_code(static_cast(bind_errno)); + return; + } + + // Defense in depth for filesystems that do not honor the umask on AF_UNIX socket + // inodes. if (chmod(_conf.sockPathName.c_str(), mode) < 0) { - // Some filesystems don't support chmod on AF_UNIX socket inodes. - // Keep running in that case and rely on default umask-derived permissions. if (errno != EINVAL && errno != ENOTSUP && errno != EOPNOTSUPP) { ec = std::make_error_code(static_cast(errno)); return; diff --git a/src/mgmt/rpc/server/unit_tests/test_rpcserver.cc b/src/mgmt/rpc/server/unit_tests/test_rpcserver.cc index 39639a46bc8..9e686d64e14 100644 --- a/src/mgmt/rpc/server/unit_tests/test_rpcserver.cc +++ b/src/mgmt/rpc/server/unit_tests/test_rpcserver.cc @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -41,6 +42,7 @@ #include #include #include +#include #include "swoc/swoc_file.h" @@ -493,6 +495,44 @@ TEST_CASE("Basic message sending to a running server", "[socket]") REQUIRE(rpc::test_remove_handler("do_nothing")); } +TEST_CASE("JSONRPC socket inode permissions reflect restricted_api config", "[socket][permissions]") +{ + SECTION("restricted_api=true yields mode 0700 on the socket inode") + { + auto confStr{ + R"({"rpc": { "enabled": true, "unix": { "lock_path_name": ")" + lockPath + R"(", "sock_path_name": ")" + sockPath + + R"(", "backlog": 5, "max_retry_on_transient_errors": 64, "incoming_request_max_size": 32000, "restricted_api": true }}})"}; + YAML::Node n = YAML::Load(confStr); + restart_json_rpc_server(n); + + // Restore the default test server configuration on scope exit, even if an + // assertion below fails (REQUIRE throws), so subsequent test cases are not + // left running against the restricted-api server. + struct ConfigRestorer { + std::function restore; + ~ConfigRestorer() + { + try { + restore(); + } catch (...) { + } + } + } config_restorer{[&]() { + auto restoreStr{R"({"rpc": { "enabled": true, "unix": { "lock_path_name": ")" + lockPath + R"(", "sock_path_name": ")" + + sockPath + + R"(", "backlog": 5, "max_retry_on_transient_errors": 64, "incoming_request_max_size": 32000 }}})"}; + YAML::Node restoreN = YAML::Load(restoreStr); + restart_json_rpc_server(restoreN); + }}; + + struct stat st { + }; + REQUIRE(::stat(sockPath.c_str(), &st) == 0); + CHECK(S_ISSOCK(st.st_mode)); + CHECK((st.st_mode & 0777) == 0700); + } +} + TEST_CASE("Sending a message bigger than the internal server's buffer. 32000", "[buffer][error]") { REQUIRE(rpc::add_method_handler("do_nothing32000", &do_nothing)); diff --git a/src/proxy/ControlBase.cc b/src/proxy/ControlBase.cc index 80da22fae09..564889183dc 100644 --- a/src/proxy/ControlBase.cc +++ b/src/proxy/ControlBase.cc @@ -42,6 +42,9 @@ #include "proxy/ControlMatcher.h" #include "proxy/hdrs/HdrUtils.h" +#include "swoc/string_view_util.h" + +#include #include #include @@ -487,7 +490,7 @@ bool MethodMod::check(HttpRequestData *req) const { auto method{req->hdr->method_get()}; - return method.length() >= text.length() && 0 == strncasecmp(method.data(), text.data(), text.length()); + return method.length() == text.length() && 0 == strcasecmp(method, std::string_view(text.data(), text.length())); } std::unique_ptr MethodMod::make(char *value, const char **) diff --git a/src/proxy/FetchSM.cc b/src/proxy/FetchSM.cc index 11fef0f8494..047ae2d9928 100644 --- a/src/proxy/FetchSM.cc +++ b/src/proxy/FetchSM.cc @@ -23,8 +23,10 @@ #include "tscore/ink_config.h" #include "proxy/FetchSM.h" +#include #include #include +#include #include "proxy/hdrs/HTTP.h" #include "proxy/PluginVC.h" #include "proxy/PluginHttpConnect.h" @@ -38,6 +40,31 @@ namespace { DbgCtl dbg_ctl{DEBUG_TAG}; +int64_t +copy_from_reader(char *dst, IOBufferReader *reader, int64_t nbytes) +{ + int64_t copied = 0; + + while (copied < nbytes) { + if (reader->block) { + reader->skip_empty_blocks(); + } + + const int64_t read_avail = reader->block_read_avail(); + if (read_avail <= 0) { + break; + } + + const int64_t to_copy = std::min(read_avail, nbytes - copied); + + memcpy(dst + copied, reader->start(), to_copy); + reader->consume(to_copy); + copied += to_copy; + } + + return copied; +} + } // end anonymous namespace bool @@ -426,21 +453,24 @@ FetchSM::get_info_from_buffer(IOBufferReader *reader) blk = reader->block.get(); // This is the equivalent of TSIOBufferBlockReadStart() - buf = blk->start() + reader->start_offset; read_done = blk->read_avail() - reader->start_offset; if (header_done == 0 && read_done > 0) { - int bytes_used = 0; - header_done = true; + int bytes_used = 0; + IOBufferReader *header_reader = reader->clone(); + header_done = true; if (client_response_hdr.parse_resp(&http_parser, reader, &bytes_used, 0) == ParseResult::DONE) { if ((bytes_used > 0) && (bytes_used <= read_avail)) { - memcpy(info, buf, bytes_used); - info += bytes_used; - client_bytes += bytes_used; + int64_t const bytes_copied = copy_from_reader(info, header_reader, bytes_used); + + ink_release_assert(bytes_copied == bytes_used); + info += bytes_copied; + client_bytes += bytes_copied; } } else { Error("Failed to parse headers in FetchSM buffer"); } + header_reader->dealloc(); // adjust the read_avail read_avail -= bytes_used; } diff --git a/src/proxy/ParentConsistentHash.cc b/src/proxy/ParentConsistentHash.cc index 3d2be5173a1..fad0d94e6f0 100644 --- a/src/proxy/ParentConsistentHash.cc +++ b/src/proxy/ParentConsistentHash.cc @@ -238,9 +238,16 @@ ParentConsistentHash::selectParent(bool first_call, ParentResult *result, Reques // check if the host is retryable. It's retryable if the retry window has elapsed // and the global host status is HOST_STATUS_UP if (pRec && !pRec->available.load() && host_stat == TS_HOST_STATUS_UP) { - Dbg(dbg_ctl_parent_select, "Parent.failedAt = %jd, retry = %u, xact_start = %jd", pRec->failedAt.load(), retry_time, - request_info->xact_start); - if ((pRec->failedAt.load() + retry_time) < request_info->xact_start) { + time_t observed = pRec->failedAt.load(); + Dbg(dbg_ctl_parent_select, "Parent.failedAt = %jd, retry = %u, xact_start = %jd", static_cast(observed), + retry_time, static_cast(request_info->xact_start)); + // Atomically push failedAt to (xact_start - retry_time) so that only + // one concurrent transaction with this xact_start takes the retry slot. + // Sequential retries with a later xact_start still pass the window + // check (failedAt + retry_time == xact_start, so xact_start' > xact_start + // satisfies the < check). Losers fall through to other parents. + if ((observed + retry_time) < request_info->xact_start && + pRec->failedAt.compare_exchange_strong(observed, request_info->xact_start - retry_time)) { parentRetry = true; // make sure that the proper state is recorded in the result structure result->last_parent = pRec->idx; diff --git a/src/proxy/ParentRoundRobin.cc b/src/proxy/ParentRoundRobin.cc index cffae0f611b..c6e10bbe7ca 100644 --- a/src/proxy/ParentRoundRobin.cc +++ b/src/proxy/ParentRoundRobin.cc @@ -160,8 +160,22 @@ ParentRoundRobin::selectParent(bool first_call, ParentResult *result, RequestDat parentUp = true; } } else { - if ((result->wrap_around) || - (((parents[cur_index].failedAt + retry_time) < request_info->xact_start) && host_stat == TS_HOST_STATUS_UP)) { + bool retryable = false; + if (result->wrap_around) { + // Wrap-around: force a retry of the parent regardless of the timer. + retryable = true; + } else if (host_stat == TS_HOST_STATUS_UP) { + // Atomically push failedAt to (xact_start - retry_time) so only one + // concurrent transaction with this xact_start takes the retry slot; + // sequential retries with a later xact_start still pass the window + // check. + time_t observed = parents[cur_index].failedAt.load(); + if ((observed + retry_time) < request_info->xact_start && + parents[cur_index].failedAt.compare_exchange_strong(observed, request_info->xact_start - retry_time)) { + retryable = true; + } + } + if (retryable) { Dbg(dbg_ctl_parent_select, "Parent[%d].failedAt = %u, retry = %u, xact_start = %" PRId64 " but wrap = %d", cur_index, static_cast(parents[cur_index].failedAt.load()), retry_time, static_cast(request_info->xact_start), result->wrap_around); diff --git a/src/proxy/ProxyTransaction.cc b/src/proxy/ProxyTransaction.cc index a301ce20deb..05fb0c8877e 100644 --- a/src/proxy/ProxyTransaction.cc +++ b/src/proxy/ProxyTransaction.cc @@ -257,6 +257,12 @@ ProxyTransaction::expect_send_trailer() const return false; } +bool +ProxyTransaction::can_send_h2_trailer() const +{ + return false; +} + void ProxyTransaction::set_expect_send_trailer() { diff --git a/src/proxy/ReverseProxy.cc b/src/proxy/ReverseProxy.cc index 8368ad515e8..8600013624f 100644 --- a/src/proxy/ReverseProxy.cc +++ b/src/proxy/ReverseProxy.cc @@ -29,8 +29,10 @@ #include "tscore/ink_platform.h" #include "tscore/Filenames.h" +#include "tscore/TSSystemState.h" #include #include "iocore/cache/Cache.h" +#include "iocore/eventsystem/Freer.h" #include "proxy/ReverseProxy.h" #include "mgmt/config/ConfigContextDiags.h" #include "mgmt/config/ConfigRegistry.h" @@ -49,12 +51,44 @@ Ptr reconfig_mutex; DbgCtl dbg_ctl_url_rewrite{"url_rewrite"}; +// Steers UrlRewriteDeleter to inline-delete; see shutdown_url_rewrite(). +std::atomic rewrite_table_shutdown{false}; + +// Defer teardown to ET_TASK; UrlRewrite destruction can be slow. +struct UrlRewriteDeleter { + void + operator()(UrlRewrite *p) const noexcept + { + if (!p) { + return; + } + if (rewrite_table_shutdown.load(std::memory_order_acquire) || TSSystemState::is_event_system_shut_down()) { + // Leak; plugin teardown is unsafe post-shutdown. + return; + } + // new_Deleter allocates; fall back to inline delete so we don't escape noexcept. + try { + new_Deleter(p, 0); + } catch (...) { + delete p; + } + } +}; + } // end anonymous namespace // Global Ptrs -std::atomic rewrite_table = nullptr; +AtomicSharedPtr rewrite_table; thread_local PluginThreadContext *pluginThreadContext = nullptr; +void +shutdown_url_rewrite() +{ + // Drain before flag: this ref destructs normally; later drops leak. + rewrite_table.exchange(nullptr); + rewrite_table_shutdown.store(true, std::memory_order_release); +} + // Tokens for the Callback function #define FILE_CHANGED 0 #define REVERSE_CHANGED 1 @@ -71,10 +105,9 @@ static void init_table_volume_host_records(UrlRewrite &table); int init_reverse_proxy() { - ink_assert(rewrite_table.load() == nullptr); - reconfig_mutex = new_ProxyMutex(); - auto *initial_table = new UrlRewrite(); - initial_table->acquire(); + ink_assert(rewrite_table.load(std::memory_order_acquire) == nullptr); + reconfig_mutex = new_ProxyMutex(); + auto initial_table = std::make_unique(); // Register with ConfigRegistry BEFORE load() so that remap.config is in // FileManager's bindings when .include directives call configFileChild() @@ -100,7 +133,8 @@ init_reverse_proxy() init_table_volume_host_records(*initial_table); } - rewrite_table.store(initial_table, std::memory_order_release); + // Publish: shared_ptr semantics replace the prior bespoke acquire()/release() refcount on UrlRewrite. + rewrite_table.store(std::shared_ptr(initial_table.release(), UrlRewriteDeleter{}), std::memory_order_release); RecRegisterConfigUpdateCb("proxy.config.reverse_proxy.enabled", url_rewrite_CB, (void *)REVERSE_CHANGED); return 0; @@ -145,25 +179,19 @@ reloadUrlRewrite(ConfigContext ctx) std::string msg_buffer; msg_buffer.reserve(1024); - UrlRewrite *newTable, *oldTable; CfgLoadLog(ctx, DL_Note, "%s loading ...", ts::filename::REMAP); Dbg(dbg_ctl_url_rewrite, "%s updated, reloading...", ts::filename::REMAP); - newTable = new UrlRewrite(); + auto newTable = std::make_unique(); if (newTable->load(ctx)) { swoc::bwprint(msg_buffer, "{} finished loading", ts::filename::REMAP); - // Hold at least one lease, until we reload the configuration - newTable->acquire(); - - // Swap configurations - oldTable = rewrite_table.exchange(newTable); - - ink_assert(oldTable != nullptr); - - // Release the old one - oldTable->release(); + // Atomic publish: an old reader's shared_ptr keeps the prior table alive until its last + // ref is dropped; new readers see the new table. The prior race between load() and + // acquire() on the bespoke refcount cannot revive a table whose refcount was driven to + // zero, because there is no separate refcount. + rewrite_table.exchange(std::shared_ptr(newTable.release(), UrlRewriteDeleter{}), std::memory_order_acq_rel); Dbg(dbg_ctl_url_rewrite, "%s", msg_buffer.c_str()); CfgLoadComplete(ctx, "%s finished loading", ts::filename::REMAP); @@ -171,7 +199,7 @@ reloadUrlRewrite(ConfigContext ctx) } else { swoc::bwprint(msg_buffer, "{} failed to load", ts::filename::REMAP); - delete newTable; + // newTable is a unique_ptr; falling out of scope deletes it. Dbg(dbg_ctl_url_rewrite, "%s", msg_buffer.c_str()); CfgLoadFail(ctx, "%s failed to load", ts::filename::REMAP); return false; @@ -233,25 +261,23 @@ init_remap_volume_host_records() return; } - UrlRewrite *table = rewrite_table.load(std::memory_order_acquire); + auto table = rewrite_table.load(std::memory_order_acquire); if (!table) { return; } - table->acquire(); - if (table->is_valid()) { init_table_volume_host_records(*table); } - - table->release(); } int url_rewrite_CB(const char * /* name ATS_UNUSED */, RecDataT /* data_type ATS_UNUSED */, RecData data, void * /* cookie ATS_UNUSED */) { - rewrite_table.load()->SetReverseFlag(data.rec_int); + if (auto table = rewrite_table.load(std::memory_order_acquire); table != nullptr) { + table->SetReverseFlag(data.rec_int); + } return 0; } diff --git a/src/proxy/hdrs/HTTP.cc b/src/proxy/hdrs/HTTP.cc index 62b6357421e..22c0ade88ac 100644 --- a/src/proxy/hdrs/HTTP.cc +++ b/src/proxy/hdrs/HTTP.cc @@ -25,6 +25,7 @@ #include "tscore/ink_platform.h" #include "tscore/ink_inet.h" #include +#include #include #include #include @@ -913,7 +914,7 @@ http_parser_parse_req(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const (end[-2] ^ '\r') | (end[-1] ^ '\n')) != 0) { goto slow_case; } - if (!(isdigit(end[-5]) && isdigit(end[-3]))) { + if (!(ParseRules::is_digit(end[-5]) && ParseRules::is_digit(end[-3]))) { goto slow_case; } if (!(ParseRules::is_space(cur[3]) && (!ParseRules::is_space(cur[4])) && (!ParseRules::is_space(end[-12])) && @@ -1010,7 +1011,7 @@ http_parser_parse_req(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const } version_end = cur + 1; parse_version2: - if (isdigit(*cur)) { + if (ParseRules::is_digit(*cur)) { GETPREV(parse_url); goto parse_version2; } @@ -1020,7 +1021,7 @@ http_parser_parse_req(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const } goto parse_url; parse_version3: - if (isdigit(*cur)) { + if (ParseRules::is_digit(*cur)) { GETPREV(parse_url); goto parse_version3; } @@ -1270,18 +1271,17 @@ validate_hdr_content_length(HdrHeap *heap, HTTPHdrImpl *hh) // status code and then close the connection std::string_view value = content_length_field->value_get(); - // RFC 9110 section 8.6. - // Content-Length = 1*DIGIT - // - if (value.empty()) { - Dbg(dbg_ctl_http, "Content-Length headers don't match the ABNF, returning parse error"); - return ParseResult::ERROR; - } - - // If the content-length value contains a non-numeric value, the header is invalid - if (std::find_if(value.cbegin(), value.cend(), [](std::string_view::value_type c) { return !std::isdigit(c); }) != - value.cend()) { - Dbg(dbg_ctl_http, "Content-Length value contains non-digit, returning parse error"); + // RFC 9110 section 8.6: Content-Length = 1*DIGIT + // RFC 9110 section 8.6: "a recipient MUST anticipate potentially large + // decimal numerals and prevent parsing errors due to integer conversion + // overflows" + // RFC 9112 section 6.3: an invalid Content-Length is an unrecoverable + // framing error (request → 400, proxied response → 502). + // from_chars rejects empty, non-digit, and overflow in one pass. + int64_t cl; + auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), cl); + if (ec != std::errc{} || ptr != value.data() + value.size() || cl < 0) { + Dbg(dbg_ctl_http, "Content-Length value is invalid, returning parse error"); return ParseResult::ERROR; } @@ -1362,8 +1362,9 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const if (end - cur >= 16) { int http_match = ((cur[0] ^ 'H') | (cur[1] ^ 'T') | (cur[2] ^ 'T') | (cur[3] ^ 'P') | (cur[4] ^ '/') | (cur[6] ^ '.') | (cur[8] ^ ' ')); - if ((http_match != 0) || (!(isdigit(cur[5]) && isdigit(cur[7]) && isdigit(cur[9]) && isdigit(cur[10]) && isdigit(cur[11]) && - (!ParseRules::is_space(cur[13]))))) { + if ((http_match != 0) || + (!(ParseRules::is_digit(cur[5]) && ParseRules::is_digit(cur[7]) && ParseRules::is_digit(cur[9]) && + ParseRules::is_digit(cur[10]) && ParseRules::is_digit(cur[11]) && (!ParseRules::is_space(cur[13]))))) { goto slow_case; } @@ -1422,7 +1423,7 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const } GETNEXT(eoh); parse_version2: - if (isdigit(*cur)) { + if (ParseRules::is_digit(*cur)) { GETNEXT(eoh); goto parse_version2; } @@ -1432,7 +1433,7 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const } goto eoh; parse_version3: - if (isdigit(*cur)) { + if (ParseRules::is_digit(*cur)) { GETNEXT(eoh); goto parse_version3; } @@ -1451,7 +1452,7 @@ http_parser_parse_resp(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const status_start = cur; parse_status2: status_end = cur; - if (isdigit(*cur)) { + if (ParseRules::is_digit(*cur)) { GETNEXT(done); goto parse_status2; } @@ -1523,8 +1524,11 @@ http_parse_status(const char *start, const char *end) start += 1; } - while ((start != end) && isdigit(*start)) { + while ((start != end) && ParseRules::is_digit(*start)) { status = (status * 10) + (*start++ - '0'); + if (status > 999) { + return HTTPStatus::NONE; + } } return static_cast(status); @@ -1549,7 +1553,7 @@ http_parse_version(const char *start, const char *end) maj = 0; min = 0; - while ((start != end) && isdigit(*start)) { + while ((start != end) && ParseRules::is_digit(*start)) { maj = (maj * 10) + (*start - '0'); start += 1; } @@ -1558,7 +1562,7 @@ http_parse_version(const char *start, const char *end) start += 1; } - while ((start != end) && isdigit(*start)) { + while ((start != end) && ParseRules::is_digit(*start)) { min = (min * 10) + (*start - '0'); start += 1; } @@ -1628,7 +1632,7 @@ http_parse_qvalue(const char *&buf, int &len) http_skip_ws(buf, len); n = 0.0; - while (len > 0 && *buf && isdigit(*buf)) { + while (len > 0 && *buf && ParseRules::is_digit(*buf)) { n = (n * 10) + (*buf++ - '0'); len -= 1; } @@ -1638,7 +1642,7 @@ http_parse_qvalue(const char *&buf, int &len) len -= 1; f = 10; - while (len > 0 && *buf && isdigit(*buf)) { + while (len > 0 && *buf && ParseRules::is_digit(*buf)) { n += (*buf++ - '0') / static_cast(f); f *= 10; len -= 1; diff --git a/src/proxy/hdrs/HdrHeap.cc b/src/proxy/hdrs/HdrHeap.cc index 1aab23b93f1..b744adaad22 100644 --- a/src/proxy/hdrs/HdrHeap.cc +++ b/src/proxy/hdrs/HdrHeap.cc @@ -855,7 +855,11 @@ HdrHeap::check_marshalled(uint32_t buf_length) return false; } - if ((uintptr_t)(this->m_size + m_ronly_heap[0].m_heap_start) > buf_length) { + if (m_ronly_heap[0].m_heap_len < 0) { + return false; + } + + if (static_cast(this->m_size) + static_cast(m_ronly_heap[0].m_heap_len) > buf_length) { return false; } @@ -898,11 +902,15 @@ HdrHeap::unmarshal(int buf_length, int obj_type, HdrHeapObjImpl **found_obj, Ref return -1; } - int unmarshal_size = this->unmarshal_size(); - if (unmarshal_size > buf_length) { + if (m_size < static_cast(HDR_HEAP_HDR_SIZE) || // heap too small for header + m_size != (uintptr_t)m_ronly_heap[0].m_heap_start || // string heap offset inconsistent + m_ronly_heap[0].m_heap_len < 0 || // invalid string heap length + buf_length < 0 || // invalid buf_length + static_cast(m_size) + static_cast(m_ronly_heap[0].m_heap_len) > static_cast(buf_length)) { ink_assert(!"HdrHeap::unmarshal truncated header"); return -1; } + int unmarshal_size = this->unmarshal_size(); #ifdef HDR_HEAP_CHECKSUMS if (m_free_start != NULL) { uint32_t stored_sum = (uint32_t)m_free_start; diff --git a/src/proxy/hdrs/HeaderValidator.cc b/src/proxy/hdrs/HeaderValidator.cc index 9cbe9457a59..2db4eeff318 100644 --- a/src/proxy/hdrs/HeaderValidator.cc +++ b/src/proxy/hdrs/HeaderValidator.cc @@ -60,11 +60,12 @@ HeaderValidator::is_h2_h3_header_valid(const HTTPHdr &hdr, bool is_response, boo } } - // rfc7540,sec8.1.2.2 and rfc9114,sec4.2: Any message containing + // rfc9113,sec8.2.2 and rfc9114,sec4.2: Any message containing // connection-specific header fields MUST be treated as malformed. if (hdr.field_find(static_cast(MIME_FIELD_CONNECTION)) != nullptr || hdr.field_find(static_cast(MIME_FIELD_KEEP_ALIVE)) != nullptr || hdr.field_find(static_cast(MIME_FIELD_PROXY_CONNECTION)) != nullptr || + hdr.field_find(static_cast(MIME_FIELD_TRANSFER_ENCODING)) != nullptr || hdr.field_find(static_cast(MIME_FIELD_UPGRADE)) != nullptr) { return false; } diff --git a/src/proxy/hdrs/MIME.cc b/src/proxy/hdrs/MIME.cc index 66f95c6bf2f..df70b80a053 100644 --- a/src/proxy/hdrs/MIME.cc +++ b/src/proxy/hdrs/MIME.cc @@ -33,6 +33,7 @@ #include #include #include +#include #include #include "proxy/hdrs/MIME.h" #include "proxy/hdrs/HdrHeap.h" @@ -1365,7 +1366,12 @@ mime_field_create_named(HdrHeap *heap, MIMEHdrImpl *mh, std::string_view name) { MIMEField *field = mime_field_create(heap, mh); int field_name_wks_idx = hdrtoken_tokenize(name.data(), static_cast(name.length())); - mime_field_name_set(heap, mh, field, field_name_wks_idx, name, true); + if (!mime_field_name_set(heap, mh, field, field_name_wks_idx, name, true)) { + // The name exceeds the uint16_t field-length limit and was rejected. Tear + // the detached field back down so callers do not get a half-formed field. + mime_field_destroy(mh, field); + return nullptr; + } return field; } @@ -1671,18 +1677,28 @@ MIMEField::name_get() const return {m_ptr_name, m_len_name}; } -void +bool mime_field_name_set(HdrHeap *heap, MIMEHdrImpl * /* mh ATS_UNUSED */, MIMEField *field, int16_t name_wks_idx_or_neg1, std::string_view name, bool must_copy_string) { ink_assert(field->m_readiness == MIME_FIELD_SLOT_READINESS_DETACHED); + // A name longer than UINT16_MAX cannot be stored in the uint16_t m_len_name. + // Reject it without mutating the field so the caller never observes a name + // whose stored length disagrees with its data. + if (name.length() > static_cast(UINT16_MAX)) { + Warning("mime_field_name_set: rejecting oversized name of length %zu (max %u)", name.length(), UINT16_MAX); + return false; + } + field->m_wks_idx = name_wks_idx_or_neg1; mime_str_u16_set(heap, name, &(field->m_ptr_name), &(field->m_len_name), must_copy_string); if ((name_wks_idx_or_neg1 == MIME_WKSIDX_CACHE_CONTROL) || (name_wks_idx_or_neg1 == MIME_WKSIDX_PRAGMA)) { field->m_flags |= MIME_FIELD_SLOT_FLAGS_COOKED; } + + return true; } int @@ -2045,9 +2061,21 @@ mime_field_value_extend_comma_val(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *fie } } -void +bool mime_field_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, std::string_view value, bool must_copy_string) { + // Cap the value length at UINT16_MAX as a deliberate, uniform header + // field-size limit matching the name field (m_len_name is uint16_t). The + // value storage (m_len_value is uint32_t : 24) could physically hold more, + // but we reject longer values to keep a single consistent limit on both the + // name and value of a header field. + if (value.length() > UINT16_MAX) { + Warning("mime_field_value_set: rejecting oversized value of length %zu (max %u)", value.length(), UINT16_MAX); + // Reject without mutating the field: a live field keeps its current value + // and cooked state, so a rejected set is a clean no-op. + return false; + } + heap->free_string(field->m_ptr_value, field->m_len_value); if (must_copy_string && value.data()) { @@ -2063,6 +2091,8 @@ mime_field_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, std::stri if (field->is_live() && field->is_cooked()) { mh->recompute_cooked_stuff(field); } + + return true; } void @@ -2097,7 +2127,7 @@ mime_field_value_set_date(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, time mime_field_value_set(heap, mh, field, std::string_view{buf, static_cast(len)}, true); } -void +bool mime_field_name_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int16_t name_wks_idx_or_neg1, std::string_view name, std::string_view value, int n_v_raw_printable, int n_v_raw_length, bool must_copy_strings) { @@ -2107,9 +2137,21 @@ mime_field_name_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int1 ink_assert(field->m_readiness == MIME_FIELD_SLOT_READINESS_DETACHED); + // m_len_name is uint16_t, so a name longer than UINT16_MAX would truncate its + // stored length; m_len_value is uint32_t : 24 and could hold more, but the value + // is capped at the same UINT16_MAX for a single uniform limit. Reject oversized + // strings up front, before either branch mutates the field, so a rejected set is + // an all-or-nothing no-op rather than a half-stored (name xor value) field. + if (name_length < 0 || name_length > static_cast(UINT16_MAX) || value_length < 0 || + value_length > static_cast(UINT16_MAX)) { + Warning("mime_field_name_value_set: rejecting oversized name=%d value=%d (max %u)", name_length, value_length, UINT16_MAX); + return false; + } + if (must_copy_strings) { - mime_field_name_set(heap, mh, field, name_wks_idx_or_neg1, name, true); - mime_field_value_set(heap, mh, field, value, true); + bool name_stored = mime_field_name_set(heap, mh, field, name_wks_idx_or_neg1, name, true); + bool value_stored = mime_field_value_set(heap, mh, field, value, true); + return name_stored && value_stored; } else { field->m_wks_idx = name_wks_idx_or_neg1; field->m_ptr_name = name.data(); @@ -2131,6 +2173,7 @@ mime_field_name_value_set(HdrHeap *heap, MIMEHdrImpl *mh, MIMEField *field, int1 mh->recompute_cooked_stuff(field); } } + return true; } void @@ -2282,11 +2325,19 @@ MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, b // After a LF, the next line might be a continuation / folded line. That's indicated by a // starting whitespace. If that's the case, back up over the preceding CR/LF with space and // pretend it's the same line. + // + // NOTE: obs-fold is only detected here when the LF and the continuation whitespace are in + // the same get() call. If the CRLF falls exactly at the end of an input buffer, the + // post-loop cleanup returns OK before we see the next byte, so the fold is silently lost + // and the continuation becomes a separate field. Fixing this requires changing the scanner + // to return CONT in AFTER state, which affects all callers including TSMimeHdrParse. if (ParseRules::is_ws(*text)) { // folded line. char *unfold = const_cast(text.data() - 1); - *unfold-- = ' '; - if (ParseRules::is_cr(*unfold)) { - *unfold = ' '; + *unfold = ' '; + if (unfold > input.data() && ParseRules::is_cr(*(unfold - 1))) { + *(unfold - 1) = ' '; + } else if (!m_line.empty() && ParseRules::is_cr(m_line.back())) { + m_line.back() = ' '; } m_state = MimeParseState::INSIDE; // back inside the field. } else { @@ -2471,6 +2522,14 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char return ParseResult::ERROR; } + // m_len_name is uint16_t, so a name longer than UINT16_MAX would truncate + // its stored length; m_len_value is uint32_t : 24 and could hold more, but the + // value is capped at the same UINT16_MAX for a uniform limit. Reject the + // message rather than keep a field whose length and data disagree. + if (field_name.size() > UINT16_MAX || field_value.size() > UINT16_MAX) { + return ParseResult::ERROR; + } + // int total_line_length = (int)(field_line_last - field_line_first + 1); ////////////////////////////////////////////////////////////////////// @@ -2672,6 +2731,12 @@ to_same_char(int ch) return ch; } +int +to_lower_char(int ch) +{ + return std::tolower(static_cast(ch)); +} + } // end anonymous namespace int @@ -2683,7 +2748,7 @@ mime_mem_print(std::string_view src, char *buf_start, int buf_length, int *buf_i int mime_mem_print_lc(std::string_view src, char *buf_start, int buf_length, int *buf_index_inout, int *buf_chars_to_skip_inout) { - return mime_mem_print_(src, buf_start, buf_length, buf_index_inout, buf_chars_to_skip_inout, std::tolower); + return mime_mem_print_(src, buf_start, buf_length, buf_index_inout, buf_chars_to_skip_inout, to_lower_char); } int @@ -2753,7 +2818,14 @@ const char * mime_str_u16_set(HdrHeap *heap, std::string_view src, const char **d_str, uint16_t *d_len, bool must_copy) { auto s_len{static_cast(src.length())}; - ink_assert(s_len >= 0 && s_len < UINT16_MAX); + // The out-length (*d_len) is uint16_t. A length greater than UINT16_MAX + // would be silently truncated by the assignment below, leaving the stored + // length out of sync with the data. Reject the value without mutating + // *d_str/*d_len so any existing stored string is preserved intact. + if (s_len < 0 || s_len > static_cast(UINT16_MAX)) { + Warning("mime_str_u16_set: rejecting oversized field of length %d (max %u)", s_len, UINT16_MAX); + return nullptr; + } // INKqa08287 - keep track of free string space. // INVARIANT: passed in result pointers must be to // either NULL or be valid ptr for a string already @@ -3006,130 +3078,72 @@ mime_format_date(char *buffer, time_t value) return buf - buffer; // not counting NUL } +// RFC 9110 §17.5: recipients must limit processing of numeric values to prevent +// arithmetic overflows. These parsers clamp to the max of their respective type on +// overflow rather than returning an error. Callers that need stricter policy +// (e.g. rejecting an overflowing Content-Length per RFC 9112 §6.3, or clamping +// Age to 2^31 per RFC 9111 §1.2.2) apply it at their own layer. int32_t mime_parse_int(const char *buf, const char *end) { - int32_t num; - bool negative; - if (!buf || (buf == end)) { return 0; } - if (is_digit(*buf)) { // fast case - num = *buf++ - '0'; - while ((buf != end) && is_digit(*buf)) { - if (num != INT_MAX) { - int new_num = (num * 10) + (*buf++ - '0'); - - num = (new_num < num ? INT_MAX : new_num); // Check for overflow - } else { - ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT - } - } - - return num; - } else { - num = 0; - negative = false; - - while ((buf != end) && ParseRules::is_space(*buf)) { - buf += 1; - } - - if ((buf != end) && (*buf == '-')) { - negative = true; - buf += 1; - } - // NOTE: we first compute the value as negative then correct the - // sign back to positive. This enables us to correctly parse MININT. - while ((buf != end) && is_digit(*buf)) { - if (num != INT_MIN) { - int new_num = (num * 10) - (*buf++ - '0'); - - num = (new_num > num ? INT_MIN : new_num); // Check for overflow, so to speak, see above re: negative - } else { - ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT - } - } + while ((buf != end) && ParseRules::is_space(*buf)) { + buf += 1; + } - if (!negative) { - num = -num; - } + int32_t num = 0; + auto [ptr, ec] = std::from_chars(buf, end, num); - return num; + if (ec == std::errc::result_out_of_range) { + return (*buf == '-') ? INT32_MIN : INT32_MAX; } + + return num; } uint32_t mime_parse_uint(const char *buf, const char *end) { - uint32_t num; - if (!buf || (buf == end)) { return 0; } - if (is_digit(*buf)) // fast case - { - num = *buf++ - '0'; - while ((buf != end) && is_digit(*buf)) { - num = (num * 10) + (*buf++ - '0'); - } - return num; - } else { - num = 0; - while ((buf != end) && ParseRules::is_space(*buf)) { - buf += 1; - } - while ((buf != end) && is_digit(*buf)) { - num = (num * 10) + (*buf++ - '0'); - } - return num; + while ((buf != end) && ParseRules::is_space(*buf)) { + buf += 1; + } + + uint32_t num = 0; + auto [ptr, ec] = std::from_chars(buf, end, num); + + if (ec == std::errc::result_out_of_range) { + return UINT32_MAX; } + + return num; } int64_t mime_parse_int64(const char *buf, const char *end) { - int64_t num; - bool negative; - if (!buf || (buf == end)) { return 0; } - if (is_digit(*buf)) // fast case - { - num = *buf++ - '0'; - while ((buf != end) && is_digit(*buf)) { - num = (num * 10) + (*buf++ - '0'); - } - return num; - } else { - num = 0; - negative = false; - - while ((buf != end) && ParseRules::is_space(*buf)) { - buf += 1; - } - - if ((buf != end) && (*buf == '-')) { - negative = true; - buf += 1; - } - // NOTE: we first compute the value as negative then correct the - // sign back to positive. This enables us to correctly parse MININT. - while ((buf != end) && is_digit(*buf)) { - num = (num * 10) - (*buf++ - '0'); - } + while ((buf != end) && ParseRules::is_space(*buf)) { + buf += 1; + } - if (!negative) { - num = -num; - } + int64_t num = 0; + auto [ptr, ec] = std::from_chars(buf, end, num); - return num; + if (ec == std::errc::result_out_of_range) { + return (*buf == '-') ? INT64_MIN : INT64_MAX; } + + return num; } /*------------------------------------------------------------------------- @@ -3496,52 +3510,14 @@ mime_parse_integer(const char *&buf, const char *end, int *integer) return false; } - int32_t num; - bool negative; - - // This code is copied verbatim from mime_parse_int ... Sigh. Maybe amc is right, and - // we really need to clean this up. But, as such, we should redo all these interfaces, - // and that's a big undertaking (and we'd want to move these strings all to string_view's). - if (is_digit(*buf)) { // fast case - num = *buf++ - '0'; - while ((buf != end) && is_digit(*buf)) { - if (num != INT_MAX) { - int new_num = (num * 10) + (*buf++ - '0'); + int32_t num = 0; + auto [ptr, ec] = std::from_chars(buf, end, num); - num = (new_num < num ? INT_MAX : new_num); // Check for overflow - } else { - ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT - } - } - } else { - num = 0; - negative = false; - - while ((buf != end) && ParseRules::is_space(*buf)) { - buf += 1; - } - - if ((buf != end) && (*buf == '-')) { - negative = true; - buf += 1; - } - // NOTE: we first compute the value as negative then correct the - // sign back to positive. This enables us to correctly parse MININT. - while ((buf != end) && is_digit(*buf)) { - if (num != INT_MIN) { - int new_num = (num * 10) - (*buf++ - '0'); - - num = (new_num > num ? INT_MIN : new_num); // Check for overflow, so to speak, see above re: negative - } else { - ++buf; // Skip the remaining (valid) digits since we reached MAX/MIN_INT - } - } - - if (!negative) { - num = -num; - } + if (ec == std::errc::result_out_of_range) { + num = (*buf == '-') ? INT32_MIN : INT32_MAX; } + buf = ptr; *integer = num; return true; diff --git a/src/proxy/hdrs/URL.cc b/src/proxy/hdrs/URL.cc index fa75ae25df3..d579cf706c1 100644 --- a/src/proxy/hdrs/URL.cc +++ b/src/proxy/hdrs/URL.cc @@ -474,15 +474,23 @@ URLImpl::set_port(HdrHeap *heap, std::string_view value, bool copy_string) if (value.empty()) { value = {nullptr, 0}; } - mime_str_u16_set(heap, value, &(this->m_ptr_port), &(this->m_len_port), copy_string); this->m_port = 0; for (auto digit : value) { if (!ParseRules::is_digit(digit)) { break; } - this->m_port = this->m_port * 10 + (digit - '0'); + unsigned int next = this->m_port * 10 + (digit - '0'); + if (next > 65535) { + // Reject out-of-range port: drop the parsed text as well so the URL is + // fully treated as if no port were given. + mime_str_u16_set(heap, {nullptr, 0}, &(this->m_ptr_port), &(this->m_len_port), copy_string); + this->m_port = 0; + return; + } + this->m_port = static_cast(next); } + mime_str_u16_set(heap, value, &(this->m_ptr_port), &(this->m_len_port), copy_string); } /*------------------------------------------------------------------------- @@ -683,6 +691,22 @@ url_string_get_buf(URLImpl *url, char *dstbuf, int dstbuf_size, int *length) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ +namespace +{ +/** Construct a string_view from a URL part pointer/length pair, guarding + * against the nullptr case. Constructing std::string_view(nullptr, 0) is UB. + * URL parts that are absent store a nullptr m_ptr_* and zero m_len_*. + */ +inline std::string_view +make_part_view(const char *ptr, int len) +{ + if (ptr == nullptr) { + return {}; + } + return {ptr, static_cast(len)}; +} +} // namespace + std::string_view URLImpl::get_scheme() const noexcept { @@ -690,7 +714,7 @@ URLImpl::get_scheme() const noexcept return {hdrtoken_index_to_wks(this->m_scheme_wks_idx), static_cast(hdrtoken_index_to_length(this->m_scheme_wks_idx))}; } else { - return {this->m_ptr_scheme, static_cast(this->m_len_scheme)}; + return make_part_view(this->m_ptr_scheme, this->m_len_scheme); } } @@ -700,7 +724,7 @@ URLImpl::get_scheme() const noexcept std::string_view URLImpl::get_user() const noexcept { - return {this->m_ptr_user, static_cast(this->m_len_user)}; + return make_part_view(this->m_ptr_user, this->m_len_user); } /*------------------------------------------------------------------------- @@ -709,7 +733,7 @@ URLImpl::get_user() const noexcept std::string_view URLImpl::get_password() const noexcept { - return {this->m_ptr_password, static_cast(this->m_len_password)}; + return make_part_view(this->m_ptr_password, this->m_len_password); } /*------------------------------------------------------------------------- @@ -718,7 +742,7 @@ URLImpl::get_password() const noexcept std::string_view URLImpl::get_host() const noexcept { - return {this->m_ptr_host, static_cast(this->m_len_host)}; + return make_part_view(this->m_ptr_host, this->m_len_host); } /*------------------------------------------------------------------------- @@ -736,7 +760,7 @@ URLImpl::get_port() std::string_view URLImpl::get_path() const noexcept { - return {this->m_ptr_path, static_cast(this->m_len_path)}; + return make_part_view(this->m_ptr_path, this->m_len_path); } /*------------------------------------------------------------------------- @@ -745,7 +769,7 @@ URLImpl::get_path() const noexcept std::string_view URLImpl::get_query() const noexcept { - return {this->m_ptr_query, static_cast(this->m_len_query)}; + return make_part_view(this->m_ptr_query, this->m_len_query); } /*------------------------------------------------------------------------- @@ -754,7 +778,7 @@ URLImpl::get_query() const noexcept std::string_view URLImpl::get_fragment() const noexcept { - return {this->m_ptr_fragment, static_cast(this->m_len_fragment)}; + return make_part_view(this->m_ptr_fragment, this->m_len_fragment); } /*------------------------------------------------------------------------- diff --git a/src/proxy/hdrs/XPACK.cc b/src/proxy/hdrs/XPACK.cc index 78acdf056d8..a45d2185b55 100644 --- a/src/proxy/hdrs/XPACK.cc +++ b/src/proxy/hdrs/XPACK.cc @@ -78,12 +78,18 @@ xpack_decode_integer(uint64_t &dst, const uint8_t *buf_start, const uint8_t *buf } uint64_t added_value = *p & 0x7f; - if ((UINT64_MAX >> m) < added_value) { + if ((m >= 64) || ((UINT64_MAX >> m) < added_value)) { // Excessively large integer encodings - in value or octet // length - MUST be treated as a decoding error. return XPACK_ERROR_COMPRESSION_ERROR; } - dst += added_value << m; + uint64_t const shifted = added_value << m; + if (dst > (UINT64_MAX - shifted)) { + // Excessively large integer encodings - in value or octet + // length - MUST be treated as a decoding error. + return XPACK_ERROR_COMPRESSION_ERROR; + } + dst += shifted; m += 7; } while (*p & 0x80); } @@ -96,7 +102,8 @@ xpack_decode_integer(uint64_t &dst, const uint8_t *buf_start, const uint8_t *buf // return content from String Data (Length octets) with huffman decoding if it is encoded // int64_t -xpack_decode_string(Arena &arena, char **str, uint64_t &str_length, const uint8_t *buf_start, const uint8_t *buf_end, uint8_t n) +xpack_decode_string(Arena &arena, char **str, uint64_t &str_length, const uint8_t *buf_start, const uint8_t *buf_end, + uint64_t max_string_len, uint8_t n) { if (buf_start >= buf_end) { return XPACK_ERROR_COMPRESSION_ERROR; @@ -117,6 +124,10 @@ xpack_decode_string(Arena &arena, char **str, uint64_t &str_length, const uint8_ return XPACK_ERROR_COMPRESSION_ERROR; } + if (encoded_string_len > max_string_len) { + return XPACK_ERROR_COMPRESSION_ERROR; + } + if (isHuffman) { // Allocate temporary area twice the size of before decoded data uint32_t const str_len = encoded_string_len * 2; diff --git a/src/proxy/hdrs/unit_tests/test_HdrHeap.cc b/src/proxy/hdrs/unit_tests/test_HdrHeap.cc index c2017e714c3..22a5fb46eb7 100644 --- a/src/proxy/hdrs/unit_tests/test_HdrHeap.cc +++ b/src/proxy/hdrs/unit_tests/test_HdrHeap.cc @@ -231,3 +231,47 @@ TEST_CASE("allocator inuse stays balanced across freelist reuse", "[proxy][hdrhe } #endif // TS_USE_ALLOCATOR_METRICS + +TEST_CASE("HdrHeap check_marshalled rejects corrupt data", "[proxy][hdrheap]") +{ + HdrHeap *heap = new_HdrHeap(); + URLImpl *url = url_create(heap); + url->set_path(heap, {"/test", 5}, true); + + char buf[2048]; + int len = heap->marshal(buf, sizeof(buf)); + REQUIRE(len > 0); + + auto *hdr = reinterpret_cast(buf); + + SECTION("valid marshalled buffer passes") + { + CHECK(hdr->check_marshalled(len) == true); + } + + SECTION("negative m_heap_len is rejected") + { + hdr->m_ronly_heap[0].m_heap_len = -1; + CHECK(hdr->check_marshalled(len) == false); + } + + SECTION("m_heap_len exceeding buf_length is rejected") + { + hdr->m_ronly_heap[0].m_heap_len = len; + CHECK(hdr->check_marshalled(len) == false); + } + + SECTION("m_heap_start inconsistent with m_size is rejected") + { + hdr->m_ronly_heap[0].m_heap_start = reinterpret_cast(static_cast(hdr->m_size + 1)); + CHECK(hdr->check_marshalled(len) == false); + } + + SECTION("m_size smaller than HDR_HEAP_HDR_SIZE is rejected") + { + hdr->m_size = 1; + CHECK(hdr->check_marshalled(len) == false); + } + + heap->destroy(); +} diff --git a/src/proxy/hdrs/unit_tests/test_Hdrs.cc b/src/proxy/hdrs/unit_tests/test_Hdrs.cc index 4e90846edb4..92a92c82623 100644 --- a/src/proxy/hdrs/unit_tests/test_Hdrs.cc +++ b/src/proxy/hdrs/unit_tests/test_Hdrs.cc @@ -55,32 +55,37 @@ TEST_CASE("HdrTestHttpParse", "[proxy][hdrtest]") }; static const std::vector http_parse_tests = { - {"GET /index.html HTTP/1.0\r\n", ParseResult::DONE, 26}, - {"GET /index.html HTTP/1.0\r\n\r\n***BODY****", ParseResult::DONE, 28}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n***BODY****", ParseResult::DONE, 48}, - {"GET", ParseResult::ERROR, 3 }, - {"GET /index.html", ParseResult::ERROR, 15}, - {"GET /index.html\r\n", ParseResult::ERROR, 17}, - {"GET /index.html HTTP/1.0", ParseResult::ERROR, 24}, - {"GET /index.html HTTP/1.0\r", ParseResult::ERROR, 25}, - {"GET /index.html HTTP/1.0\n", ParseResult::DONE, 25}, - {"GET /index.html HTTP/1.0\n\n", ParseResult::DONE, 26}, - {"GET /index.html HTTP/1.0\r\n\r\n", ParseResult::DONE, 28}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar", ParseResult::ERROR, 44}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\n", ParseResult::DONE, 45}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", ParseResult::DONE, 46}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n", ParseResult::DONE, 48}, - {"GET /index.html HTTP/1.0\nUser-Agent: foobar\n", ParseResult::DONE, 44}, - {"GET /index.html HTTP/1.0\nUser-Agent: foobar\nBoo: foo\n", ParseResult::DONE, 53}, - {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", ParseResult::DONE, 46}, - {"GET /index.html HTTP/1.0\r\n", ParseResult::DONE, 26}, - {"GET /index.html hTTP/1.0\r\n", ParseResult::ERROR, 26}, - {"POST /index.html HTTP/1.0\r\nContent-Length: 0\r\n\r\n", ParseResult::DONE, 48}, - {"POST /index.html HTTP/1.0\r\nContent-Length: \r\n\r\n", ParseResult::ERROR, 47}, - {"POST /index.html HTTP/1.0\r\nContent-Length:\r\n\r\n", ParseResult::ERROR, 46}, - {"CONNECT foo.example HTTP/1.1\r\n", ParseResult::DONE, 30}, - {"GET foo.example HTTP/1.1\r\n", ParseResult::ERROR, 26}, - {"", ParseResult::ERROR, 0 }, + {"GET /index.html HTTP/1.0\r\n", ParseResult::DONE, 26}, + {"GET /index.html HTTP/1.0\r\n\r\n***BODY****", ParseResult::DONE, 28}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n***BODY****", ParseResult::DONE, 48}, + {"GET", ParseResult::ERROR, 3 }, + {"GET /index.html", ParseResult::ERROR, 15}, + {"GET /index.html\r\n", ParseResult::ERROR, 17}, + {"GET /index.html HTTP/1.0", ParseResult::ERROR, 24}, + {"GET /index.html HTTP/1.0\r", ParseResult::ERROR, 25}, + {"GET /index.html HTTP/1.0\n", ParseResult::DONE, 25}, + {"GET /index.html HTTP/1.0\n\n", ParseResult::DONE, 26}, + {"GET /index.html HTTP/1.0\r\n\r\n", ParseResult::DONE, 28}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar", ParseResult::ERROR, 44}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\n", ParseResult::DONE, 45}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", ParseResult::DONE, 46}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n\r\n", ParseResult::DONE, 48}, + {"GET /index.html HTTP/1.0\nUser-Agent: foobar\n", ParseResult::DONE, 44}, + {"GET /index.html HTTP/1.0\nUser-Agent: foobar\nBoo: foo\n", ParseResult::DONE, 53}, + {"GET /index.html HTTP/1.0\r\nUser-Agent: foobar\r\n", ParseResult::DONE, 46}, + {"GET /index.html HTTP/1.0\r\n", ParseResult::DONE, 26}, + {"GET /index.html hTTP/1.0\r\n", ParseResult::ERROR, 26}, + {"POST /index.html HTTP/1.0\r\nContent-Length: 0\r\n\r\n", ParseResult::DONE, 48}, + {"POST /index.html HTTP/1.0\r\nContent-Length: 2147483648\r\n\r\n", ParseResult::DONE, 57}, + {"POST /index.html HTTP/1.0\r\nContent-Length: 9223372036854775807\r\n\r\n", ParseResult::DONE, 66}, + {"POST /index.html HTTP/1.0\r\nContent-Length: 9223372036854775808\r\n\r\n", ParseResult::ERROR, 66}, + {"POST /index.html HTTP/1.0\r\nContent-Length: 99999999999999999999\r\n\r\n", ParseResult::ERROR, 67}, + {"POST /index.html HTTP/1.0\r\nContent-Length: -1\r\n\r\n", ParseResult::ERROR, 49}, + {"POST /index.html HTTP/1.0\r\nContent-Length: \r\n\r\n", ParseResult::ERROR, 47}, + {"POST /index.html HTTP/1.0\r\nContent-Length:\r\n\r\n", ParseResult::ERROR, 46}, + {"CONNECT foo.example HTTP/1.1\r\n", ParseResult::DONE, 30}, + {"GET foo.example HTTP/1.1\r\n", ParseResult::ERROR, 26}, + {"", ParseResult::ERROR, 0 }, }; auto test = GENERATE(from_range(http_parse_tests)); @@ -104,6 +109,71 @@ TEST_CASE("HdrTestHttpParse", "[proxy][hdrtest]") req_hdr.destroy(); } +TEST_CASE("HTTPHdr destroy clears cached request URL", "[proxy][hdrtest]") +{ + constexpr swoc::TextView msg = "GET /index.html HTTP/1.1\r\nHost: example.com\r\n\r\n"sv; + + HTTPParser parser; + http_parser_init(&parser); + + HTTPHdr req_hdr; + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); + + req_hdr.create(HTTPType::REQUEST, HTTP_1_1, heap); + + auto start = msg.data(); + REQUIRE(req_hdr.parse_req(&parser, &start, msg.data_end(), true) == ParseResult::DONE); + + // Force population of m_url_cached so it holds a pointer into the heap. + URL *u = req_hdr.url_get(); + REQUIRE(u != nullptr); + REQUIRE(u->valid()); + REQUIRE(req_hdr.m_url_cached.valid()); + + // Force population of the target cache so m_host_mime points into the heap. + std::string_view host = req_hdr.host_get(); + REQUIRE(!host.empty()); + REQUIRE(req_hdr.m_target_cached); + + req_hdr.destroy(); + + // After destroy(), no member should reference the freed heap. + REQUIRE(req_hdr.m_http == nullptr); + REQUIRE(req_hdr.m_mime == nullptr); + REQUIRE_FALSE(req_hdr.m_url_cached.valid()); + REQUIRE(req_hdr.m_host_mime == nullptr); + REQUIRE_FALSE(req_hdr.m_target_cached); +} + +// A single header field whose name or value exceeds the uniform UINT16_MAX +// field-size limit must be rejected by the parser. m_len_name is uint16_t, so an +// oversized name would truncate its stored length; m_len_value is uint32_t : 24 +// and could hold more, but the value is capped at the same limit. The caller +// (HttpSM) turns the parse error into a 400. +TEST_CASE("HdrTestOversizedFieldRejected", "[proxy][hdrtest]") +{ + HTTPParser parser; + http_parser_init(&parser); + + HTTPHdr req_hdr; + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); + + req_hdr.create(HTTPType::REQUEST, HTTP_1_1, heap); + + // 70000-byte value: name+value stays under the default max_hdr_field_size + // (131070) but the value alone exceeds UINT16_MAX (65535). + std::string msg = "GET /index.html HTTP/1.0\r\nX-Big: " + std::string(70000, 'a') + "\r\n\r\n"; + swoc::TextView tv{msg}; + + auto start = tv.data(); + auto ret = req_hdr.parse_req(&parser, &start, tv.data_end(), true); + + REQUIRE(ret == ParseResult::ERROR); + + req_hdr.destroy(); + http_parser_clear(&parser); +} + TEST_CASE("MIMEScanner_fragments", "[proxy][mimescanner_fragments]") { constexpr swoc::TextView const message = "GET /index.html HTTP/1.0\r\n"; @@ -138,6 +208,44 @@ TEST_CASE("MIMEScanner_fragments", "[proxy][mimescanner_fragments]") REQUIRE(message == output); } +TEST_CASE("MIMEScanner obs-fold single buffer", "[proxy][mimescanner_obsfold]") +{ + MIMEScanner scanner; + swoc::TextView output; + bool shares_input = true; + + char buf[] = "Foo: bar\r\n baz\r\n"; + auto input = swoc::TextView{buf}; + auto result = scanner.get(input, output, shares_input, false, MIMEScanner::ScanType::FIELD); + + REQUIRE(result == ParseResult::OK); + REQUIRE(output == "Foo: bar baz\r\n"sv); +} + +TEST_CASE("MIMEScanner obs-fold CR/LF split across fragments", "[proxy][mimescanner_obsfold]") +{ + // CR at the end of fragment 1, LF + continuation whitespace at the start + // of fragment 2. The unfold code must not walk backwards past the start of + // the second fragment's buffer. + MIMEScanner scanner; + swoc::TextView output; + bool shares_input = true; + + char first[] = "Foo: bar\r"; + auto input = swoc::TextView{first}; + auto result = scanner.get(input, output, shares_input, false, MIMEScanner::ScanType::FIELD); + REQUIRE(result == ParseResult::CONT); + + char second[] = "\n baz\r\n"; + input = swoc::TextView{second}; + shares_input = true; + result = scanner.get(input, output, shares_input, false, MIMEScanner::ScanType::FIELD); + + REQUIRE(result == ParseResult::OK); + REQUIRE(shares_input == false); + REQUIRE(output == "Foo: bar baz\r\n"sv); +} + namespace { static const char * @@ -2696,3 +2804,118 @@ TEST_CASE("HdrPromotesOnlyValidHostHeaderMutations", "[proxy][hdrtest]") http_parser_clear(&parser); req_hdr.destroy(); } + +TEST_CASE("http_parse_status overflow protection", "[proxy][hdrtest]") +{ + SECTION("valid 3-digit status codes") + { + std::string_view s200 = "200"; + CHECK(http_parse_status(s200.data(), s200.data() + s200.size()) == static_cast(200)); + + std::string_view s404 = "404"; + CHECK(http_parse_status(s404.data(), s404.data() + s404.size()) == static_cast(404)); + + std::string_view s999 = "999"; + CHECK(http_parse_status(s999.data(), s999.data() + s999.size()) == static_cast(999)); + } + + SECTION("4-digit status returns NONE") + { + std::string_view s1000 = "1000"; + CHECK(http_parse_status(s1000.data(), s1000.data() + s1000.size()) == HTTPStatus::NONE); + + std::string_view s9999 = "9999"; + CHECK(http_parse_status(s9999.data(), s9999.data() + s9999.size()) == HTTPStatus::NONE); + } + + SECTION("very long digit sequence returns NONE") + { + std::string s_long = "99999999999999999999"; + CHECK(http_parse_status(s_long.data(), s_long.data() + s_long.size()) == HTTPStatus::NONE); + } + + SECTION("empty and zero") + { + std::string_view empty = ""; + CHECK(http_parse_status(empty.data(), empty.data()) == HTTPStatus::NONE); + + std::string_view zero = "0"; + CHECK(http_parse_status(zero.data(), zero.data() + zero.size()) == static_cast(0)); + } + + SECTION("leading spaces are skipped") + { + std::string_view s = " 200"; + CHECK(http_parse_status(s.data(), s.data() + s.size()) == static_cast(200)); + } + + SECTION("high-bit bytes are rejected without UB") + { + char buf[] = {'\xff', '\xff', '\xff'}; + CHECK(http_parse_status(buf, buf + sizeof(buf)) == static_cast(0)); + } +} + +TEST_CASE("HTTP parser tolerates high-bit bytes without UB", "[proxy][hdrtest]") +{ + struct Test { + std::string_view msg; + ParseResult expected_result; + }; + + static const std::vector tests = { + {"GET /x HTTP/\xff.0\r\n\r\n"sv, ParseResult::ERROR}, + {"GET /x HTTP/1.\xff\r\n\r\n"sv, ParseResult::ERROR}, + {"GET /\xff HTTP/1.0\r\n\r\n"sv, ParseResult::DONE }, + }; + + auto test = GENERATE(from_range(tests)); + CAPTURE(test.expected_result); + + HTTPParser parser; + http_parser_init(&parser); + + HTTPHdr req_hdr; + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); + + req_hdr.create(HTTPType::REQUEST, HTTP_1_1, heap); + + auto start = test.msg.data(); + auto ret = req_hdr.parse_req(&parser, &start, test.msg.data() + test.msg.size(), true); + + CHECK(ret == test.expected_result); + + req_hdr.destroy(); +} + +TEST_CASE("HTTP response parser tolerates high-bit bytes without UB", "[proxy][hdrtest]") +{ + struct Test { + std::string_view msg; + ParseResult expected_result; + }; + + static const std::vector tests = { + {"HTTP/\xff.0 200 OK\r\n\r\n"sv, ParseResult::ERROR}, + {"HTTP/1.\xff 200 OK\r\n\r\n"sv, ParseResult::ERROR}, + {"HTTP/1.0 \xff\xff\xff OK\r\n\r\n"sv, ParseResult::DONE }, + }; + + auto test = GENERATE(from_range(tests)); + CAPTURE(test.expected_result); + + HTTPParser parser; + http_parser_init(&parser); + + HTTPHdr resp_hdr; + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); + + resp_hdr.create(HTTPType::RESPONSE, HTTP_1_1, heap); + + auto start = test.msg.data(); + auto ret = resp_hdr.parse_resp(&parser, &start, test.msg.data() + test.msg.size(), true); + + CHECK(ret == test.expected_result); + + resp_hdr.destroy(); +} diff --git a/src/proxy/hdrs/unit_tests/test_HeaderValidator.cc b/src/proxy/hdrs/unit_tests/test_HeaderValidator.cc index 75c90404cec..1cec307148d 100644 --- a/src/proxy/hdrs/unit_tests/test_HeaderValidator.cc +++ b/src/proxy/hdrs/unit_tests/test_HeaderValidator.cc @@ -245,6 +245,19 @@ TEST_CASE("testIsHeaderValid", "[proxy][hdrtest]") // Connection-specific headers are not allowed. check_header(fields, hdr, !IS_VALID_HEADER); } + SECTION("Test request with Transfer-Encoding headers") + { + hdr.create(HTTPType::REQUEST, HTTP_1_1, heap); + Fields_type fields = { + {":method", "POST" }, + {":scheme", "https" }, + {":authority", "www.this.com"}, + {":path", "/some/path" }, + {"Transfer-Encoding", "chunked" }, + }; + // Connection-specific headers are not allowed. + check_header(fields, hdr, !IS_VALID_HEADER); + } // teardown hdr.destroy(); // coverity[leaked_storage] - heap is freed via hdr.destroy() which calls diff --git a/src/proxy/hdrs/unit_tests/test_URL.cc b/src/proxy/hdrs/unit_tests/test_URL.cc index dc5ff4ade74..f1963c9039f 100644 --- a/src/proxy/hdrs/unit_tests/test_URL.cc +++ b/src/proxy/hdrs/unit_tests/test_URL.cc @@ -322,6 +322,44 @@ std::vector url_parse_test_cases = { IS_VALID, IS_VALID }, + { + // Maximum value that fits in the 16-bit port field; preserved verbatim. + "https://www.example.com:65535/", + "https://www.example.com:65535/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com:65535/", + IS_VALID, + IS_VALID + }, + { + // One past the 16-bit boundary: the port is rejected and the URL is + // emitted with no explicit port so default-port logic applies. + "https://www.example.com:65536/", + "https://www.example.com/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/", + IS_VALID, + IS_VALID + }, + { + // Five-digit value above the 16-bit range. + "https://www.example.com:99999/", + "https://www.example.com/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/", + IS_VALID, + IS_VALID + }, + { + // Six-digit value whose low 16 bits coincide with a well-known port (80); + // must not be silently retained as if the user had asked for that port. + "https://www.example.com:131152/", + "https://www.example.com/", + VERIFY_HOST_CHARACTERS, + "https://www.example.com/", + IS_VALID, + IS_VALID + }, { "https://www.example.com/a/path", "https://www.example.com/a/path", @@ -569,6 +607,74 @@ TEST_CASE("UrlParse", "[proxy][parseurl]") test_parse(test_case, URL_PARSE_REGEX); } +TEST_CASE("UrlParsePortStorage", "[proxy][parseurl]") +{ + // Validate the underlying URLImpl port storage rather than the printed form, + // so the rejection path cannot leave the parsed text behind even when other + // serialization paths only inspect m_ptr_port. + struct Case { + std::string input_uri; + int expected_port; // numeric m_port value + bool expect_port_text; // true if m_ptr_port should be non-null + }; + + // clang-format off + static const std::vector cases = { + // In-range ports keep both the numeric value and the parsed text. + {"https://www.example.com:8080/", 8080, true }, + {"https://www.example.com:65535/", 65535, true }, + // Out-of-range ports clear both. + {"https://www.example.com:65536/", 0, false}, + {"https://www.example.com:99999/", 0, false}, + {"https://www.example.com:131152/", 0, false}, + }; + // clang-format on + + auto c = GENERATE(from_range(cases)); + CAPTURE(c.input_uri, c.expected_port, c.expect_port_text); + + URL url; + HdrHeap *heap = new_HdrHeap(); + url.create(heap); + REQUIRE(url.parse(c.input_uri) == ParseResult::DONE); + + CHECK(url.m_url_impl->m_port == c.expected_port); + if (c.expect_port_text) { + CHECK(url.m_url_impl->m_ptr_port != nullptr); + CHECK(url.m_url_impl->m_len_port > 0); + } else { + CHECK(url.m_url_impl->m_ptr_port == nullptr); + CHECK(url.m_url_impl->m_len_port == 0); + } + + heap->destroy(); +} + +TEST_CASE("UrlPrintLowerCaseSchemeHostHandlesHighBitBytes", "[proxy][urlprint]") +{ + constexpr auto HIGH_ORDER_BIT = static_cast(0x80); + + std::string input_uri{"HTTP://High"}; + input_uri.push_back(HIGH_ORDER_BIT); + input_uri.append(".Example/path"); + + std::string expected_url{"http://high"}; + expected_url.push_back(HIGH_ORDER_BIT); + expected_url.append(".example/path"); + + URL url; + HdrHeap *heap = new_HdrHeap(); + url.create(heap); + REQUIRE(url.parse_no_host_check(input_uri) == ParseResult::DONE); + + int length = 0; + char *printed_url = url.string_get_ref(&length, URLNormalize::LC_SCHEME_HOST); + REQUIRE(printed_url != nullptr); + CHECK(std::string_view{printed_url, static_cast(length)} == expected_url); + + heap->destroy(); +} + struct get_hash_test_case { const std::string description; const std::string uri_1; @@ -745,3 +851,39 @@ TEST_CASE("UrlPathGet", "[url][path_get]") } } } + +// URL getters must not construct std::string_view from a nullptr pointer +// (which is UB). Parts that are not present in the URL should return an +// empty string_view with data() == nullptr. +TEST_CASE("UrlMissingParts", "[url][missing_parts]") +{ + URL url; + HdrHeap *heap = new_HdrHeap(); + url.create(heap); + + // A freshly created URL with no parse has no components set. + auto scheme{url.scheme_get()}; + auto user{url.user_get()}; + auto password{url.password_get()}; + auto host{url.host_get()}; + auto path{url.path_get()}; + auto query{url.query_get()}; + auto fragment{url.fragment_get()}; + + CHECK(scheme.empty()); + CHECK(scheme.data() == nullptr); + CHECK(user.empty()); + CHECK(user.data() == nullptr); + CHECK(password.empty()); + CHECK(password.data() == nullptr); + CHECK(host.empty()); + CHECK(host.data() == nullptr); + CHECK(path.empty()); + CHECK(path.data() == nullptr); + CHECK(query.empty()); + CHECK(query.data() == nullptr); + CHECK(fragment.empty()); + CHECK(fragment.data() == nullptr); + + heap->destroy(); +} diff --git a/src/proxy/hdrs/unit_tests/test_XPACK.cc b/src/proxy/hdrs/unit_tests/test_XPACK.cc index c0f9a40d639..e7ed4a875c7 100644 --- a/src/proxy/hdrs/unit_tests/test_XPACK.cc +++ b/src/proxy/hdrs/unit_tests/test_XPACK.cc @@ -29,7 +29,8 @@ #include "proxy/hdrs/XPACK.h" #include "proxy/hdrs/HuffmanCodec.h" -static constexpr int BUFSIZE_FOR_REGRESSION_TEST = 128; +static constexpr int BUFSIZE_FOR_REGRESSION_TEST = 128; +static constexpr uint64_t MAX_FIELD_SIZE = 32768; std::string get_long_string(int size) @@ -79,6 +80,24 @@ TEST_CASE("XPACK_Integer", "[xpack]") REQUIRE(actual == i.raw_integer); } } + + SECTION("Decoding rejects integer overflow") + { + const uint8_t encoded_field[] = {0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01}; + uint64_t actual = 0; + int64_t len = xpack_decode_integer(actual, encoded_field, encoded_field + sizeof(encoded_field), 5); + + REQUIRE(len == XPACK_ERROR_COMPRESSION_ERROR); + } + + SECTION("Decoding rejects overlong integer encodings") + { + const uint8_t encoded_field[] = {0x1f, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00}; + uint64_t actual = 0; + int64_t len = xpack_decode_integer(actual, encoded_field, encoded_field + sizeof(encoded_field), 5); + + REQUIRE(len == XPACK_ERROR_COMPRESSION_ERROR); + } } TEST_CASE("XPACK_String", "[xpack]") @@ -127,7 +146,8 @@ TEST_CASE("XPACK_String", "[xpack]") Arena arena; char *actual = nullptr; uint64_t actual_len = 0; - int len = xpack_decode_string(arena, &actual, actual_len, i.encoded_field, i.encoded_field + i.encoded_field_len); + int len = + xpack_decode_string(arena, &actual, actual_len, i.encoded_field, i.encoded_field + i.encoded_field_len, MAX_FIELD_SIZE); REQUIRE(len == i.encoded_field_len); REQUIRE(actual_len == i.raw_string_len); @@ -135,6 +155,87 @@ TEST_CASE("XPACK_String", "[xpack]") } } + SECTION("max_string_len enforcement") + { + // "custom-key" (10 bytes), non-huffman encoded: length byte 0x0a + raw string + uint8_t encoded[] = "\x0a" + "custom-key"; + int encoded_len = 11; + + SECTION("exact limit allows decoding") + { + Arena arena; + char *actual = nullptr; + uint64_t actual_len = 0; + int len = xpack_decode_string(arena, &actual, actual_len, encoded, encoded + encoded_len, 10); + + REQUIRE(len == encoded_len); + REQUIRE(actual_len == 10); + REQUIRE(memcmp(actual, "custom-key", 10) == 0); + } + + SECTION("limit below string length rejects") + { + Arena arena; + char *actual = nullptr; + uint64_t actual_len = 0; + int len = xpack_decode_string(arena, &actual, actual_len, encoded, encoded + encoded_len, 9); + + REQUIRE(len == XPACK_ERROR_COMPRESSION_ERROR); + } + + SECTION("zero limit rejects non-empty string") + { + Arena arena; + char *actual = nullptr; + uint64_t actual_len = 0; + int len = xpack_decode_string(arena, &actual, actual_len, encoded, encoded + encoded_len, 0); + + REQUIRE(len == XPACK_ERROR_COMPRESSION_ERROR); + } + + SECTION("huffman-encoded string checked against limit") + { + // "custom-key" huffman-encoded: 0x88 (huffman flag + length 8) + 8 bytes + uint8_t huff_encoded[] = "\x88\x25\xa8\x49\xe9\x5b\xa9\x7d\x7f"; + int huff_encoded_len = 9; + + SECTION("limit above encoded length allows") + { + Arena arena; + char *actual = nullptr; + uint64_t actual_len = 0; + int len = xpack_decode_string(arena, &actual, actual_len, huff_encoded, huff_encoded + huff_encoded_len, 8); + + REQUIRE(len == huff_encoded_len); + REQUIRE(actual_len == 10); + REQUIRE(memcmp(actual, "custom-key", 10) == 0); + } + + SECTION("limit below encoded length rejects") + { + Arena arena; + char *actual = nullptr; + uint64_t actual_len = 0; + int len = xpack_decode_string(arena, &actual, actual_len, huff_encoded, huff_encoded + huff_encoded_len, 7); + + REQUIRE(len == XPACK_ERROR_COMPRESSION_ERROR); + } + } + + SECTION("empty string with any limit succeeds") + { + uint8_t empty_encoded[] = "\x0"; + Arena arena; + char *actual = nullptr; + uint64_t actual_len = 0; + int len = xpack_decode_string(arena, &actual, actual_len, empty_encoded, empty_encoded + 1, 0); + + REQUIRE(len == 1); + REQUIRE(actual_len == 0); + } + } + SECTION("Zero-size Dynamic Table") { XpackDynamicTable dt(0); diff --git a/src/proxy/hdrs/unit_tests/test_mime.cc b/src/proxy/hdrs/unit_tests/test_mime.cc index 857f008ae0e..6a6d067a0c1 100644 --- a/src/proxy/hdrs/unit_tests/test_mime.cc +++ b/src/proxy/hdrs/unit_tests/test_mime.cc @@ -22,6 +22,8 @@ */ #include +#include +#include #include @@ -31,7 +33,26 @@ using namespace std::literals; #include #include #include "tscore/ink_platform.h" +#include "tscore/Diags.h" +#include "tscore/BaseLogFile.h" #include "proxy/hdrs/MIME.h" +#include "proxy/hdrs/HdrHeap.h" + +namespace +{ +// Build a filler string of `n` copies of `c`. The length is laundered through a +// volatile so GCC at -O3 cannot constant-fold it into the inlined +// hdrtoken_wks_to_index() lookup performed by the field setters. With a known +// constant size GCC emits a -Warray-bounds false positive (a [-1] subscript it +// cannot rule out through the hdrtoken_is_wks() guard, which does prevent the +// access at run time). clang does not warn; the volatile keeps both happy. +std::string +mime_filler(std::size_t n, char c) +{ + volatile std::size_t len = n; + return std::string(len, c); +} +} // namespace TEST_CASE("Mime", "[proxy][mime]") { @@ -209,6 +230,45 @@ TEST_CASE("MimeParsers", "[proxy][mimeparsers]") CHECK(value == val); } +TEST_CASE("MimeParseInt64Overflow", "[proxy][mimeparseint64]") +{ + static const std::vector> tests = { + {"0", 0 }, + {"12345", 12345 }, + {"-12345", -12345 }, + {"9223372036854775807", INT64_MAX}, + {"-9223372036854775808", INT64_MIN}, + {"9223372036854775808", INT64_MAX}, + {"-9223372036854775809", INT64_MIN}, + {"99999999999999999999", INT64_MAX}, + {" 42", 42 }, + }; + + auto [buf, val] = GENERATE(from_range(tests)); + CAPTURE(buf, val); + + const char *end = buf + strlen(buf); + CHECK(mime_parse_int64(buf, end) == val); +} + +TEST_CASE("MimeParseUintOverflow", "[proxy][mimeparseuint]") +{ + static const std::vector> tests = { + {"0", 0 }, + {"12345", 12345 }, + {"4294967295", UINT32_MAX}, + {"4294967296", UINT32_MAX}, + {"9999999999", UINT32_MAX}, + {" 42", 42 }, + }; + + auto [buf, val] = GENERATE(from_range(tests)); + CAPTURE(buf, val); + + const char *end = buf + strlen(buf); + CHECK(mime_parse_uint(buf, end) == val); +} + TEST_CASE("MimeDateParser", "[proxy][mimedateparser]") { const char *date1 = "Sun, 05 Dec 1999 08:49:37 GMT"; @@ -219,3 +279,316 @@ TEST_CASE("MimeDateParser", "[proxy][mimedateparser]") CHECK(d1 == d2); } + +// m_len_name is uint16_t (m_len_value is uint32_t : 24 and could hold more, but +// is capped at UINT16_MAX for a uniform limit). A name of 65536+n bytes used to +// wrap to length n while the byte pointer still spanned the full input, so the +// stored length no longer matched the data. mime_str_u16_set() and the no-copy +// branch of mime_field_name_value_set() now reject oversized strings. +TEST_CASE("MimeOversizedNameValueRejected", "[proxy][mime]") +{ + // Storing oversized strings calls Warning(), which dereferences diags(). Make + // sure diags is initialized so these calls do not crash under the test binary. + [[maybe_unused]] static bool diags_initialized = []() { + if (diags() == nullptr) { + DiagsPtr::set(new Diags("test_mime", nullptr, nullptr, new BaseLogFile("stderr"))); + } + return true; + }(); + + // 70000 bytes > UINT16_MAX (65535). On unpatched code, length wraps to 4464. + std::string const big = mime_filler(70000, 'a'); + std::string_view big_sv{big}; + + SECTION("mime_str_u16_set rejects oversized input and clears length") + { + HdrHeap *heap = new_HdrHeap(); + const char *d_str = nullptr; + uint16_t d_len = 0; + + const char *ret = mime_str_u16_set(heap, big_sv, &d_str, &d_len, true); + + REQUIRE(ret == nullptr); + REQUIRE(d_str == nullptr); + REQUIRE(d_len == 0); + + heap->destroy(); + } + + SECTION("mime_str_u16_set rejects oversized input on no-copy path too") + { + HdrHeap *heap = new_HdrHeap(); + const char *d_str = nullptr; + uint16_t d_len = 0; + + const char *ret = mime_str_u16_set(heap, big_sv, &d_str, &d_len, false); + + REQUIRE(ret == nullptr); + REQUIRE(d_str == nullptr); + REQUIRE(d_len == 0); + + heap->destroy(); + } + + SECTION("MIMEHdr::name_set with oversized name preserves the existing name") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + bool const stored = field->name_set(hdr.m_heap, hdr.m_mime, big_sv); + + // A rejected rename is a no-op: the prior name is left intact. + REQUIRE(stored == false); + REQUIRE(field->name_get() == "X-Test"sv); + + hdr.destroy(); + } + + SECTION("MIMEHdr value_set with oversized value leaves stored value empty") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + field->value_set(hdr.m_heap, hdr.m_mime, big_sv); + + auto value = field->value_get(); + REQUIRE(value.length() == 0); + + hdr.destroy(); + } + + SECTION("mime_field_name_value_set no-copy branch rejects oversized name") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = mime_field_create(hdr.m_heap, hdr.m_mime); + REQUIRE(field != nullptr); + + int const raw_len = static_cast(big_sv.length()) + 4; + mime_field_name_value_set(hdr.m_heap, hdr.m_mime, field, -1, big_sv, "v"sv, 1, raw_len, false); + + REQUIRE(field->m_len_name == 0); + REQUIRE(field->m_len_value == 0); + + hdr.destroy(); + } + + SECTION("mime_field_name_value_set no-copy branch rejects oversized value") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = mime_field_create(hdr.m_heap, hdr.m_mime); + REQUIRE(field != nullptr); + + int const raw_len = static_cast(big_sv.length()) + 4; + mime_field_name_value_set(hdr.m_heap, hdr.m_mime, field, -1, "X-Test"sv, big_sv, 1, raw_len, false); + + REQUIRE(field->m_len_name == 0); + REQUIRE(field->m_len_value == 0); + + hdr.destroy(); + } +} + +// These tests verify that the setters return bool (true = stored, false = +// rejected as oversized with the field left empty). +// +// The header field-size limit is a uniform UINT16_MAX (65535) cap on both the +// name and the value: UINT16_MAX is the largest accepted length and 65536 and +// above are rejected. (m_len_name is uint16_t; m_len_value is uint32_t : 24 and +// could hold more, but the same cap is applied for consistency.) +TEST_CASE("MimeSetterBoolReturn", "[proxy][mime]") +{ + // Rejecting oversized strings calls Warning(), which dereferences diags(). + // Initialize diags so these calls do not crash under the test binary. + [[maybe_unused]] static bool diags_initialized = []() { + if (diags() == nullptr) { + DiagsPtr::set(new Diags("test_mime", nullptr, nullptr, new BaseLogFile("stderr"))); + } + return true; + }(); + + SECTION("value_set with normal length returns true and stores the value") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + std::string const value = mime_filler(100, 'v'); + std::string_view value_sv{value}; + + bool const stored = field->value_set(hdr.m_heap, hdr.m_mime, value_sv); + + REQUIRE(stored == true); + REQUIRE(field->value_get().length() == 100); + + hdr.destroy(); + } + + SECTION("value_set at the largest accepted length (UINT16_MAX) returns true") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + std::string const value = mime_filler(UINT16_MAX, 'v'); // 65535 + std::string_view value_sv{value}; + + bool const stored = field->value_set(hdr.m_heap, hdr.m_mime, value_sv); + + REQUIRE(stored == true); + REQUIRE(field->value_get().length() == UINT16_MAX); + + hdr.destroy(); + } + + SECTION("value_set at UINT16_MAX+1 returns false and leaves the value empty") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + std::string const value = mime_filler(UINT16_MAX + 1, 'v'); // 65536 + std::string_view value_sv{value}; + + bool const stored = field->value_set(hdr.m_heap, hdr.m_mime, value_sv); + + REQUIRE(stored == false); + REQUIRE(field->value_get().length() == 0); + + hdr.destroy(); + } + + SECTION("value_set above UINT16_MAX returns false and leaves the value empty") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + std::string const value = mime_filler(UINT16_MAX + 5, 'v'); // 65540 + std::string_view value_sv{value}; + + bool const stored = field->value_set(hdr.m_heap, hdr.m_mime, value_sv); + + REQUIRE(stored == false); + REQUIRE(field->value_get().length() == 0); + + hdr.destroy(); + } + + SECTION("oversized value_set is a no-op that preserves the existing value") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + std::string const original = mime_filler(100, 'v'); + REQUIRE(field->value_set(hdr.m_heap, hdr.m_mime, std::string_view{original}) == true); + REQUIRE(field->value_get().length() == 100); + + std::string const oversized = mime_filler(UINT16_MAX + 5, 'V'); // 65540 + bool const stored = field->value_set(hdr.m_heap, hdr.m_mime, std::string_view{oversized}); + + // The rejected set must not disturb the previously stored value. + REQUIRE(stored == false); + REQUIRE(field->value_get() == std::string_view{original}); + + hdr.destroy(); + } + + SECTION("name_set with normal length returns true and stores the name") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + std::string const name = mime_filler(100, 'n'); + std::string_view name_sv{name}; + + bool const stored = field->name_set(hdr.m_heap, hdr.m_mime, name_sv); + + REQUIRE(stored == true); + REQUIRE(field->name_get().length() == 100); + + hdr.destroy(); + } + + SECTION("name_set at the largest accepted length (UINT16_MAX) returns true") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + std::string const name = mime_filler(UINT16_MAX, 'n'); // 65535 + std::string_view name_sv{name}; + + bool const stored = field->name_set(hdr.m_heap, hdr.m_mime, name_sv); + + REQUIRE(stored == true); + REQUIRE(field->name_get().length() == UINT16_MAX); + + hdr.destroy(); + } + + SECTION("name_set at UINT16_MAX+1 returns false and preserves the existing name") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + std::string const name = mime_filler(UINT16_MAX + 1, 'n'); // 65536 + std::string_view name_sv{name}; + + bool const stored = field->name_set(hdr.m_heap, hdr.m_mime, name_sv); + + // The rejected rename must leave the prior name intact. + REQUIRE(stored == false); + REQUIRE(field->name_get() == "X-Test"sv); + + hdr.destroy(); + } + + SECTION("name_set above UINT16_MAX returns false and preserves the existing name") + { + MIMEHdr hdr; + hdr.create(nullptr); + + MIMEField *field = hdr.field_create("X-Test"sv); + REQUIRE(field != nullptr); + + std::string const name = mime_filler(UINT16_MAX + 5, 'n'); // 65540 + std::string_view name_sv{name}; + + bool const stored = field->name_set(hdr.m_heap, hdr.m_mime, name_sv); + + // The rejected rename must leave the prior name intact. + REQUIRE(stored == false); + REQUIRE(field->name_get() == "X-Test"sv); + + hdr.destroy(); + } +} diff --git a/src/proxy/http/ConnectingEntry.cc b/src/proxy/http/ConnectingEntry.cc index 992602439ed..6ca7d9158c6 100644 --- a/src/proxy/http/ConnectingEntry.cc +++ b/src/proxy/http/ConnectingEntry.cc @@ -23,6 +23,7 @@ */ #include "../../iocore/net/P_UnixNetVConnection.h" +#include "../../iocore/net/P_SSLClientUtils.h" #include "tsutil/DbgCtl.h" #include "proxy/http/ConnectingEntry.h" #include "proxy/http/HttpSM.h" @@ -91,14 +92,30 @@ ConnectingEntry::state_http_server_open(int event, void *data) int count = 0; if (new_session->is_multiplexing()) { // Hand off to all queued up ConnectSM's. + bool session_handed_off = false; while (!connect_sms.empty()) { - Dbg(dbg_ctl_http_connect, "ConnectingEntry Pass along CONNECT_EVENT_TXN %d", count++); - auto entry = connect_sms.begin(); - + auto entry = connect_sms.begin(); + auto event = CONNECT_EVENT_TXN; + void *event_data = new_session; + + if (!validate_server_certificate_hostname(new_session->get_netvc(), (*entry)->get_outbound_sni_for_cert_verification())) { + // Retry without joining another multiplexed connect queue so this + // transaction gets its own TLS handshake and certificate check. + event = CONNECT_EVENT_DIRECT; + event_data = nullptr; + } else { + session_handed_off = true; + } + + Dbg(dbg_ctl_http_connect, "ConnectingEntry Pass along %s %d", + event == CONNECT_EVENT_TXN ? "CONNECT_EVENT_TXN" : "CONNECT_EVENT_DIRECT", count++); SCOPED_MUTEX_LOCK(lock, (*entry)->mutex, this_ethread()); - (*entry)->handleEvent(CONNECT_EVENT_TXN, new_session); + (*entry)->handleEvent(event, event_data); connect_sms.erase(entry); } + if (!session_handed_off) { + new_session->do_io_close(); + } } else { // Hand off to one and tell all of the others to connect directly Dbg(dbg_ctl_http_connect, "ConnectingEntry send CONNECT_EVENT_TXN to first %d", count++); diff --git a/src/proxy/http/Http1ClientSession.cc b/src/proxy/http/Http1ClientSession.cc index 999e0c0ad69..e2c11b71d2c 100644 --- a/src/proxy/http/Http1ClientSession.cc +++ b/src/proxy/http/Http1ClientSession.cc @@ -217,6 +217,9 @@ Http1ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB EThread *ethis = this_ethread(); Ptr lmutex = this->mutex; MUTEX_TAKE_LOCK(lmutex, ethis); + if (has_session_hook(TS_HTTP_SSN_START_HOOK)) { + _vc->cancel_inactivity_timeout(); + } do_api_callout(TS_HTTP_SSN_START_HOOK); MUTEX_UNTAKE_LOCK(lmutex, ethis); lmutex.clear(); @@ -427,6 +430,10 @@ Http1ClientSession::release(ProxyTransaction *trans) // Timeout events should be delivered to the session this->do_io_write(this, 0, nullptr); + } else { + HttpConfigParams *params = HttpConfig::acquire(); + set_inactivity_timeout(HRTIME_SECONDS(params->accept_no_activity_timeout)); + HttpConfig::release(params); } h1trans->reset(); diff --git a/src/proxy/http/HttpConfig.cc b/src/proxy/http/HttpConfig.cc index 2bfb487b439..f0b1b535c32 100644 --- a/src/proxy/http/HttpConfig.cc +++ b/src/proxy/http/HttpConfig.cc @@ -28,6 +28,7 @@ #include "tscore/Filenames.h" #include "tscore/Tokenizer.h" #include +#include #include #include "proxy/http/HttpConfig.h" #include "proxy/hdrs/HTTP.h" @@ -331,24 +332,28 @@ register_stat_callbacks() http_rsb.https_total_client_connections = Metrics::Counter::createPtr("proxy.process.https.total_client_connections"); http_rsb.incoming_requests = Metrics::Counter::createPtr("proxy.process.http.incoming_requests"); http_rsb.incoming_responses = Metrics::Counter::createPtr("proxy.process.http.incoming_responses"); - http_rsb.invalid_client_requests = Metrics::Counter::createPtr("proxy.process.http.invalid_client_requests"); - http_rsb.misc_count = Metrics::Counter::createPtr("proxy.process.http.misc_count"); - http_rsb.misc_origin_server_bytes = Metrics::Counter::createPtr("proxy.process.http.http_misc_origin_server_bytes"); - http_rsb.misc_user_agent_bytes = Metrics::Counter::createPtr("proxy.process.http.misc_user_agent_bytes"); - http_rsb.missing_host_hdr = Metrics::Counter::createPtr("proxy.process.http.missing_host_hdr"); - http_rsb.no_remap_matched = Metrics::Counter::createPtr("proxy.process.http.no_remap_matched"); - http_rsb.options_requests = Metrics::Counter::createPtr("proxy.process.http.options_requests"); - http_rsb.origin_body = Metrics::Counter::createPtr("proxy.process.http.origin.body"); - http_rsb.origin_close_private = Metrics::Counter::createPtr("proxy.process.http.origin.close_private"); - http_rsb.origin_connect_adjust_thread = Metrics::Counter::createPtr("proxy.process.http.origin.connect.adjust_thread"); - http_rsb.origin_connections_throttled = Metrics::Counter::createPtr("proxy.process.http.origin_connections_throttled_out"); - http_rsb.origin_make_new = Metrics::Counter::createPtr("proxy.process.http.origin.make_new"); - http_rsb.origin_no_sharing = Metrics::Counter::createPtr("proxy.process.http.origin.no_sharing"); - http_rsb.origin_not_found = Metrics::Counter::createPtr("proxy.process.http.origin.not_found"); - http_rsb.origin_private = Metrics::Counter::createPtr("proxy.process.http.origin.private"); - http_rsb.origin_raw = Metrics::Counter::createPtr("proxy.process.http.origin.raw"); - http_rsb.origin_reuse = Metrics::Counter::createPtr("proxy.process.http.origin.reuse"); - http_rsb.origin_reuse_fail = Metrics::Counter::createPtr("proxy.process.http.origin.reuse_fail"); + http_rsb.client_request_at_headers_stripped = + Metrics::Counter::createPtr("proxy.process.http.client_request_at_headers_stripped"); + http_rsb.origin_response_at_headers_stripped = + Metrics::Counter::createPtr("proxy.process.http.origin_response_at_headers_stripped"); + http_rsb.invalid_client_requests = Metrics::Counter::createPtr("proxy.process.http.invalid_client_requests"); + http_rsb.misc_count = Metrics::Counter::createPtr("proxy.process.http.misc_count"); + http_rsb.misc_origin_server_bytes = Metrics::Counter::createPtr("proxy.process.http.http_misc_origin_server_bytes"); + http_rsb.misc_user_agent_bytes = Metrics::Counter::createPtr("proxy.process.http.misc_user_agent_bytes"); + http_rsb.missing_host_hdr = Metrics::Counter::createPtr("proxy.process.http.missing_host_hdr"); + http_rsb.no_remap_matched = Metrics::Counter::createPtr("proxy.process.http.no_remap_matched"); + http_rsb.options_requests = Metrics::Counter::createPtr("proxy.process.http.options_requests"); + http_rsb.origin_body = Metrics::Counter::createPtr("proxy.process.http.origin.body"); + http_rsb.origin_close_private = Metrics::Counter::createPtr("proxy.process.http.origin.close_private"); + http_rsb.origin_connect_adjust_thread = Metrics::Counter::createPtr("proxy.process.http.origin.connect.adjust_thread"); + http_rsb.origin_connections_throttled = Metrics::Counter::createPtr("proxy.process.http.origin_connections_throttled_out"); + http_rsb.origin_make_new = Metrics::Counter::createPtr("proxy.process.http.origin.make_new"); + http_rsb.origin_no_sharing = Metrics::Counter::createPtr("proxy.process.http.origin.no_sharing"); + http_rsb.origin_not_found = Metrics::Counter::createPtr("proxy.process.http.origin.not_found"); + http_rsb.origin_private = Metrics::Counter::createPtr("proxy.process.http.origin.private"); + http_rsb.origin_raw = Metrics::Counter::createPtr("proxy.process.http.origin.raw"); + http_rsb.origin_reuse = Metrics::Counter::createPtr("proxy.process.http.origin.reuse"); + http_rsb.origin_reuse_fail = Metrics::Counter::createPtr("proxy.process.http.origin.reuse_fail"); http_rsb.origin_server_request_document_total_size = Metrics::Counter::createPtr("proxy.process.http.origin_server_request_document_total_size"); http_rsb.origin_server_request_header_total_size = @@ -499,6 +504,7 @@ register_stat_callbacks() http_rsb.total_transactions_time = Metrics::Counter::createPtr("proxy.process.http.total_transactions_time"); http_rsb.total_x_redirect = Metrics::Counter::createPtr("proxy.process.http.total_x_redirect_count"); http_rsb.trace_requests = Metrics::Counter::createPtr("proxy.process.http.trace_requests"); + http_rsb.tunnel_chunked_throttle = Metrics::Counter::createPtr("proxy.process.http.tunnel.chunked_throttle"); http_rsb.tunnel_current_active_connections = Metrics::Gauge::createPtr("proxy.process.tunnel.current_active_connections"); http_rsb.tunnels = Metrics::Counter::createPtr("proxy.process.http.tunnels"); http_rsb.ua_begin_time = Metrics::Counter::createPtr("proxy.process.http.milestone.ua_begin"); @@ -1275,6 +1281,16 @@ HttpConfig::reconfigure() params->http_hdr_field_max_size = m_master.http_hdr_field_max_size; params->pp_hdr_max_size = m_master.pp_hdr_max_size; + // A header field name and value are each stored with a uint16_t length, so a + // single field cannot exceed UINT16_MAX octets. Clamp the configured + // per-field limit to that ceiling: a larger value can never be honored and an + // oversized field is rejected at parse time regardless. + if (params->http_hdr_field_max_size > UINT16_MAX) { + Note("proxy.config.http.header_field_max_size %" PRId64 " exceeds the %d-octet storage limit; clamping to %d", + static_cast(params->http_hdr_field_max_size), UINT16_MAX, UINT16_MAX); + params->http_hdr_field_max_size = UINT16_MAX; + } + if (params->oride.connection_tracker_config.server_max > 0 && params->oride.connection_tracker_config.server_max < params->oride.connection_tracker_config.server_min) { Warning("'%s' < per_server.min_keep_alive_connections, setting min=max , please correct your %s", diff --git a/src/proxy/http/HttpProxyServerMain.cc b/src/proxy/http/HttpProxyServerMain.cc index ed4431adf07..133f70116ee 100644 --- a/src/proxy/http/HttpProxyServerMain.cc +++ b/src/proxy/http/HttpProxyServerMain.cc @@ -158,6 +158,9 @@ make_net_accept_options(const HttpProxyPort *port, unsigned nthreads) net.local_ip = HttpConfig::m_master.inbound.ip4().network_order(); } else if (AF_UNIX == port->m_family) { net.local_path = port->m_unix_path; + net.unix_perm = port->m_unix_perm; + net.unix_uid = port->m_unix_uid; + net.unix_gid = port->m_unix_gid; net.sockopt_flags &= ~(NetVCOptions::SOCK_OPT_NO_DELAY | NetVCOptions::SOCK_OPT_TCP_FAST_OPEN | NetVCOptions::SOCK_OPT_TCP_NOTSENT_LOWAT); } diff --git a/src/proxy/http/HttpSM.cc b/src/proxy/http/HttpSM.cc index 9a623ffa9e4..eaa71d56539 100644 --- a/src/proxy/http/HttpSM.cc +++ b/src/proxy/http/HttpSM.cc @@ -285,13 +285,7 @@ HttpSM::~HttpSM() // coverity[exn_spec_violation] - release() only does ref counting and delete on POD types HttpConfig::release(t_state.http_config_param); - - // m_remap->release() can allocate (new_Deleter), so catch potential bad_alloc - try { - m_remap->release(); - } catch (...) { - Error("Exception in ~HttpSM during m_remap->release"); - } + m_remap.reset(); // coverity[exn_spec_violation] - cancel_pending_action() cancels pending cache work and clears tracked pointers cache_sm.cancel_pending_action(); @@ -334,8 +328,9 @@ HttpSM::init(bool from_early_data) t_state.state_machine = this; t_state.http_config_param = HttpConfig::acquire(); - // Acquire a lease on the global remap / rewrite table (stupid global name ...) - m_remap = rewrite_table.load()->acquire(); + // Snapshot the global remap / rewrite table. shared_ptr keeps it alive across the txn + // even if reload swaps the global pointer concurrently. + m_remap = rewrite_table.load(std::memory_order_acquire); // Simply point to the global config for the time being, no need to copy this // entire struct if nothing is going to change it. @@ -1752,6 +1747,9 @@ HttpSM::handle_api_return() case HttpTransact::StateMachineAction_t::INTERNAL_CACHE_DELETE: case HttpTransact::StateMachineAction_t::INTERNAL_CACHE_UPDATE_HEADERS: case HttpTransact::StateMachineAction_t::SEND_ERROR_CACHE_NOOP: { + // Consume any unforwarded request body, or close the connection when it cannot be + // fully consumed, so leftover bytes are not framed as the next request. + do_drain_request_body(t_state.hdr_info.client_response); setup_internal_transfer(&HttpSM::tunnel_handler); break; } @@ -2165,7 +2163,8 @@ HttpSM::state_read_server_response_header(int event, void *data) t_state.current.state = HttpTransact::CONNECTION_ALIVE; t_state.transact_return_point = HttpTransact::HandleResponse; - t_state.api_next_action = HttpTransact::StateMachineAction_t::API_READ_RESPONSE_HDR; + HttpTransact::strip_at_headers(t_state.hdr_info.server_response, HttpTransact::AtHeaderSource::ORIGIN_RESPONSE, sm_id); + t_state.api_next_action = HttpTransact::StateMachineAction_t::API_READ_RESPONSE_HDR; // if exceeded limit deallocate postdata buffers and disable redirection if (!(enable_redirection && (redirection_tries < t_state.txn_conf->number_of_redirections))) { @@ -2916,12 +2915,18 @@ HttpSM::tunnel_handler_post(int event, void *data) // post failed switch (t_state.client_info.state) { case HttpTransact::ACTIVE_TIMEOUT: + tunnel.deallocate_buffers(); + tunnel.reset(); call_transact_and_set_next_state(HttpTransact::PostActiveTimeoutResponse); return 0; case HttpTransact::INACTIVE_TIMEOUT: + tunnel.deallocate_buffers(); + tunnel.reset(); call_transact_and_set_next_state(HttpTransact::PostInactiveTimeoutResponse); return 0; case HttpTransact::PARSE_ERROR: + tunnel.deallocate_buffers(); + tunnel.reset(); call_transact_and_set_next_state(HttpTransact::BadRequest); return 0; default: @@ -2986,7 +2991,8 @@ HttpSM::tunnel_handler_post(int event, void *data) if (milestones[TS_MILESTONE_SERVER_READ_HEADER_DONE] != 0) { t_state.current.state = HttpTransact::CONNECTION_ALIVE; t_state.transact_return_point = HttpTransact::HandleResponse; - t_state.api_next_action = HttpTransact::StateMachineAction_t::API_READ_RESPONSE_HDR; + HttpTransact::strip_at_headers(t_state.hdr_info.server_response, HttpTransact::AtHeaderSource::ORIGIN_RESPONSE, sm_id); + t_state.api_next_action = HttpTransact::StateMachineAction_t::API_READ_RESPONSE_HDR; do_api_callout(); } break; @@ -3006,11 +3012,23 @@ HttpSM::setup_tunnel_handler_trailer(HttpTunnelProducer *p) SMDbg(dbg_ctl_http, "Wait for the trailing header"); + ProxyTransaction *ua_txn = _ua.get_txn(); + if (!ua_txn) { + tunnel.local_finish_all(p); + return; + } + + if (!ua_txn->can_send_h2_trailer()) { + SMDbg(dbg_ctl_http, "User agent transaction cannot send HTTP/2 trailers; dropping the origin trailer"); + tunnel.local_finish_all(p); + return; + } + // Mark this before the body tunnel completes so HTTP/2 does not send END_STREAM + // on the final DATA frame. + ua_txn->set_expect_send_trailer(); + // Swap out the default hander to set up the new tunnel for the trailer exchange. HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler_trailer); - if (_ua.get_txn()) { - _ua.get_txn()->set_expect_send_trailer(); - } tunnel.local_finish_all(p); } @@ -3033,27 +3051,45 @@ HttpSM::tunnel_handler_trailer(int event, void *data) // Set up a new tunnel to transport the trailing header to the UA HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler); + ProxyTransaction *ua_txn = _ua.get_txn(); + HttpVCTableEntry *ua_entry = _ua.get_entry(); + IOBufferReader *server_reader = server_txn ? server_txn->get_remote_reader() : nullptr; + if (!ua_txn || !ua_entry || !ua_entry->vc || !server_reader || !server_entry || !server_entry->vc) { + if (server_reader) { + server_reader->consume(server_reader->read_avail()); + } + SMDbg(dbg_ctl_http, "Cannot set up trailer tunnel; dropping the origin trailer"); + return tunnel_handler(event, data); + } + + if (!ua_txn->expect_send_trailer()) { + if (!ua_txn->can_send_h2_trailer()) { + server_reader->consume(server_reader->read_avail()); + SMDbg(dbg_ctl_http, "User agent transaction cannot send HTTP/2 trailers; dropping the origin trailer"); + return tunnel_handler(event, data); + } + ua_txn->set_expect_send_trailer(); + } + MIOBuffer *trailer_buffer = new_MIOBuffer(HTTP_HEADER_BUFFER_SIZE_INDEX); IOBufferReader *buf_start = trailer_buffer->alloc_reader(); size_t nbytes = INT64_MAX; - int start_bytes = trailer_buffer->write(server_txn->get_remote_reader(), server_txn->get_remote_reader()->read_avail()); - server_txn->get_remote_reader()->consume(start_bytes); + int start_bytes = trailer_buffer->write(server_reader, server_reader->read_avail()); + server_reader->consume(start_bytes); // The server has already sent all it has if (server_txn->is_read_closed()) { nbytes = start_bytes; } - // Signal the _ua.get_txn() to get ready for a trailer - _ua.get_txn()->set_expect_send_trailer(); tunnel.deallocate_buffers(); tunnel.reset(); HttpTunnelProducer *p = tunnel.add_producer(server_entry->vc, nbytes, buf_start, &HttpSM::tunnel_handler_trailer_server, HttpTunnelType_t::HTTP_SERVER, "http server trailer"); - tunnel.add_consumer(_ua.get_entry()->vc, server_entry->vc, &HttpSM::tunnel_handler_trailer_ua, HttpTunnelType_t::HTTP_CLIENT, + tunnel.add_consumer(ua_entry->vc, server_entry->vc, &HttpSM::tunnel_handler_trailer_ua, HttpTunnelType_t::HTTP_CLIENT, "user agent trailer"); - _ua.get_entry()->in_tunnel = true; - server_entry->in_tunnel = true; + ua_entry->in_tunnel = true; + server_entry->in_tunnel = true; tunnel.tunnel_run(p); @@ -3726,7 +3762,9 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) break; } - if (event == VC_EVENT_WRITE_COMPLETE && server_txn && server_txn->expect_receive_trailer()) { + ProxyTransaction *ua_txn = _ua.get_txn(); + if (event == VC_EVENT_WRITE_COMPLETE && server_txn && server_txn->expect_receive_trailer() && ua_txn && + ua_txn->expect_send_trailer()) { // Don't shutdown if we are still expecting a trailer } else if (close_connection) { // If the client could be pipelining or is doing a POST, we need to @@ -4517,7 +4555,7 @@ HttpSM::state_remap_request(int event, void * /* data ATS_UNUSED */) case EVENT_REMAP_COMPLETE: { pending_action = nullptr; SMDbg(dbg_ctl_url_rewrite, "completed processor-based remapping request"); - t_state.url_remap_success = remapProcessor.finish_remap(&t_state, m_remap); + t_state.url_remap_success = remapProcessor.finish_remap(&t_state, m_remap.get()); call_transact_and_set_next_state(nullptr); break; } @@ -4571,7 +4609,8 @@ HttpSM::check_sni_host() Log::error("%s", error_bw_buffer.c_str()); this->t_state.client_connection_allowed = false; } - } else if (strncasecmp(host_name.data(), sni_value, host_len) != 0) { // Name mismatch + } else if (strlen(sni_value) != static_cast(host_len) || + strncasecmp(host_name.data(), sni_value, host_len) != 0) { // Name mismatch Warning("SNI/hostname mismatch sni=%s host=%.*s action=%s", sni_value, host_len, host_name.data(), action_value); SMDbg(dbg_ctl_ssl_sni, "SNI/hostname mismatch sni=%s host=%.*s action=%s", sni_value, host_len, host_name.data(), action_value); @@ -4594,7 +4633,7 @@ HttpSM::do_remap_request(bool run_inline) { SMDbg(dbg_ctl_http_seq, "Remapping request"); SMDbg(dbg_ctl_url_rewrite, "Starting a possible remapping for request"); - bool ret = remapProcessor.setup_for_remap(&t_state, m_remap); + bool ret = remapProcessor.setup_for_remap(&t_state, m_remap.get()); check_sni_host(); @@ -4646,6 +4685,11 @@ HttpSM::do_hostdb_lookup() // If directed to not look up fqdns then mark as resolved if (t_state.txn_conf->no_dns_forward_to_parent && t_state.parent_result.result == ParentResultType::UNDEFINED) { + // resolved_p documents that dns_info.addr holds a valid resolved address. + // We are not actually resolving anything here, so set addr to INADDR_ANY + // so the ats_is_ip_any check in HttpTransact::OSDNSLookup produces a + // clean Bad Request response rather than reusing leftover State data. + ats_ip4_set(&t_state.dns_info.addr, INADDR_ANY, 0); t_state.dns_info.resolved_p = true; call_transact_and_set_next_state(nullptr); return; @@ -5253,7 +5297,7 @@ HttpSM::do_cache_lookup_and_read() } void -HttpSM::do_cache_delete_all_alts(Continuation *cont) +HttpSM::do_cache_delete_all_alts() { // Do not delete a non-existent object. ink_assert(t_state.cache_info.object_read); @@ -5263,9 +5307,7 @@ HttpSM::do_cache_delete_all_alts(Continuation *cont) HttpCacheKey key; Cache::generate_key(&key, t_state.cache_info.lookup_url, t_state.txn_conf->cache_ignore_query, t_state.txn_conf->cache_generation_number); - pending_action = cacheProcessor.remove(cont, &key); - - return; + cacheProcessor.remove(nullptr, &key); } inline void @@ -5427,7 +5469,7 @@ HttpSM::get_outbound_sni() const if (auto *netvc = _ua.get_txn()->get_netvc(); netvc) { snis = netvc->get_service(); if (snis && snis->hints_from_sni.outbound_sni_policy.has_value()) { - policy.assign(snis->hints_from_sni.outbound_sni_policy->data(), swoc::TextView::npos); + policy.assign(snis->hints_from_sni.outbound_sni_policy->data(), snis->hints_from_sni.outbound_sni_policy->size()); } } } @@ -5435,7 +5477,7 @@ HttpSM::get_outbound_sni() const if (policy.empty() || policy == "host"_tv) { // By default the host header field value is used for the SNI. zret = t_state.hdr_info.server_request.host_get(); - } else if (_ua.get_txn() && policy == "server_name"_tv) { + } else if (_ua.get_txn() && snis && policy == "server_name"_tv) { const char *const server_name = snis->get_sni_server_name(); if (nullptr == server_name || server_name[0] == '\0') { zret.assign(nullptr, swoc::TextView::npos); @@ -5451,6 +5493,21 @@ HttpSM::get_outbound_sni() const return zret; } +std::string_view +HttpSM::get_outbound_sni_for_cert_verification() const +{ + auto zret = this->get_outbound_sni(); + if (zret.empty()) { + zret = t_state.hdr_info.server_request.host_get(); + } + if (zret.empty()) { + if (t_state.current.server != nullptr && t_state.current.server->name != nullptr) { + zret = t_state.current.server->name; + } + } + return zret; +} + bool HttpSM::apply_ip_allow_filter() { @@ -5619,8 +5676,13 @@ HttpSM::do_http_server_open(bool raw, bool only_direct) } } - // Check for self loop. - if (!_ua.get_txn()->is_outbound_transparent() && HttpTransact::will_this_request_self_loop(&t_state)) { + // Check for self loop. In outbound-transparent mode the client's original + // destination is the legitimate upstream, so the client_info.dst_addr match + // inside the loop check is skipped. The self-IP check (when + // max_proxy_cycles == 0) and the Via-header multi-hop check are still + // enforced so a client targeting ATS's own listening address or looping + // through its own Via UUID is still caught. + if (HttpTransact::will_this_request_self_loop(&t_state, _ua.get_txn()->is_outbound_transparent())) { call_transact_and_set_next_state(HttpTransact::SelfLoop); return; } @@ -6473,6 +6535,15 @@ HttpSM::do_drain_request_body(HTTPHdr &response) int64_t content_length = t_state.hdr_info.client_request.get_content_length(); int64_t avail = _ua.get_txn()->get_remote_reader()->read_avail(); + // Consuming the client request buffer is only safe when no request-body tunnel is active: a live + // producer could still be reading from it, so consuming here would race and desync the connection. + // Callers today reach this after the request-body tunnel is done, so the guard is normally a no-op; + // keep it so a future path with a live tunnel drops keep-alive instead of double-consuming the reader. + if (tunnel.is_tunnel_active()) { + SMDbg(dbg_ctl_http, "request-body tunnel still active, setting the response to non-keepalive"); + goto close_connection; + } + if (t_state.client_info.transfer_encoding == HttpTransact::TransferEncoding_t::CHUNKED) { SMDbg(dbg_ctl_http, "Chunked body, setting the response to non-keepalive"); goto close_connection; @@ -8182,7 +8253,7 @@ HttpSM::set_next_state() case HttpTransact::StateMachineAction_t::REMAP_REQUEST: { do_remap_request(true); /* run inline */ SMDbg(dbg_ctl_url_rewrite, "completed inline remapping request"); - t_state.url_remap_success = remapProcessor.finish_remap(&t_state, m_remap); + t_state.url_remap_success = remapProcessor.finish_remap(&t_state, m_remap.get()); if (t_state.next_action == HttpTransact::StateMachineAction_t::SEND_ERROR_CACHE_NOOP && t_state.transact_return_point == nullptr) { // It appears that we can now set the next_action to error and transact_return_point to nullptr when @@ -8231,7 +8302,12 @@ HttpSM::set_next_state() break; } else if (t_state.dns_info.looking_up == ResolveInfo::ORIGIN_SERVER && t_state.txn_conf->no_dns_forward_to_parent && t_state.parent_result.result != ParentResultType::UNDEFINED) { - t_state.dns_info.resolved_p = true; // seems dangerous - where's the IP address? + // We claim resolved here so the SM does not stall on origin DNS, but no + // address has been obtained. Set addr to INADDR_ANY so the + // ats_is_ip_any guard in HttpTransact::OSDNSLookup produces a clean + // Bad Request rather than reusing leftover State data. + ats_ip4_set(&t_state.dns_info.addr, INADDR_ANY, 0); + t_state.dns_info.resolved_p = true; call_transact_and_set_next_state(nullptr); break; } else if (t_state.dns_info.resolved_p) { @@ -8417,8 +8493,6 @@ HttpSM::set_next_state() release_server_session(); } - do_drain_request_body(t_state.hdr_info.client_response); - // If we're in state SEND_API_RESPONSE_HDR, it means functions // registered to hook SEND_RESPONSE_HDR have already been called. So we do not // need to call do_api_callout. Otherwise TS loops infinitely in this state ! @@ -8435,7 +8509,7 @@ HttpSM::set_next_state() // Nuke all the alternates since this is mostly likely // the result of a delete method cache_sm.end_both(); - do_cache_delete_all_alts(nullptr); + do_cache_delete_all_alts(); release_server_session(); t_state.api_next_action = HttpTransact::StateMachineAction_t::API_SEND_RESPONSE_HDR; @@ -8626,8 +8700,9 @@ HttpSM::redirect_request(const char *arg_redirect_url, const int arg_redirect_le if (auto tmpOrigHost{t_state.hdr_info.server_request.value_get(static_cast(MIME_FIELD_HOST))}; !tmpOrigHost.empty()) { - memcpy(origHost, tmpOrigHost.data(), tmpOrigHost.length()); - origHost[std::min(tmpOrigHost.length(), sizeof(origHost) - 1)] = '\0'; + auto hostLen = std::min(tmpOrigHost.length(), sizeof(origHost) - 1); + memcpy(origHost, tmpOrigHost.data(), hostLen); + origHost[hostLen] = '\0'; } else { valid_origHost = false; } diff --git a/src/proxy/http/HttpSessionAccept.cc b/src/proxy/http/HttpSessionAccept.cc index 6233a05e90d..4f3752c3b63 100644 --- a/src/proxy/http/HttpSessionAccept.cc +++ b/src/proxy/http/HttpSessionAccept.cc @@ -45,8 +45,10 @@ HttpSessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferReade break; } else if (IpAllow::Subject::PROXY == IpAllow::subjects[i] && netvc->get_proxy_protocol_version() != ProxyProtocolVersion::UNDEFINED) { - client_ip = netvc->get_proxy_protocol_src_addr(); - break; + if (sockaddr const *proxy_ip = netvc->get_proxy_protocol_src_addr(); proxy_ip != nullptr) { + client_ip = proxy_ip; + break; + } } } diff --git a/src/proxy/http/HttpSessionManager.cc b/src/proxy/http/HttpSessionManager.cc index cf54c5c81bd..8827f60c2eb 100644 --- a/src/proxy/http/HttpSessionManager.cc +++ b/src/proxy/http/HttpSessionManager.cc @@ -31,10 +31,12 @@ ****************************************************************************/ #include "../../iocore/net/P_UnixNetVConnection.h" +#include "../../iocore/net/P_SSLClientUtils.h" #include "proxy/http/HttpSessionManager.h" #include "proxy/ProxySession.h" #include "proxy/http/HttpSM.h" #include "proxy/http/HttpDebugNames.h" +#include "iocore/eventsystem/IOBuffer.h" #include "iocore/net/TLSSNISupport.h" #include "ts/ats_probe.h" #include @@ -43,6 +45,13 @@ namespace { DbgCtl dbg_ctl_http_ss{"http_ss"}; +bool +validate_session_origin_cert(HttpSM *sm, PoolableSession *session) +{ + return !session->is_multiplexing() || + validate_server_certificate_hostname(session->get_netvc(), sm->get_outbound_sni_for_cert_verification()); +} + } // end anonymous namespace // Initialize a thread to handle HTTP session management @@ -100,7 +109,7 @@ ServerSessionPool::validate_host_sni(HttpSM *sm, NetVConnection *netvc) // TS-4468: If the connection matches, make sure the SNI server // name (if present) matches the request hostname auto req_host{sm->t_state.hdr_info.server_request.host_get()}; - retval = strncasecmp(session_sni, req_host.data(), req_host.length()) == 0; + retval = strlen(session_sni) == req_host.length() && strncasecmp(session_sni, req_host.data(), req_host.length()) == 0; Dbg(dbg_ctl_http_ss, "validate_host_sni host=%*.s, sni=%s", static_cast(req_host.length()), req_host.data(), session_sni); } @@ -177,7 +186,8 @@ ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostna if (port == ats_ip_port_cast(iter->get_remote_addr()) && (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_SNI) || validate_sni(sm, iter->get_netvc())) && (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTSNISYNC) || validate_host_sni(sm, iter->get_netvc())) && - (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_CERT) || validate_cert(sm, iter->get_netvc()))) { + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_CERT) || validate_cert(sm, iter->get_netvc())) && + validate_session_origin_cert(sm, &*iter)) { zret = HSMresult_t::DONE; break; } @@ -204,14 +214,21 @@ ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostna if ((!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTONLY) || iter->hostname_hash == hostname_hash) && (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_SNI) || validate_sni(sm, iter->get_netvc())) && (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTSNISYNC) || validate_host_sni(sm, iter->get_netvc())) && - (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_CERT) || validate_cert(sm, iter->get_netvc()))) { + (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_CERT) || validate_cert(sm, iter->get_netvc())) && + validate_session_origin_cert(sm, &*iter)) { + zret = HSMresult_t::DONE; + break; + } + ++iter; + } + } else { + while (iter != end) { + if (validate_session_origin_cert(sm, &*iter)) { zret = HSMresult_t::DONE; break; } ++iter; } - } else if (iter != end) { - zret = HSMresult_t::DONE; } if (zret == HSMresult_t::DONE) { to_return = &*iter; @@ -223,9 +240,16 @@ ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostna return zret; } -void +bool ServerSessionPool::releaseSession(PoolableSession *ss) { + IOBufferReader *remote_reader = ss->get_remote_reader(); + if (remote_reader->read_avail() > 0) { + // The caller is responsible for closing when this returns false. + Dbg(dbg_ctl_http_ss, "[%" PRId64 "] [release session] origin sent unexpected bytes; not pooling", ss->connection_id()); + return false; + } + ss->state = PoolableSession::PooledState::KA_POOLED; // Now we need to issue a read on the connection to detect // if it closes on us. We will get called back in the @@ -233,7 +257,7 @@ ServerSessionPool::releaseSession(PoolableSession *ss) // to remove the connection from our lists // Actually need to have a buffer here, otherwise the vc is // disabled - ss->do_io_read(this, INT64_MAX, ss->get_remote_reader()->mbuf); + ss->do_io_read(this, INT64_MAX, remote_reader->mbuf); // Transfer control of the write side as well ss->do_io_write(this, 0, nullptr); @@ -248,6 +272,7 @@ ServerSessionPool::releaseSession(PoolableSession *ss) "[%" PRId64 "] [release session] " "session placed into shared pool", ss->connection_id()); + return true; } // Called from the NetProcessor to let us know that a @@ -378,7 +403,8 @@ HttpSessionManager::acquire_session(HttpSM *sm, sockaddr const *ip, const char * (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_HOSTSNISYNC) || ServerSessionPool::validate_host_sni(sm, to_return->get_netvc())) && (!(match_style & TS_SERVER_SESSION_SHARING_MATCH_MASK_CERT) || - ServerSessionPool::validate_cert(sm, to_return->get_netvc()))) { + ServerSessionPool::validate_cert(sm, to_return->get_netvc())) && + validate_session_origin_cert(sm, to_return)) { Dbg(dbg_ctl_http_ss, "[%" PRId64 "] [acquire session] returning attached session ", to_return->connection_id()); to_return->state = PoolableSession::PooledState::SSN_IN_USE; sm->create_server_txn(to_return); @@ -536,8 +562,12 @@ HttpSessionManager::release_session(PoolableSession *to_release) bool const locked = lockSessionPool(pool->mutex, ethread, this->get_pool_type(), &mlock, &tlock); if (locked) { - pool->releaseSession(to_release); - ATS_PROBE2(http_ss_release_session_global, to_release->connection_id(), to_release->get_netvc()->get_socket()); + bool const pooled = pool->releaseSession(to_release); + ATS_PROBE3(http_ss_release_session_global, to_release->connection_id(), to_release->get_netvc()->get_socket(), pooled); + if (!pooled) { + // close & free session + to_release->do_io_close(); + } } else if (this->get_pool_type() == TS_SERVER_SESSION_SHARING_POOL_HYBRID) { // Try again with the thread pool to_release->sharing_pool = TS_SERVER_SESSION_SHARING_POOL_THREAD; diff --git a/src/proxy/http/HttpTransact.cc b/src/proxy/http/HttpTransact.cc index 8180f845a22..edf6b2c862e 100644 --- a/src/proxy/http/HttpTransact.cc +++ b/src/proxy/http/HttpTransact.cc @@ -28,7 +28,10 @@ #include "tscore/ink_platform.h" #include +#include #include +#include +#include using namespace std::literals; @@ -79,8 +82,65 @@ DbgCtl dbg_ctl_http_trans_websocket_upgrade_post_remap{"http_trans_websocket_upg DbgCtl dbg_ctl_parent_down{"parent_down"}; DbgCtl dbg_ctl_url_rewrite{"url_rewrite"}; DbgCtl dbg_ctl_ip_allow{"ip_allow"}; + +std::string_view +get_at_header_source_name(HttpTransact::AtHeaderSource source) +{ + using AtHeaderSource = HttpTransact::AtHeaderSource; + + switch (source) { + case AtHeaderSource::CLIENT_REQUEST: + return "client request"; + case AtHeaderSource::ORIGIN_RESPONSE: + return "origin response"; + } + + return "unknown source"; +} + +Metrics::Counter::AtomicType * +get_at_header_source_metric(HttpTransact::AtHeaderSource source) +{ + using AtHeaderSource = HttpTransact::AtHeaderSource; + + switch (source) { + case AtHeaderSource::CLIENT_REQUEST: + return http_rsb.client_request_at_headers_stripped; + case AtHeaderSource::ORIGIN_RESPONSE: + return http_rsb.origin_response_at_headers_stripped; + } + + return nullptr; +} } // namespace +/** + * Remove internal @ headers from a parsed header before plugin hooks run. + * + * @param[in,out] header The header to sanitize in place. + * @param[in] source The source of the header being sanitized. + * @param[in] sm_id The state machine identifier for diagnostic logging. + */ +void +HttpTransact::strip_at_headers(HTTPHdr &header, AtHeaderSource source, std::int64_t sm_id) +{ + auto const metric = get_at_header_source_metric(source); + auto const source_name = get_at_header_source_name(source); + + for (auto field = header.begin(); field != header.end();) { + auto current = field++; + auto name = current->name_get(); + + if (!name.empty() && name[0] == '@') { + Metrics::Counter::increment(metric); + Error("[%" PRId64 "] stripped internal @ header from %.*s: %.*s", sm_id, static_cast(source_name.size()), + source_name.data(), static_cast(name.size()), name.data()); + header.field_delete(&*current, false); + } + } +} + +////////////////////////////////////////////////////////////////////////// // Support ip_resolve override. const MgmtConverter HttpTransact::HOST_RES_CONV{[](const void *data) -> std::string_view { const HostResData *host_res_data = static_cast(data); @@ -933,6 +993,12 @@ HttpTransact::BadRequest(State *s) case HTTPStatus::HTTPVER_NOT_SUPPORTED: status = s->http_return_code; reason = "Unsupported HTTP Version"; + break; + case HTTPStatus::BAD_GATEWAY: + status = s->http_return_code; + reason = "Bad Gateway"; + body_factory_template = "default"; + break; default: break; } @@ -1521,6 +1587,8 @@ HttpTransact::ModifyRequest(State *s) } } + strip_at_headers(request, AtHeaderSource::CLIENT_REQUEST, s->state_machine_id()); + TxnDbg(dbg_ctl_http_trans, "END HttpTransact::ModifyRequest"); TRANSACT_RETURN(StateMachineAction_t::API_READ_REQUEST_HDR, HttpTransact::StartRemapRequest); @@ -1534,7 +1602,7 @@ HttpTransact::handleIfRedirect(State *s) mapping_type answer; URL redirect_url; - answer = request_url_remap_redirect(&s->hdr_info.client_request, &redirect_url, s->state_machine->m_remap); + answer = request_url_remap_redirect(&s->hdr_info.client_request, &redirect_url, s->state_machine->m_remap.get()); if ((answer == mapping_type::PERMANENT_REDIRECT) || (answer == mapping_type::TEMPORARY_REDIRECT)) { s->remap_redirect = redirect_url.string_get_ref(nullptr); if (answer == mapping_type::TEMPORARY_REDIRECT) { @@ -2680,6 +2748,18 @@ HttpTransact::CallOSDNSLookup(State *s) HostStatus &pstatus = HostStatus::instance(); HostStatRec *hst = pstatus.getHostStatus(s->server_info.name); if (hst && hst->status == TSHostStatus::TS_HOST_STATUS_DOWN) { + // Guard against unbounded recursion when the host is DOWN, the cache has + // a HIT, and a Range request cannot be satisfied from cache. Without this + // flag, CallOSDNSLookup -> handle_server_connection_not_open -> + // build_response_from_cache (Range branch) -> CallOSDNSLookup recurses on + // the stack until overflow. + if (s->host_down_cache_fallback_attempted) { + TxnDbg(dbg_ctl_http, "host down cache fallback already attempted; returning 502"); + build_error_response(s, HTTPStatus::BAD_GATEWAY, "Next Hop Connection Failed", "connect#failed_connect"); + s->next_action = StateMachineAction_t::SEND_ERROR_CACHE_NOOP; + return; + } + s->host_down_cache_fallback_attempted = true; TxnDbg(dbg_ctl_http, "%d ", static_cast(s->cache_lookup_result)); s->current.state = OUTBOUND_CONGESTION; if (s->cache_lookup_result == CacheLookupResult_t::HIT_STALE || s->cache_lookup_result == CacheLookupResult_t::HIT_WARNING || @@ -3792,7 +3872,9 @@ HttpTransact::handle_response_from_parent(State *s) if (s->current.retry_type == ParentRetry_t::SIMPLE) { s->current.simple_retry_attempts++; } else { - markParentDown(s); + if (is_request_retryable(s)) { + markParentDown(s); + } s->current.unavailable_server_retry_attempts++; } next_lookup = find_server_and_update_current_info(s); @@ -4053,6 +4135,14 @@ HttpTransact::handle_response_from_server(State *s) void HttpTransact::error_log_connection_failure(State *s, ServerState_t conn_state) { + // No origin server was selected (e.g. a host marked DOWN whose Range request + // falls back to a cache hit), so there is no connection to log. Bail out + // before dereferencing a null current.server. + if (s->current.server == nullptr) { + TxnDbg(dbg_ctl_http_trans, "no current server; skipping connection failure log"); + return; + } + ip_port_text_buffer addrbuf; TxnDbg(dbg_ctl_http_trans, "[%d] failed to connect [%d] to %s", s->current.retry_attempts.get(), conn_state, ats_ip_nptop(&s->current.server->dst_addr.sa, addrbuf, sizeof(addrbuf))); @@ -5229,16 +5319,33 @@ HttpTransact::set_headers_for_cache_write(State *s, HTTPInfo *cache_info, HTTPHd void HttpTransact::merge_response_header_with_cached_header(HTTPHdr *cached_header, HTTPHdr *response_header) { + // Connection tokens are hop-by-hop header names to strip. Real-world Connection headers + // rarely contain more than a few tokens; use a fixed-capacity buffer to avoid allocation. + // If a pathological response exceeds the buffer, fall back to scanning the Connection field + // directly to ensure no token is silently skipped. + static constexpr size_t MAX_CONN_TOKENS = 16; + std::array conn_tokens; + size_t conn_token_count = 0; + bool conn_overflow = false; + + MIMEField *conn_field = response_header->field_find(static_cast(MIME_FIELD_CONNECTION)); + if (conn_field != nullptr) { + HdrCsvIter csv; + + for (auto token = csv.get_first(conn_field, true); token; token = csv.get_next()) { + if (conn_token_count < MAX_CONN_TOKENS) { + conn_tokens[conn_token_count++] = token; + } else { + conn_overflow = true; + break; + } + } + } + for (auto spot = response_header->begin(), limit = response_header->end(); spot != limit; ++spot) { MIMEField &field{*spot}; auto name{field.name_get()}; - /////////////////////////// - // is hop-by-hop header? // - /////////////////////////// - if (HttpTransactHeaders::is_this_a_hop_by_hop_header(name.data())) { - continue; - } ///////////////////////////////////// // dont cache content-length field and transfer encoding // ///////////////////////////////////// @@ -5267,6 +5374,32 @@ HttpTransact::merge_response_header_with_cached_header(HTTPHdr *cached_header, H if (name.data() == MIME_FIELD_WARNING.c_str()) { continue; } + /////////////////////////// + // is hop-by-hop header? // + /////////////////////////// + if (HttpTransactHeaders::is_this_a_hop_by_hop_header(name.data())) { + continue; + } + // Check if named in Connection header (linear scan — after cheap pointer checks above). + // On overflow, fall back to re-scanning the Connection CSV to guarantee correctness. + if (conn_field != nullptr) { + bool is_conn_token = false; + if (!conn_overflow) { + is_conn_token = std::any_of(conn_tokens.begin(), conn_tokens.begin() + conn_token_count, + [&name](swoc::TextView conn_token) { return strcasecmp(name, conn_token) == 0; }); + } else { + HdrCsvIter csv; + for (auto token = csv.get_first(conn_field, true); token; token = csv.get_next()) { + if (strcasecmp(name, token) == 0) { + is_conn_token = true; + break; + } + } + } + if (is_conn_token) { + continue; + } + } // Reconcile the cached header's fields of this name to exactly match the // response's values for this name. Each response field name is processed @@ -5586,6 +5719,30 @@ HttpTransact::check_request_validity(State *s, HTTPHdr *incoming_hdr) return RequestError_t::NON_EXISTANT_REQUEST_HEADER; } + // RFC 9112: If chunked is present in Transfer-Encoding, it must be the + // final encoding. Reject the request if chunked appears before other values. + if (incoming_hdr->presence(MIME_PRESENCE_TRANSFER_ENCODING)) { + MIMEField *field = incoming_hdr->field_find(static_cast(MIME_FIELD_TRANSFER_ENCODING)); + bool found_chunked = false; + + while (field) { + HdrCsvIter enc_val_iter; + int enc_val_len; + const char *enc_value = enc_val_iter.get_first(field, &enc_val_len); + + while (enc_value) { + const char *wks_value = hdrtoken_string_to_wks(enc_value, enc_val_len); + if (wks_value == HTTP_VALUE_CHUNKED.c_str()) { + found_chunked = true; + } else if (found_chunked) { + return RequestError_t::BAD_HTTP_HEADER_SYNTAX; + } + enc_value = enc_val_iter.get_next(&enc_val_len); + } + field = field->m_next_dup; + } + } + if (!(HttpTransactHeaders::is_request_proxy_authorized(incoming_hdr))) { return RequestError_t::FAILED_PROXY_AUTHORIZATION; } @@ -5713,7 +5870,10 @@ HttpTransact::set_client_request_state(State *s, HTTPHdr *incoming_hdr) while (enc_value) { const char *wks_value = hdrtoken_string_to_wks(enc_value, enc_val_len); if (wks_value == HTTP_VALUE_CHUNKED.c_str()) { - s->client_info.transfer_encoding = TransferEncoding_t::CHUNKED; + // Only treat as chunked if it is the last Transfer-Encoding value (RFC 9112) + if (enc_val_iter.get_next(&enc_val_len) == nullptr) { + s->client_info.transfer_encoding = TransferEncoding_t::CHUNKED; + } break; } enc_value = enc_val_iter.get_next(&enc_val_len); @@ -6818,7 +6978,7 @@ HttpTransact::process_quick_http_filter(State *s, int method) } bool -HttpTransact::will_this_request_self_loop(State *s) +HttpTransact::will_this_request_self_loop(State *s, bool is_outbound_transparent) { // The self-loop detection for this ATS node will allow up to max_proxy_cycles // (each time it sees it returns to itself it is one cycle) before declaring a self-looping condition detected. @@ -6837,10 +6997,14 @@ HttpTransact::will_this_request_self_loop(State *s) in_port_t dst_port = s->hdr_info.client_request.url_get()->port_get(); // going to this port. in_port_t local_port = s->client_info.dst_addr.host_order_port(); // already connected proxy port. // It's a loop if connecting to the same port as it already connected to the proxy and - // it's a proxy address or the same address it already connected to. + // it's any of the proxy's local addresses. In outbound-transparent mode we also allow + // the outbound destination to match the client's original connection destination + // (client_info.dst_addr), because that equality is expected for every transparent + // request (the client's original destination IS the legitimate upstream) and would + // false-positive this check. TxnDbg(dbg_ctl_http_transact, "dst_port = %d local_port = %d", dst_port, local_port); - if (dst_port == local_port && ((s->dns_info.active->data.ip == &Machine::instance()->ip.sa) || - (s->dns_info.active->data.ip == s->client_info.dst_addr))) { + if (dst_port == local_port && (Machine::instance()->is_self(s->dns_info.active->data.ip) || + (!is_outbound_transparent && s->dns_info.active->data.ip == s->client_info.dst_addr))) { switch (s->dns_info.looking_up) { case ResolveInfo::ORIGIN_SERVER: TxnDbg(dbg_ctl_http_transact, "host ip and port same as local ip and port - bailing"); @@ -8239,7 +8403,7 @@ HttpTransact::build_response(State *s, HTTPHdr *base_response, HTTPHdr *outgoing // process reverse mappings on the location header // TS-1364: do this regardless of response code - response_url_remap(outgoing_response, s->state_machine->m_remap); + response_url_remap(outgoing_response, s->state_machine->m_remap.get()); if (s->http_config_param->enable_http_stats) { HttpTransactHeaders::generate_and_set_squid_codes(outgoing_response, s->via_string, &s->squid_codes); diff --git a/src/proxy/http/HttpTransactHeaders.cc b/src/proxy/http/HttpTransactHeaders.cc index fab348dd521..1235b2fb202 100644 --- a/src/proxy/http/HttpTransactHeaders.cc +++ b/src/proxy/http/HttpTransactHeaders.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "tscore/ink_platform.h" @@ -226,6 +227,33 @@ HttpTransactHeaders::copy_header_fields(HTTPHdr *src_hdr, HTTPHdr *new_hdr, bool // Start with an exact duplicate new_hdr->copy(src_hdr); + if (MIMEField *conn_field = new_hdr->field_find(static_cast(MIME_FIELD_CONNECTION)); conn_field != nullptr) { + HdrCsvIter csv; + + for (auto token = csv.get_first(conn_field, true); !token.empty(); token = csv.get_next()) { + if (token[0] == '@') { + continue; + } + + // Look up the named header; skip if not present + MIMEField *target = new_hdr->field_find(std::string_view{token}); + if (target == nullptr) { + continue; + } + + // Use wksidx for cheap well-known header exemption checks + int const wks_idx = target->m_wks_idx; + if (wks_idx == MIME_WKSIDX_TE || wks_idx == MIME_WKSIDX_CONNECTION) { + continue; + } + if (retain_proxy_auth_hdrs && (wks_idx == MIME_WKSIDX_PROXY_AUTHENTICATE || wks_idx == MIME_WKSIDX_PROXY_AUTHORIZATION)) { + continue; + } + + new_hdr->field_delete(target); + } + } + // Nuke hop-by-hop headers // // The hop-by-hop header fields are laid out by the spec diff --git a/src/proxy/http/HttpTunnel.cc b/src/proxy/http/HttpTunnel.cc index d177b10603c..2f0b0afcc0f 100644 --- a/src/proxy/http/HttpTunnel.cc +++ b/src/proxy/http/HttpTunnel.cc @@ -88,18 +88,21 @@ ChunkedHandler::init_by_action(IOBufferReader *buffer_in, Action action, bool dr this->strict_chunk_parsing = parse_chunk_strictly; switch (action) { - case Action::DOCHUNK: - dechunked_reader = buffer_in->mbuf->clone_reader(buffer_in); - dechunked_reader->mbuf->water_mark = min_block_transfer_bytes; - chunked_buffer = new_MIOBuffer(CHUNK_IOBUFFER_SIZE_INDEX); - chunked_size = 0; + case Action::DOCHUNK: { + dechunked_reader = buffer_in->mbuf->clone_reader(buffer_in); + chunked_buffer = new_MIOBuffer(CHUNK_IOBUFFER_SIZE_INDEX); + chunked_buffer->water_mark = buffer_in->mbuf->water_mark; + chunked_size = 0; break; - case Action::DECHUNK: - chunked_reader = buffer_in->mbuf->clone_reader(buffer_in); - dechunked_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_256); - dechunked_size = 0; + } + case Action::DECHUNK: { + chunked_reader = buffer_in->mbuf->clone_reader(buffer_in); + dechunked_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_256); + dechunked_buffer->water_mark = buffer_in->mbuf->water_mark; + dechunked_size = 0; break; - case Action::PASSTHRU: + } + case Action::PASSTHRU: { chunked_reader = buffer_in->mbuf->clone_reader(buffer_in); if (drop_chunked_trailers) { // Note that dropping chunked trailers only applies in the passthrough @@ -110,10 +113,12 @@ ChunkedHandler::init_by_action(IOBufferReader *buffer_in, Action action, bool dr // filtering out the trailers. Otherwise, a simple passthrough needs no // intermediary buffer as consumers will simply read directly from // chunked_reader. - chunked_buffer = new_MIOBuffer(CHUNK_IOBUFFER_SIZE_INDEX); - chunked_size = 0; + chunked_buffer = new_MIOBuffer(CHUNK_IOBUFFER_SIZE_INDEX); + chunked_buffer->water_mark = buffer_in->mbuf->water_mark; + chunked_size = 0; } break; + } default: ink_release_assert(!"Unknown action"); } @@ -191,6 +196,12 @@ ChunkedHandler::read_size() state = ChunkedState::READ_ERROR; done = true; break; + } else if (*tmp == ';') { + // Start of a chunk extension. Parse it explicitly so that the value + // of a quoted-string extension is consumed up to its closing DQUOTE. + in_quoted_string = false; + in_escape = false; + state = ChunkedState::READ_EXTENSION; } else { if (ParseRules::is_cr(*tmp)) { ++num_cr; @@ -198,6 +209,53 @@ ChunkedHandler::read_size() state = ChunkedState::READ_SIZE_CRLF; // now look for CRLF } } + } else if (state == ChunkedState::READ_EXTENSION) { + // Parse a chunk extension per RFC 9112 Section 7.1.1: + // chunk-ext = *( BWS ";" BWS chunk-ext-name [ BWS "=" BWS chunk-ext-val ] ) + // chunk-ext-val = token / quoted-string + // A quoted-string (RFC 9110 Section 5.6.4) cannot contain a bare CR or LF + // (neither qdtext nor quoted-pair permits them), so either octet inside a + // quoted-string is a protocol error, not a line terminator. Rejecting it + // keeps a request whose extension embeds CR/LF from being forwarded and + // framed differently by a downstream parser. + if (in_quoted_string) { + if (ParseRules::is_cr(*tmp) || ParseRules::is_lf(*tmp)) { + state = ChunkedState::READ_ERROR; + done = true; + break; + } else if (in_escape) { + in_escape = false; // Consume the escaped octet of a quoted-pair. + } else if (*tmp == '\\') { + in_escape = true; // Begin a quoted-pair. + } else if (*tmp == '"') { + in_quoted_string = false; // Closing DQUOTE. + } + // Any other octet is part of the quoted-string value. + } else { + if (*tmp == '"') { + in_quoted_string = true; // Opening DQUOTE. + } else if (ParseRules::is_cr(*tmp)) { + // End of the chunk size line; hand off to the CRLF scanner. + ++num_cr; + state = ChunkedState::READ_SIZE_CRLF; + } else if (ParseRules::is_lf(*tmp)) { + // A bare LF (no preceding CR) ends the chunk size line. This is a + // protocol violation: reject it under strict parsing and tolerate it + // otherwise, matching how READ_SIZE_CRLF handles a bare LF. + Dbg(dbg_ctl_http_chunk, "Found an LF without a preceding CR (protocol violation) in chunk extension"); + if (strict_chunk_parsing) { + state = ChunkedState::READ_ERROR; + done = true; + break; + } + cur_chunk_bytes_left = (cur_chunk_size = running_sum); + state = (running_sum == 0) ? ChunkedState::READ_TRAILER_BLANK : ChunkedState::READ_CHUNK; + done = true; + num_cr = 0; + break; + } + // Any other octet is part of the extension name or unquoted value. + } } else if (state == ChunkedState::READ_SIZE_CRLF) { // Scan for a linefeed if (ParseRules::is_lf(*tmp)) { if (!prev_is_cr) { @@ -221,6 +279,15 @@ ChunkedHandler::read_size() break; } ++num_cr; + } else if (*tmp == ';' && num_cr == 0) { + // Optional whitespace (BWS) is allowed between the chunk size and the + // ';' that begins a chunk extension (RFC 9112 Section 7.1.1). Reaching + // here with num_cr == 0 means a ';' followed that whitespace, so parse + // the extension rather than scanning for CRLF; otherwise a quoted-string + // value preceded by BWS would not be handled. + in_quoted_string = false; + in_escape = false; + state = ChunkedState::READ_EXTENSION; } } else if (state == ChunkedState::READ_SIZE_START) { Dbg(dbg_ctl_http_chunk, "ChunkedState::READ_SIZE_START 0x%02x", *tmp); @@ -359,10 +426,16 @@ ChunkedHandler::read_trailer() // must a LF state = (state == ChunkedState::READ_TRAILER_BLANK) ? ChunkedState::READ_TRAILER_CR : ChunkedState::READ_TRAILER_LINE; } else if (ParseRules::is_lf(*tmp)) { - // For a LF to signal we are done reading the - // trailer, the line must have either been blank - // or must have only had a CR on it - if (state == ChunkedState::READ_TRAILER_CR || state == ChunkedState::READ_TRAILER_BLANK) { + // For a LF to signal we are done reading the trailer, the line must have + // been blank or have had only a CR on it. In RFC 9112 Section 7.1 the + // empty line that ends the chunked body (after the trailer section) is a + // full CRLF. A bare LF blank line is accepted only in non-strict mode; + // under strict parsing it is a protocol error, mirroring the chunk size + // line, so any bytes following the bare LF cannot be framed by a + // downstream parser as a separate request. + const bool valid_terminator = + state == ChunkedState::READ_TRAILER_CR || (state == ChunkedState::READ_TRAILER_BLANK && !strict_chunk_parsing); + if (valid_terminator) { state = ChunkedState::READ_DONE; Dbg(dbg_ctl_http_chunk, "completed read of trailers"); @@ -374,6 +447,12 @@ ChunkedHandler::read_trailer() } done = true; break; + } else if (state == ChunkedState::READ_TRAILER_BLANK) { + // Strict parsing: a bare LF blank line is not a valid trailer terminator. + Dbg(dbg_ctl_http_chunk, "rejecting bare LF trailer terminator under strict parsing"); + state = ChunkedState::READ_ERROR; + done = true; + break; } else { // A LF that does not terminate the trailer // indicates a new line @@ -399,6 +478,7 @@ ChunkedHandler::process_chunked_content() while (chunked_reader->is_read_avail_more_than(0) && state != ChunkedState::READ_DONE && state != ChunkedState::READ_ERROR) { switch (state) { case ChunkedState::READ_SIZE: + case ChunkedState::READ_EXTENSION: case ChunkedState::READ_SIZE_CRLF: case ChunkedState::READ_SIZE_START: bytes_read += read_size(); @@ -487,6 +567,25 @@ ChunkedHandler::generate_chunked_content() return std::make_pair(consumed_bytes, false); } +bool +ChunkedHandler::is_read_avail() +{ + switch (action) { + case Action::DOCHUNK: + return dechunked_reader->is_read_avail_more_than(0); + case Action::DECHUNK: + return chunked_reader->is_read_avail_more_than(0); + case Action::PASSTHRU: + // Plain passthrough has no intermediate output buffer (consumers read + // directly from chunked_reader), so a synthetic READ_READY would only + // re-walk the parser pointlessly. The kick is only meaningful when we + // are filtering chunked trailers into chunked_buffer. + return drop_chunked_trailers && chunked_reader->is_read_avail_more_than(0); + default: + return false; + } +} + HttpTunnelProducer::HttpTunnelProducer() : consumer_list() {} uint64_t @@ -894,14 +993,18 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) if (p->vc != HTTP_TUNNEL_STATIC_PRODUCER) { if (action == TunnelChunkingAction_t::CHUNK_CONTENT) { p->do_chunking = true; + Dbg(dbg_ctl_http_tunnel, "chunk mode"); } else if (action == TunnelChunkingAction_t::DECHUNK_CONTENT) { p->do_dechunking = true; + Dbg(dbg_ctl_http_tunnel, "dechunk mode"); } else if (action == TunnelChunkingAction_t::PASSTHRU_CHUNKED_CONTENT) { p->do_chunked_passthru = true; + Dbg(dbg_ctl_http_tunnel, "passthru mode"); // Dechunk the chunked content into the cache. if (cache_write_consumer != nullptr) { p->do_dechunking = true; + Dbg(dbg_ctl_http_tunnel, "dechunk mode is also enabled for cache write"); } } } @@ -1164,6 +1267,7 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) // If the producer is not alive (precomplete) make sure to kick the consumers for (c = p->consumer_list.head; c; c = c->link.next) { if (c->alive && c->write_vio) { + Dbg(dbg_ctl_http_tunnel, "re-enable %s write_vio", c->name); c->write_vio->reenable(); } } @@ -1274,6 +1378,31 @@ HttpTunnel::producer_handler_chunked(int event, HttpTunnelProducer *p) return event; } +/** + Disable the producer read when a consumer has buffered past its high water mark. + Returns the tripped consumer, or nullptr if none. + */ +HttpTunnelConsumer * +HttpTunnel::_throttle_chunked_producer(HttpTunnelProducer *p) +{ + for (HttpTunnelConsumer *ci = p->consumer_list.head; ci; ci = ci->link.next) { + if (!ci->alive || !ci->buffer_reader || !ci->buffer_reader->mbuf || !ci->buffer_reader->mbuf->water_mark) { + continue; + } + + if (ci->buffer_reader->high_water()) { + if (p->read_vio) { + Dbg(dbg_ctl_http_tunnel, "disable %s read_vio - %s read_avail = %" PRId64 " / %" PRId64, p->name, ci->name, + ci->buffer_reader->read_avail(), ci->buffer_reader->mbuf->water_mark); + Metrics::Counter::increment(http_rsb.tunnel_chunked_throttle); + p->read_vio->disable(); + } + return ci; + } + } + return nullptr; +} + // // bool HttpTunnel::producer_handler(int event, HttpTunnelProducer* p) // @@ -1297,12 +1426,38 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) p->bytes_consumed, p->ntodo); // Handle chunking/dechunking/chunked-passthrough if necessary. - if (p->do_chunking) { - // This will update body_bytes_to_copy with the number of bytes copied. - event = producer_handler_dechunked(event, p); - } else if (p->do_dechunking || p->do_chunked_passthru) { + if (p->is_handling_chunked_content()) { + // Chunked passthru (drop=0) lets the consumer read read_buffer directly, the + // same buffer chunked_reader walks; the parser must run before the throttle or + // chunked_reader pins read_buffer at high water and deadlocks. Buffered paths + // fill an output buffer, so they keep throttling before the walk. + bool const plain_passthru = p->do_chunked_passthru && !p->chunked_handler.drop_chunked_trailers; + + // Only throttle on READ_READY. Terminal events (READ_COMPLETE / EOS / PRECOMPLETE) must fall through + // to the completion path below so the producer is marked !alive and the SM is notified. + if (event == VC_EVENT_READ_READY && !plain_passthru) { + if (HttpTunnelConsumer *ci = _throttle_chunked_producer(p); ci != nullptr) { + if (ci->write_vio) { + Dbg(dbg_ctl_http_tunnel, "re-enable %s write_vio", ci->name); + ci->write_vio->reenable(); + } + return false; + } + } + // This will update body_bytes_to_copy with the number of bytes copied. - event = producer_handler_chunked(event, p); + if (p->do_chunking) { + event = producer_handler_dechunked(event, p); + } else { // p->do_dechunking or p->do_chunked_passthru + event = producer_handler_chunked(event, p); + } + + // Passthru: throttle after the walk. chunked_reader is now drained, so + // high_water reflects only consumer backlog and still bounds the cache-write + // dechunked_buffer. Fall through so consumers are re-enabled to drain. + if (event == VC_EVENT_READ_READY && plain_passthru) { + _throttle_chunked_producer(p); + } } else { p->last_event = event; } @@ -1349,7 +1504,7 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) // Data read from producer, reenable consumers for (c = p->consumer_list.head; c; c = c->link.next) { if (c->alive && c->write_vio) { - Dbg(dbg_ctl_http_redirect, "Read ready alive"); + Dbg(dbg_ctl_http_tunnel, "re-enable %s write_vio", c->name); c->write_vio->reenable(); } } @@ -1396,6 +1551,7 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) if (c->write_vio->nbytes == INT64_MAX) { c->write_vio->nbytes = p->bytes_consumed - c->skip_bytes; } + Dbg(dbg_ctl_http_tunnel, "re-enable %s write_vio", c->name); c->write_vio->reenable(); } } @@ -1439,8 +1595,46 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) return sm_callback; } -void -HttpTunnel::consumer_reenable(HttpTunnelConsumer *c) +/** + Check if producer should be re-enabled based on ChunkHandler flow control. + */ +bool +HttpTunnel::_should_reenable_for_chunk_handler_fc(HttpTunnelConsumer *c) +{ + HttpTunnelProducer *p = c->producer; + + if (p == nullptr || !p->alive || !p->read_vio) { + return false; + } + + if (!p->is_handling_chunked_content()) { + // pass this check + return true; + } + + // Check high_water of consumers - re-enable producer read_vio if all consumers don't hit the high_water() + for (auto *ci = p->consumer_list.head; ci; ci = ci->link.next) { + if (!ci->alive || !ci->buffer_reader || !ci->buffer_reader->mbuf || !ci->buffer_reader->mbuf->water_mark) { + continue; + } + + Dbg(dbg_ctl_http_tunnel, "check %s consumer - read_avail = %" PRId64 " / %" PRId64, ci->name, ci->buffer_reader->read_avail(), + ci->buffer_reader->mbuf->water_mark); + + if (ci->buffer_reader->high_water()) { + Dbg(dbg_ctl_http_tunnel, "wait until data in the buffer is consumed"); + return false; + } + } + + return true; +} + +/** + Check if producer should be re-enabled based on tunnel flow control. + */ +bool +HttpTunnel::_should_reenable_for_tunnel_chain_fc(HttpTunnelConsumer *c) { HttpTunnelProducer *p = c->producer; @@ -1460,6 +1654,7 @@ HttpTunnel::consumer_reenable(HttpTunnelConsumer *c) Dbg(dbg_ctl_http_tunnel, "[%" PRId64 "] Throttle %p %" PRId64 " / %" PRId64, sm->sm_id, p, backlog, p->backlog()); } p->throttle(); // p becomes srcp for future calls to this method + return false; } else { if (srcp && srcp->alive && c->is_sink()) { // Check if backlog is below low water - note we need to check @@ -1476,6 +1671,7 @@ HttpTunnel::consumer_reenable(HttpTunnelConsumer *c) } srcp->unthrottle(); if (srcp->read_vio) { + Dbg(dbg_ctl_http_tunnel, "re-enable %s read_vio", srcp->name); srcp->read_vio->reenable(); } // Kick source producer to get flow ... well, flowing. @@ -1492,11 +1688,11 @@ HttpTunnel::consumer_reenable(HttpTunnelConsumer *c) } } } - if (p->read_vio) { - p->read_vio->reenable(); - } + return true; } } + + return false; } // @@ -1524,14 +1720,23 @@ HttpTunnel::consumer_handler(int event, HttpTunnelConsumer *c) ink_assert(c->alive == true); switch (event) { - case VC_EVENT_WRITE_READY: - this->consumer_reenable(c); + case VC_EVENT_WRITE_READY: { + if (_should_reenable_for_tunnel_chain_fc(c) && _should_reenable_for_chunk_handler_fc(c) && p->read_vio) { + Dbg(dbg_ctl_http_tunnel, "re-enable %s read_vio", p->name); + p->read_vio->reenable(); + + // Sometimes producer needs synthetic VC_EVENT_READ_READY event to resume after hitting high water mark + if (p->is_handling_chunked_content() && p->chunked_handler.is_read_avail()) { + this->producer_handler(VC_EVENT_READ_READY, p); + } + } + // Once we get a write ready from the origin, we can assume the connect to some degree succeeded if (c->vc_type == HttpTunnelType_t::HTTP_SERVER) { sm->t_state.current.server->clear_connect_fail(); } break; - + } case VC_EVENT_WRITE_COMPLETE: case VC_EVENT_EOS: case VC_EVENT_ERROR: @@ -1540,6 +1745,7 @@ HttpTunnel::consumer_handler(int event, HttpTunnelConsumer *c) ink_assert(c->alive); ink_assert(c->buffer_reader); if (c->write_vio) { + Dbg(dbg_ctl_http_tunnel, "re-enable %s write_vio", c->name); c->write_vio->reenable(); } c->alive = false; @@ -1583,10 +1789,14 @@ HttpTunnel::consumer_handler(int event, HttpTunnelConsumer *c) // updating the buffer state for the VConnection // that is being reenabled if (p->alive && p->read_vio) { - if (p->is_throttled()) { - this->consumer_reenable(c); - } else { + if (_should_reenable_for_tunnel_chain_fc(c) && _should_reenable_for_chunk_handler_fc(c)) { + Dbg(dbg_ctl_http_tunnel, "re-enable %s read_vio", p->name); p->read_vio->reenable(); + + // Sometimes producer needs synthetic VC_EVENT_READ_READY event to resume after hitting high water mark + if (p->is_handling_chunked_content() && p->chunked_handler.is_read_avail()) { + this->producer_handler(VC_EVENT_READ_READY, p); + } } } // [amc] I don't think this happens but we'll leave a debug trap diff --git a/src/proxy/http/remap/NextHopConsistentHash.cc b/src/proxy/http/remap/NextHopConsistentHash.cc index 635e153c1e6..e307844fd3b 100644 --- a/src/proxy/http/remap/NextHopConsistentHash.cc +++ b/src/proxy/http/remap/NextHopConsistentHash.cc @@ -448,7 +448,11 @@ NextHopConsistentHash::findNextHop(TSHttpTxn txnp, void * /* ih ATS_UNUSED */, t // for retry. if (!pRec->available.load() && host_stat == TS_HOST_STATUS_UP) { _now == 0 ? _now = time(nullptr) : _now = now; - if ((pRec->failedAt.load() + retry_time) < _now) { + // Atomically push failedAt to (_now - retry_time) so only one + // concurrent transaction with this _now takes the retry slot; + // sequential retries with a later _now still pass the window check. + time_t observed = pRec->failedAt.load(); + if ((observed + retry_time) < _now && pRec->failedAt.compare_exchange_strong(observed, _now - retry_time)) { nextHopRetry = true; result.last_parent = pRec->host_index; result.last_lookup = pRec->group_index; diff --git a/src/proxy/http/remap/NextHopRoundRobin.cc b/src/proxy/http/remap/NextHopRoundRobin.cc index c04acbdaa50..9556b54c6d2 100644 --- a/src/proxy/http/remap/NextHopRoundRobin.cc +++ b/src/proxy/http/remap/NextHopRoundRobin.cc @@ -148,7 +148,22 @@ NextHopRoundRobin::findNextHop(TSHttpTxn txnp, void * /* ih ATS_UNUSED */, time_ } else { // if not available, check to see if it can be retried. If so, set the retry flag and temporairly mark it as // available. _now == 0 ? _now = time(nullptr) : _now = now; - if (((result->wrap_around) || (cur_host->failedAt + retry_time) < _now) && host_stat == TS_HOST_STATUS_UP) { + bool retryable = false; + if (host_stat == TS_HOST_STATUS_UP) { + if (result->wrap_around) { + // Wrap-around: force a retry of the host regardless of the timer. + retryable = true; + } else { + // Atomically push failedAt to (_now - retry_time) so only one + // concurrent transaction with this _now takes the retry slot; + // sequential retries with a later _now still pass the window check. + time_t observed = cur_host->failedAt.load(); + if ((observed + retry_time) < _now && cur_host->failedAt.compare_exchange_strong(observed, _now - retry_time)) { + retryable = true; + } + } + } + if (retryable) { // Reuse the parent parentUp = true; parentRetry = true; @@ -201,6 +216,7 @@ NextHopRoundRobin::findNextHop(TSHttpTxn txnp, void * /* ih ATS_UNUSED */, time_ wrapped = wrap_around[cur_grp_index] = result->wrap_around = true; } else { start_host = cur_hst_index = 0; + hst_size = host_groups[cur_grp_index].size(); } } } diff --git a/src/proxy/http/remap/RemapConfig.cc b/src/proxy/http/remap/RemapConfig.cc index 896586ff82d..4de1ea50843 100644 --- a/src/proxy/http/remap/RemapConfig.cc +++ b/src/proxy/http/remap/RemapConfig.cc @@ -995,10 +995,11 @@ process_regex_mapping_config(const char *from_host_lower, url_mapping *new_mappi reg_map->url_map = new_mapping; - // using from_host_lower (and not new_mapping->fromURL.host_get()) - // as this one will be nullptr-terminated (required by pcre_compile) - if (reg_map->regular_expression.compile(from_host_lower) == false) { - Warning("pcre_compile failed! Regex has error starting at %s", from_host_lower); + // Compile the lowercased from-host as the matching pattern, anchored at both ends + // (RE_ANCHORED | RE_ENDANCHORED) so a rule for "cdn.example.com" matches the entire host and + // not a leading or trailing substring (e.g. "prefix.cdn.example.com" or "cdn.example.com.evil.com"). + if (reg_map->regular_expression.compile(from_host_lower, RE_ANCHORED | RE_ENDANCHORED) == false) { + Warning("Failed to compile regex for remap from-host: %s", from_host_lower); goto lFail; } diff --git a/src/proxy/http/remap/RemapProcessor.cc b/src/proxy/http/remap/RemapProcessor.cc index 550af9360e6..22cfddff932 100644 --- a/src/proxy/http/remap/RemapProcessor.cc +++ b/src/proxy/http/remap/RemapProcessor.cc @@ -50,8 +50,13 @@ RemapProcessor::setup_for_remap(HttpTransact::State *s, UrlRewrite *table) int request_port; bool proxy_request = false; - s->reverse_proxy = table->reverse_proxy; s->url_map.set(s->hdr_info.client_request.m_heap); + if (unlikely(table == nullptr)) { + Dbg(dbg_ctl_url_rewrite, "no remap table (shutdown in progress); skipping remap"); + return false; + } + + s->reverse_proxy = table->reverse_proxy; ink_assert(redirect_url != nullptr); @@ -158,13 +163,20 @@ RemapProcessor::finish_remap(HttpTransact::State *s, UrlRewrite *table) { url_mapping *map = nullptr; HTTPHdr *request_header = &s->hdr_info.client_request; - URL *request_url = request_header->url_get(); + URL *request_url = nullptr; char **redirect_url = &s->remap_redirect; char tmp_referer_buf[4096], tmp_redirect_buf[4096], tmp_buf[2048]; int tmp; int from_len; referer_info *ri; + if (unlikely(table == nullptr)) { + Dbg(dbg_ctl_url_rewrite, "no remap table (shutdown in progress); skipping remap completion"); + return false; + } + + request_url = request_header->url_get(); + map = s->url_map.getMapping(); if (nullptr == map) { Dbg(dbg_ctl_url_rewrite, "Could not find corresponding url_mapping for this transaction"); diff --git a/src/proxy/http/remap/UrlRewrite.cc b/src/proxy/http/remap/UrlRewrite.cc index 3460fdb0dd2..6223b67d852 100644 --- a/src/proxy/http/remap/UrlRewrite.cc +++ b/src/proxy/http/remap/UrlRewrite.cc @@ -481,8 +481,8 @@ UrlRewrite::PerformACLFiltering(HttpTransact::State *s, const url_mapping *const if (rp->method_restriction_enabled) { if (method_wksidx >= 0 && method_wksidx < HTTP_WKSIDX_METHODS_CNT) { method_matches = rp->standard_method_lookup[method_wksidx]; - } else if (!rp->nonstandard_methods.empty()) { - method_matches = false; + } else if (rp->nonstandard_methods.empty()) { + method_matches = false; // No nonstandard methods, nothing to match against } else { auto method{s->hdr_info.client_request.method_get()}; method_matches = rp->nonstandard_methods.count(std::string{method}); @@ -1039,6 +1039,8 @@ UrlRewrite::_regexMappingLookup(RegexMappingList ®ex_mappings, URL *request_u continue; } + // The regex is compiled anchored at both ends (see process_regex_mapping_config), so a + // successful match spans the entire request host, never a leading or trailing substring. int match_result = list_iter->regular_expression.exec(std::string_view(request_host, request_host_len), matches); if (match_result > 0) { diff --git a/src/proxy/http/unit_tests/CMakeLists.txt b/src/proxy/http/unit_tests/CMakeLists.txt index a487bef7435..ca37b0246a9 100644 --- a/src/proxy/http/unit_tests/CMakeLists.txt +++ b/src/proxy/http/unit_tests/CMakeLists.txt @@ -19,9 +19,11 @@ add_executable( test_http main.cc "${PROJECT_SOURCE_DIR}/src/iocore/cache/unit_tests/stub.cc" + test_ChunkedHandler.cc test_error_page_selection.cc test_ForwardedConfig.cc test_HttpTransact.cc + test_HttpTransactHeaders.cc test_HttpUserAgent.cc test_PreWarm.cc ) diff --git a/src/proxy/http/unit_tests/test_ChunkedHandler.cc b/src/proxy/http/unit_tests/test_ChunkedHandler.cc new file mode 100644 index 00000000000..a724075620a --- /dev/null +++ b/src/proxy/http/unit_tests/test_ChunkedHandler.cc @@ -0,0 +1,510 @@ +/** @file + + Catch-based unit tests for ChunkedHandler::read_size() and read_trailer(). + + These tests drive the real chunk size line and trailer parsers through a small + subclass fixture and a backing IOBuffer, covering hex size parsing, chunk + extensions, quoted-string extension values per RFC 9110 Section 5.6.4, + quoted-pair escapes, the trailer terminating line per RFC 9112 Section 7.1, and + strict versus non-strict line termination. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include + +#include + +#include "proxy/http/HttpTunnel.h" +#include "iocore/eventsystem/IOBuffer.h" + +// Subclass fixture that drives the otherwise private read_size() parser and lets +// a test seed the parsing state. ChunkedHandler names this class as a friend, so +// it must live at global scope to match that declaration (not in an anonymous +// namespace). +class TestableChunkedHandler : public ChunkedHandler +{ +public: + // Drive the private size parser. Returns the number of input bytes consumed. + int64_t + parse() + { + return read_size(); + } + + // Drive the private trailer parser. Returns the number of input bytes consumed. + int64_t + parse_trailer() + { + return read_trailer(); + } + + void + reset(IOBufferReader *reader, bool strict) + { + chunked_reader = reader; + state = ChunkedState::READ_SIZE; + strict_chunk_parsing = strict; + running_sum = 0; + num_digits = 0; + num_cr = 0; + prev_is_cr = false; + in_quoted_string = false; + in_escape = false; + } + + // Seed the handler at the start of the trailer section, which is where + // read_size() leaves it after the final zero-size chunk (state + // READ_TRAILER_BLANK). drop_chunked_trailers stays false so read_trailer() + // does not need a chunked_buffer. + void + reset_for_trailer(IOBufferReader *reader, bool strict) + { + chunked_reader = reader; + state = ChunkedState::READ_TRAILER_BLANK; + strict_chunk_parsing = strict; + drop_chunked_trailers = false; + } +}; + +namespace +{ + +using State = ChunkedHandler::ChunkedState; + +// Result of parsing a chunk size line: the terminal state, the parsed size, the +// number of input bytes read_size() consumed, and how many bytes were left +// unconsumed in the reader. The consumed/remaining counts are what distinguish a +// parser that keeps a quoted-string value intact (consumes the whole line) from +// one that stops at the first embedded CRLF. +struct ParseResult { + State state; + int64_t size; + int64_t bytes_left; // cur_chunk_bytes_left: body bytes the dechunker will read for this chunk. + int64_t consumed; + int64_t remaining; +}; + +// Feed a complete chunk size line into read_size() in a single buffer. +ParseResult +parse_chunk_size_line(const char *input, bool strict = true) +{ + MIOBuffer *buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + IOBufferReader *reader = buffer->alloc_reader(); + buffer->write(input, std::strlen(input)); + + TestableChunkedHandler handler; + handler.reset(reader, strict); + int64_t consumed = handler.parse(); + + ParseResult result{handler.state, handler.cur_chunk_size, handler.cur_chunk_bytes_left, consumed, reader->read_avail()}; + free_MIOBuffer(buffer); + return result; +} + +// Result of parsing a trailer section: the terminal state, the number of input +// bytes read_trailer() consumed, and how many bytes were left unconsumed. A +// parser that ends the trailers at a bare LF consumes only up to that LF and +// leaves any following bytes (a smuggled request) behind; one that requires CRLF +// rejects (READ_ERROR) instead. +struct TrailerResult { + State state; + int64_t consumed; + int64_t remaining; +}; + +// Feed a trailer section into read_trailer() in a single buffer. The input begins +// where read_size() hands off after the final zero-size chunk, so it does not +// include the leading "0\r\n". +TrailerResult +parse_chunk_trailer(const char *input, bool strict = true) +{ + MIOBuffer *buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + IOBufferReader *reader = buffer->alloc_reader(); + buffer->write(input, std::strlen(input)); + + TestableChunkedHandler handler; + handler.reset_for_trailer(reader, strict); + int64_t consumed = handler.parse_trailer(); + + TrailerResult result{handler.state, consumed, reader->read_avail()}; + free_MIOBuffer(buffer); + return result; +} + +// A single row of the size-line matrix: a label, the raw size-line bytes, the +// parsing mode, the expected terminal state, and the expected chunk size +// (negative means do not check the size, used for the malformed rows). +struct SizeCase { + const char *name; + const char *input; + bool strict; + State expect_state; + int64_t expect_size; +}; + +// A single row of the trailer matrix: a label, the trailer-section bytes (which +// begin after read_size() has consumed the final "0\r\n"), the parsing mode, and +// the expected terminal state. +struct TrailerCase { + const char *name; + const char *input; + bool strict; + State expect_state; +}; + +} // namespace + +TEST_CASE("ChunkedHandler parses chunk sizes", "[chunked]") +{ + SECTION("single hex digit") + { + auto r = parse_chunk_size_line("a\r\n"); + CHECK(r.state == State::READ_CHUNK); + CHECK(r.size == 10); + } + + SECTION("multiple hex digits") + { + auto r = parse_chunk_size_line("ff\r\n"); + CHECK(r.state == State::READ_CHUNK); + CHECK(r.size == 255); + } + + SECTION("final chunk of size zero") + { + auto r = parse_chunk_size_line("0\r\n"); + CHECK(r.state == State::READ_TRAILER_BLANK); + CHECK(r.size == 0); + } +} + +TEST_CASE("ChunkedHandler parses chunk extensions", "[chunked]") +{ + SECTION("token extension value") + { + auto r = parse_chunk_size_line("a;ext=value\r\n"); + CHECK(r.state == State::READ_CHUNK); + CHECK(r.size == 10); + } + + SECTION("quoted-string extension value") + { + auto r = parse_chunk_size_line("a;ext=\"hello world\"\r\n"); + CHECK(r.state == State::READ_CHUNK); + CHECK(r.size == 10); + } + + SECTION("multiple extensions") + { + auto r = parse_chunk_size_line("a;ext1=val1;ext2=\"val2\"\r\n"); + CHECK(r.state == State::READ_CHUNK); + CHECK(r.size == 10); + } + + SECTION("whitespace before the semicolon (BWS) still parses the extension") + { + auto r = parse_chunk_size_line("a ;ext=value\r\n"); + CHECK(r.state == State::READ_CHUNK); + CHECK(r.size == 10); + } +} + +// RFC 9110 Section 5.6.4: a quoted-string cannot contain a bare CR or LF (neither +// qdtext nor quoted-pair permits them). A chunk extension whose quoted value +// embeds CR/LF is therefore malformed, and the parser must reject it (READ_ERROR) +// rather than forward a request a downstream parser could frame differently. +TEST_CASE("ChunkedHandler rejects CR or LF inside a quoted extension value", "[chunked]") +{ + SECTION("a single embedded CRLF is rejected") + { + CHECK(parse_chunk_size_line("1;a=\"\r\nfoo\"\r\n").state == State::READ_ERROR); + } + + SECTION("several embedded CRLFs are rejected") + { + CHECK(parse_chunk_size_line("1;ext=\"line1\r\nline2\r\nline3\"\r\n").state == State::READ_ERROR); + } + + SECTION("a bare LF inside the quoted value is rejected in both modes") + { + CHECK(parse_chunk_size_line("1;a=\"x\ny\"\r\n", true).state == State::READ_ERROR); + CHECK(parse_chunk_size_line("1;a=\"x\ny\"\r\n", false).state == State::READ_ERROR); + } + + SECTION("an embedded CRLF is rejected even when whitespace (BWS) precedes the semicolon") + { + CHECK(parse_chunk_size_line("1 ;a=\"\r\nfoo\"\r\n").state == State::READ_ERROR); + } + + SECTION("a quoted-pair cannot escape a CR") + { + CHECK(parse_chunk_size_line("1;a=\"\\\rx\"\r\n").state == State::READ_ERROR); + } +} + +// A quoted-string value with no CR/LF is well formed and parses to a normal chunk. +TEST_CASE("ChunkedHandler accepts a valid quoted extension value", "[chunked]") +{ + SECTION("an escaped DQUOTE does not close the quoted string") + { + auto r = parse_chunk_size_line("1;ext=\"value\\\"more\"\r\n"); + CHECK(r.state == State::READ_CHUNK); + CHECK(r.size == 1); + CHECK(r.bytes_left == 1); // the dechunker reads exactly the 1 declared body byte + CHECK(r.remaining == 0); + } + + SECTION("an escaped backslash is consumed as a quoted-pair") + { + auto r = parse_chunk_size_line("1;ext=\"path\\\\file\"\r\n"); + CHECK(r.state == State::READ_CHUNK); + CHECK(r.size == 1); + CHECK(r.remaining == 0); + } +} + +// A chunk extension can be split across socket reads. read_size() must suspend +// in READ_EXTENSION when the reader empties mid-extension and resume cleanly on +// the next call. (process_chunked_content() also routes READ_EXTENSION back to +// read_size(); without that, a split extension would crash the dispatcher.) +TEST_CASE("ChunkedHandler resumes a chunk extension split across reads", "[chunked]") +{ + MIOBuffer *buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + IOBufferReader *reader = buffer->alloc_reader(); + + TestableChunkedHandler handler; + handler.reset(reader, true); + + // First read ends in the middle of a quoted-string extension value. + buffer->write("5;ext=\"ab", 9); + handler.parse(); + CHECK(handler.state == State::READ_EXTENSION); + CHECK(reader->read_avail() == 0); + + // The rest of the line arrives in a later read and completes the chunk. + buffer->write("cd\"\r\nhello\r\n", 12); + handler.parse(); + CHECK(handler.state == State::READ_CHUNK); + CHECK(handler.cur_chunk_size == 5); + + free_MIOBuffer(buffer); +} + +TEST_CASE("ChunkedHandler rejects malformed size lines under strict parsing", "[chunked]") +{ + SECTION("non hex, non delimiter character after the size") + { + CHECK(parse_chunk_size_line("ax\r\n", true).state == State::READ_ERROR); + } + + SECTION("extension with no preceding size digits") + { + CHECK(parse_chunk_size_line(";\r\n", true).state == State::READ_ERROR); + } + + SECTION("a second CR before the LF is a protocol error") + { + CHECK(parse_chunk_size_line("a\r\r\n", true).state == State::READ_ERROR); + } + + SECTION("a second CR after an extension is a protocol error") + { + CHECK(parse_chunk_size_line("a;ext=value\r\r\n", true).state == State::READ_ERROR); + } +} + +TEST_CASE("ChunkedHandler honors strict versus non-strict line termination", "[chunked]") +{ + SECTION("a bare LF after an extension is rejected in strict mode") + { + // A chunk size line ending in a bare LF after an extension, with body bytes + // following, must be rejected rather than accepted with the bytes as data. + auto r = parse_chunk_size_line("7;x\nabcwxyz\r\n", true); + CHECK(r.state == State::READ_ERROR); + } + + SECTION("a bare LF after an extension is accepted in non-strict mode") + { + auto lenient = parse_chunk_size_line("a;ext=value\n", false); + CHECK(lenient.state == State::READ_CHUNK); + CHECK(lenient.size == 10); + } + + SECTION("a proper CRLF terminator works in both modes") + { + CHECK(parse_chunk_size_line("a\r\n", true).state == State::READ_CHUNK); + CHECK(parse_chunk_size_line("a\r\n", false).state == State::READ_CHUNK); + } +} + +// RFC 9112 Section 7.1: the trailer section ends with an empty line, "CRLF". A +// bare LF blank line is not a valid terminator. read_trailer() must enforce CRLF +// under strict parsing, the same as the chunk size line does: a bare LF blank +// line ends the trailers on a lenient parser but a strict peer keeps reading, so +// any bytes after the bare LF can be framed by the two parsers as different +// requests (a request boundary desync). Under strict parsing the bare LF must be +// a protocol error so the ambiguous bytes are never forwarded. +TEST_CASE("ChunkedHandler honors strict versus non-strict trailer termination", "[chunked]") +{ + SECTION("a bare LF blank line does not terminate the trailers in strict mode") + { + // Bytes after the bare LF look like a smuggled request. The strict parser + // must reject rather than end the trailers here and forward the trailing GET. + auto r = parse_chunk_trailer("\nGET /smuggled HTTP/1.1\r\n\r\n", true); + CHECK(r.state == State::READ_ERROR); + } + + SECTION("a bare LF blank line still terminates the trailers in non-strict mode") + { + auto r = parse_chunk_trailer("\n", false); + CHECK(r.state == State::READ_DONE); + } + + SECTION("a CRLF blank line terminates the trailers in both modes") + { + CHECK(parse_chunk_trailer("\r\n", true).state == State::READ_DONE); + CHECK(parse_chunk_trailer("\r\n", false).state == State::READ_DONE); + } + + SECTION("a bare LF after a trailer field does not terminate the trailers in strict mode") + { + // The trailer field line ends, returning to a blank line, and the following + // bare LF must not be accepted as the terminator under strict parsing. + auto r = parse_chunk_trailer("X-Trailer: v\r\n\nGET /smuggled HTTP/1.1\r\n\r\n", true); + CHECK(r.state == State::READ_ERROR); + } + + SECTION("a trailer field followed by a CRLF blank line terminates in strict mode") + { + auto r = parse_chunk_trailer("X-Trailer: v\r\n\r\n", true); + CHECK(r.state == State::READ_DONE); + } +} + +// Broad regression matrix for the chunk size line parser. Each row is one input +// and the terminal state the parser must reach, exercised across strict and +// non-strict modes. New variations are added by appending a row, so the matrix +// grows without new boilerplate. +TEST_CASE("ChunkedHandler size-line parsing matrix", "[chunked]") +{ + auto c = GENERATE(values({ + // label input strict expected state size + {"size zero hands off to trailers", "0\r\n", true, State::READ_TRAILER_BLANK, 0 }, + {"single hex digit", "1\r\n", true, State::READ_CHUNK, 1 }, + {"lowercase hex", "a\r\n", true, State::READ_CHUNK, 10 }, + {"uppercase hex", "A\r\n", true, State::READ_CHUNK, 10 }, + {"multiple hex digits", "ff\r\n", true, State::READ_CHUNK, 255 }, + {"four hex digits", "1000\r\n", true, State::READ_CHUNK, 4096}, + {"leading zeros", "00a\r\n", true, State::READ_CHUNK, 10 }, + {"token extension", "a;ext=value\r\n", true, State::READ_CHUNK, 10 }, + {"extension token, no value", "a;ext\r\n", true, State::READ_CHUNK, 10 }, + {"quoted-string extension", "a;ext=\"hello world\"\r\n", true, State::READ_CHUNK, 10 }, + {"multiple extensions", "a;e1=v1;e2=\"v2\"\r\n", true, State::READ_CHUNK, 10 }, + {"BWS before the semicolon", "a ;ext=value\r\n", true, State::READ_CHUNK, 10 }, + {"escaped DQUOTE in quoted value", "1;e=\"a\\\"b\"\r\n", true, State::READ_CHUNK, 1 }, + {"escaped backslash in quoted value", "1;e=\"a\\\\b\"\r\n", true, State::READ_CHUNK, 1 }, + // Malformed: an embedded CR/LF in a quoted-string extension value. + {"embedded CRLF in quoted value, strict", "1;a=\"\r\nfoo\"\r\n", true, State::READ_ERROR, -1 }, + {"embedded CRLF in quoted value, lenient", "1;a=\"\r\nfoo\"\r\n", false, State::READ_ERROR, -1 }, + {"embedded bare LF in quoted value, strict", "1;a=\"x\ny\"\r\n", true, State::READ_ERROR, -1 }, + {"embedded bare LF in quoted value, lenient", "1;a=\"x\ny\"\r\n", false, State::READ_ERROR, -1 }, + {"quoted-pair cannot escape a CR", "1;a=\"\\\rx\"\r\n", true, State::READ_ERROR, -1 }, + {"embedded CRLF in quoted value after BWS", "1 ;a=\"\r\nx\"\r\n", true, State::READ_ERROR, -1 }, + // Malformed: bad size syntax. + {"non-hex character after the size", "ax\r\n", true, State::READ_ERROR, -1 }, + {"non-hex first character", "g\r\n", true, State::READ_ERROR, -1 }, + {"extension with no size digits", ";\r\n", true, State::READ_ERROR, -1 }, + {"a second CR before the LF", "a\r\r\n", true, State::READ_ERROR, -1 }, + {"a second CR after an extension", "a;ext=value\r\r\n", true, State::READ_ERROR, -1 }, + {"control character after the size", "a\x01\r\n", true, State::READ_ERROR, -1 }, + // Line termination: bare LF gated on strict parsing. + {"bare LF after a plain size, strict", "a\nbody", true, State::READ_ERROR, -1 }, + {"bare LF after extension, strict", "7;x\nabcwxyz\r\n", true, State::READ_ERROR, -1 }, + {"bare LF after extension, lenient", "a;ext=value\n", false, State::READ_CHUNK, 10 }, + {"CRLF terminator, strict", "a\r\n", true, State::READ_CHUNK, 10 }, + {"CRLF terminator, lenient", "a\r\n", false, State::READ_CHUNK, 10 }, + })); + + DYNAMIC_SECTION(c.name << (c.strict ? " [strict]" : " [lenient]")) + { + auto r = parse_chunk_size_line(c.input, c.strict); + CHECK(r.state == c.expect_state); + if (c.expect_size >= 0) { + CHECK(r.size == c.expect_size); + } + } +} + +// Broad regression matrix for the trailer parser. The input begins where +// read_size() hands off after the final zero-size chunk, so it does not include +// the leading "0\r\n". The terminating empty line must be a full CRLF under +// strict parsing; a bare LF blank line terminates only in non-strict mode. +TEST_CASE("ChunkedHandler trailer parsing matrix", "[chunked]") +{ + auto c = GENERATE(values({ + // label input strict expected state + {"CRLF terminator, strict", "\r\n", true, State::READ_DONE }, + {"CRLF terminator, lenient", "\r\n", false, State::READ_DONE }, + {"bare LF terminator, strict", "\n", true, State::READ_ERROR}, + {"bare LF terminator, lenient", "\n", false, State::READ_DONE }, + {"bare LF then smuggled request, strict", "\nGET /x HTTP/1.1\r\n\r\n", true, State::READ_ERROR}, + {"trailer field then CRLF terminator, strict", "X-T: v\r\n\r\n", true, State::READ_DONE }, + {"trailer field then CRLF terminator, lenient", "X-T: v\r\n\r\n", false, State::READ_DONE }, + {"trailer field then bare LF terminator, strict", "X-T: v\r\n\nGET /x HTTP/1.1\r\n\r\n", true, State::READ_ERROR}, + {"two trailer fields then CRLF, strict", "X-T: v\r\nY-T: w\r\n\r\n", true, State::READ_DONE }, + {"two trailer fields then CRLF, lenient", "X-T: v\r\nY-T: w\r\n\r\n", false, State::READ_DONE }, + // Only the terminating empty line is gated. A bare LF ending a non-blank + // trailer field line is still tolerated under strict parsing (it does not end + // the message, so it creates no request boundary the peers can disagree on). + {"bare LF ends a field line, tolerated, strict", "X-T: v\nY-T: w\r\n\r\n", true, State::READ_DONE }, + })); + + DYNAMIC_SECTION(c.name << (c.strict ? " [strict]" : " [lenient]")) + { + auto r = parse_chunk_trailer(c.input, c.strict); + CHECK(r.state == c.expect_state); + } +} + +// A trailer can be split across socket reads. read_trailer() must suspend when +// the reader empties mid-trailer and resume cleanly, still rejecting a bare-LF +// terminator that only arrives in a later read under strict parsing. +TEST_CASE("ChunkedHandler resumes a trailer split across reads", "[chunked]") +{ + MIOBuffer *buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_4K); + IOBufferReader *reader = buffer->alloc_reader(); + + TestableChunkedHandler handler; + handler.reset_for_trailer(reader, true); + + // First read ends after a complete trailer field line, parking at a blank line. + buffer->write("X-Trailer: v\r\n", 14); + handler.parse_trailer(); + CHECK(handler.state == State::READ_TRAILER_BLANK); + CHECK(reader->read_avail() == 0); + + // The bare-LF terminator arrives in a later read and must still be rejected. + buffer->write("\n", 1); + handler.parse_trailer(); + CHECK(handler.state == State::READ_ERROR); + + free_MIOBuffer(buffer); +} diff --git a/src/proxy/http/unit_tests/test_HttpTransact.cc b/src/proxy/http/unit_tests/test_HttpTransact.cc index af283fb6017..c509bb20bea 100644 --- a/src/proxy/http/unit_tests/test_HttpTransact.cc +++ b/src/proxy/http/unit_tests/test_HttpTransact.cc @@ -30,7 +30,9 @@ using namespace std::string_view_literals; #include "tscore/Diags.h" #include "tsutil/PostScript.h" +#include "proxy/http/HttpConfig.h" #include "proxy/http/HttpTransact.h" +#include "proxy/http/remap/RemapProcessor.h" #include "records/RecordsConfig.h" #include @@ -41,6 +43,15 @@ TEST_CASE("HttpTransact", "[http]") mime_init(); http_init(); + SECTION("RemapProcessor tolerates a missing remap table") + { + HttpTransact::State state; + RemapProcessor processor; + + CHECK_FALSE(processor.setup_for_remap(&state, nullptr)); + CHECK_FALSE(processor.finish_remap(&state, nullptr)); + } + SECTION("HttpTransact::merge_response_header_with_cached_header") { SECTION("Basic") @@ -271,6 +282,171 @@ TEST_CASE("HttpTransact", "[http]") CHECK(field->has_dups() == true); } + SECTION("Connection-named header from 304 is not merged") + { + HTTPHdr cached_headers; + HTTPHdr response_headers; + ts::PostScript cached_headers_defer([&]() -> void { cached_headers.destroy(); }); + ts::PostScript response_headers_defer([&]() -> void { response_headers.destroy(); }); + + MIMEField *field; + + struct header { + std::string_view name; + std::string_view value; + }; + + struct header cached[] = { + {"AAA", "111"}, + {"BBB", "222"}, + }; + struct header response[] = { + {"Connection", "X-Evil" }, + {"X-Evil", "injected"}, + {"CCC", "333" }, + }; + + cached_headers.create(HTTPType::RESPONSE); + for (auto &&entry : cached) { + field = cached_headers.field_create(entry.name); + cached_headers.field_attach(field); + cached_headers.field_value_set(field, entry.value.data(), entry.value.length()); + } + + response_headers.create(HTTPType::RESPONSE); + for (auto &&entry : response) { + field = response_headers.field_create(entry.name); + response_headers.field_attach(field); + response_headers.field_value_set(field, entry.value.data(), entry.value.length()); + } + + HttpTransact::merge_response_header_with_cached_header(&cached_headers, &response_headers); + + CHECK(cached_headers.fields_count() == 3); + + field = cached_headers.field_find("Connection"sv); + CHECK(field == nullptr); + + field = cached_headers.field_find("X-Evil"sv); + CHECK(field == nullptr); + + field = cached_headers.field_find("CCC"sv); + REQUIRE(field != nullptr); + auto str{field->value_get()}; + CHECK(str == "333"sv); + CHECK(field->has_dups() == false); + } + + SECTION("Multiple Connection tokens are all skipped") + { + HTTPHdr cached_headers; + HTTPHdr response_headers; + ts::PostScript cached_headers_defer([&]() -> void { cached_headers.destroy(); }); + ts::PostScript response_headers_defer([&]() -> void { response_headers.destroy(); }); + + MIMEField *field; + + struct header { + std::string_view name; + std::string_view value; + }; + + struct header cached[] = { + {"AAA", "111"}, + }; + struct header response[] = { + {"Connection", "X-Foo, X-Bar"}, + {"X-Foo", "a" }, + {"X-Bar", "b" }, + {"DDD", "444" }, + }; + + cached_headers.create(HTTPType::RESPONSE); + for (auto &&entry : cached) { + field = cached_headers.field_create(entry.name); + cached_headers.field_attach(field); + cached_headers.field_value_set(field, entry.value.data(), entry.value.length()); + } + + response_headers.create(HTTPType::RESPONSE); + for (auto &&entry : response) { + field = response_headers.field_create(entry.name); + response_headers.field_attach(field); + response_headers.field_value_set(field, entry.value.data(), entry.value.length()); + } + + HttpTransact::merge_response_header_with_cached_header(&cached_headers, &response_headers); + + CHECK(cached_headers.fields_count() == 2); + + field = cached_headers.field_find("X-Foo"sv); + CHECK(field == nullptr); + + field = cached_headers.field_find("X-Bar"sv); + CHECK(field == nullptr); + + field = cached_headers.field_find("DDD"sv); + REQUIRE(field != nullptr); + auto str{field->value_get()}; + CHECK(str == "444"sv); + CHECK(field->has_dups() == false); + } + + SECTION("Connection: TE keeps cached TE and merges normal headers") + { + HTTPHdr cached_headers; + HTTPHdr response_headers; + ts::PostScript cached_headers_defer([&]() -> void { cached_headers.destroy(); }); + ts::PostScript response_headers_defer([&]() -> void { response_headers.destroy(); }); + + MIMEField *field; + + struct header { + std::string_view name; + std::string_view value; + }; + + struct header cached[] = { + {"AAA", "111" }, + {"TE", "trailers"}, + }; + struct header response[] = { + {"Connection", "TE" }, + {"TE", "trailers"}, + {"BBB", "222" }, + }; + + cached_headers.create(HTTPType::RESPONSE); + for (auto &&entry : cached) { + field = cached_headers.field_create(entry.name); + cached_headers.field_attach(field); + cached_headers.field_value_set(field, entry.value.data(), entry.value.length()); + } + + response_headers.create(HTTPType::RESPONSE); + for (auto &&entry : response) { + field = response_headers.field_create(entry.name); + response_headers.field_attach(field); + response_headers.field_value_set(field, entry.value.data(), entry.value.length()); + } + + HttpTransact::merge_response_header_with_cached_header(&cached_headers, &response_headers); + + CHECK(cached_headers.fields_count() == 3); + + field = cached_headers.field_find("TE"sv); + REQUIRE(field != nullptr); + auto str{field->value_get()}; + CHECK(str == "trailers"sv); + CHECK(field->has_dups() == false); + + field = cached_headers.field_find("BBB"sv); + REQUIRE(field != nullptr); + str = field->value_get(); + CHECK(str == "222"sv); + CHECK(field->has_dups() == false); + } + SECTION("Have dup headers 2") { HTTPHdr hdr1; @@ -428,6 +604,7 @@ TEST_CASE("HttpTransact", "[http]") CHECK(str == "999"sv); CHECK(field->has_dups() == false); } + SECTION("Response has superset") { HTTPHdr cached_headers; @@ -744,4 +921,53 @@ TEST_CASE("HttpTransact", "[http]") } } } + + SECTION("HttpTransact::strip_at_headers removes duplicate internal headers") + { + static constexpr char raw_request[] = "GET / HTTP/1.1\r\n" + "Host: example.test\r\n" + "@Ats-Internal: first\r\n" + "X-Keep: ok\r\n" + "@Ats-Internal: second\r\n" + "@Another: third\r\n" + "\r\n"; + + HTTPHdr hdr; + ts::PostScript hdr_defer([&]() -> void { hdr.destroy(); }); + HTTPParser parser; + + if (http_rsb.client_request_at_headers_stripped == nullptr) { + http_rsb.client_request_at_headers_stripped = + Metrics::Counter::createPtr("proxy.process.http.client_request_at_headers_stripped"); + } + if (http_rsb.origin_response_at_headers_stripped == nullptr) { + http_rsb.origin_response_at_headers_stripped = + Metrics::Counter::createPtr("proxy.process.http.origin_response_at_headers_stripped"); + } + + hdr.create(HTTPType::REQUEST); + http_parser_init(&parser); + + auto *start = raw_request; + auto const *end = raw_request + sizeof(raw_request) - 1; + ParseResult err; + + while (true) { + err = hdr.parse_req(&parser, &start, end, true); + if (err != ParseResult::CONT) { + break; + } + } + + REQUIRE(err == ParseResult::DONE); + HttpTransact::strip_at_headers(hdr, HttpTransact::AtHeaderSource::CLIENT_REQUEST, 1); + + CHECK(hdr.field_find("@Ats-Internal"sv) == nullptr); + CHECK(hdr.field_find("@Another"sv) == nullptr); + CHECK(hdr.field_find("Host"sv) != nullptr); + + MIMEField *field = hdr.field_find("X-Keep"sv); + REQUIRE(field != nullptr); + CHECK(field->value_get() == "ok"sv); + } } diff --git a/src/proxy/http/unit_tests/test_HttpTransactHeaders.cc b/src/proxy/http/unit_tests/test_HttpTransactHeaders.cc new file mode 100644 index 00000000000..06c14a1e456 --- /dev/null +++ b/src/proxy/http/unit_tests/test_HttpTransactHeaders.cc @@ -0,0 +1,352 @@ +/** @file + + Unit Tests for HttpTransactHeaders (copy_header_fields) + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +using namespace std::string_view_literals; + +#include "tscore/Diags.h" +#include "tsutil/PostScript.h" +#include "tscore/ink_memory.h" + +#include "proxy/http/HttpTransactHeaders.h" +#include "proxy/hdrs/HTTP.h" +#include "proxy/hdrs/MIME.h" + +#include + +TEST_CASE("HttpTransactHeaders::copy_header_fields", "[http]") +{ + url_init(); + mime_init(); + http_init(); + + // 1) Basic dynamic removal: Connection names X-Custom; X-Custom removed; Host kept; Connection absent + SECTION("basic dynamic removal") + { + HTTPHdr src; + HTTPHdr dst; + ts::PostScript src_defer([&]() -> void { src.destroy(); }); + ts::PostScript dst_defer([&]() -> void { dst.destroy(); }); + + MIMEField *field; + + src.create(HTTPType::REQUEST); + // Host should be preserved + field = src.field_create(static_cast(MIME_FIELD_HOST)); + src.field_attach(field); + src.field_value_set(field, "example.com"sv); + + // Connection: X-Custom + field = src.field_create(static_cast(MIME_FIELD_CONNECTION)); + src.field_attach(field); + src.field_value_set(field, "X-Custom"sv); + + // X-Custom: should be removed after copy + field = src.field_create("X-Custom"sv); + src.field_attach(field); + src.field_value_set(field, "secret"sv); + + HttpTransactHeaders::copy_header_fields(&src, &dst, false, 0); + + // Host remains + auto *h = dst.field_find(static_cast(MIME_FIELD_HOST)); + REQUIRE(h != nullptr); + CHECK(std::string_view{h->value_get()} == "example.com"sv); + + // Connection header should be removed + CHECK(dst.field_find(static_cast(MIME_FIELD_CONNECTION)) == nullptr); + + // X-Custom should be removed (expected to fail before fix) + CHECK(dst.field_find("X-Custom"sv) == nullptr); + } + + // 2) Multiple Connection tokens: both named headers removed; Host kept + SECTION("multiple Connection tokens") + { + HTTPHdr src; + HTTPHdr dst; + ts::PostScript src_defer([&]() -> void { src.destroy(); }); + ts::PostScript dst_defer([&]() -> void { dst.destroy(); }); + + MIMEField *field; + + src.create(HTTPType::REQUEST); + // Host should be preserved + field = src.field_create(static_cast(MIME_FIELD_HOST)); + src.field_attach(field); + src.field_value_set(field, "example.com"sv); + + field = src.field_create(static_cast(MIME_FIELD_CONNECTION)); + src.field_attach(field); + src.field_value_set(field, "X-Foo, X-Bar"sv); + + field = src.field_create("X-Foo"sv); + src.field_attach(field); + src.field_value_set(field, "a"sv); + + field = src.field_create("X-Bar"sv); + src.field_attach(field); + src.field_value_set(field, "b"sv); + + HttpTransactHeaders::copy_header_fields(&src, &dst, false, 0); + + // Host remains + auto *h = dst.field_find(static_cast(MIME_FIELD_HOST)); + REQUIRE(h != nullptr); + CHECK(std::string_view{h->value_get()} == "example.com"sv); + + // X-Foo and X-Bar should be removed (expected to fail before fix) + CHECK(dst.field_find("X-Foo"sv) == nullptr); + CHECK(dst.field_find("X-Bar"sv) == nullptr); + } + + // 3) Connection: TE with TE: trailers preserved + SECTION("TE: trailers preserved when Connection names TE") + { + HTTPHdr src; + HTTPHdr dst; + ts::PostScript src_defer([&]() -> void { src.destroy(); }); + ts::PostScript dst_defer([&]() -> void { dst.destroy(); }); + + MIMEField *field; + + src.create(HTTPType::REQUEST); + field = src.field_create(static_cast(MIME_FIELD_HOST)); + src.field_attach(field); + src.field_value_set(field, "example.com"sv); + + field = src.field_create(static_cast(MIME_FIELD_CONNECTION)); + src.field_attach(field); + src.field_value_set(field, "TE"sv); + + field = src.field_create(static_cast(MIME_FIELD_TE)); + src.field_attach(field); + src.field_value_set(field, "trailers"sv); + + HttpTransactHeaders::copy_header_fields(&src, &dst, false, 0); + + // TE: trailers must be preserved + auto *te = dst.field_find(static_cast(MIME_FIELD_TE)); + REQUIRE(te != nullptr); + CHECK(std::string_view{te->value_get()} == "trailers"sv); + } + + // 4) Nonexistent token name: no crash/assert, Host preserved + SECTION("nonexistent token header is harmless") + { + HTTPHdr src; + HTTPHdr dst; + ts::PostScript src_defer([&]() -> void { src.destroy(); }); + ts::PostScript dst_defer([&]() -> void { dst.destroy(); }); + + MIMEField *field; + + src.create(HTTPType::REQUEST); + field = src.field_create(static_cast(MIME_FIELD_HOST)); + src.field_attach(field); + src.field_value_set(field, "example.com"sv); + + field = src.field_create(static_cast(MIME_FIELD_CONNECTION)); + src.field_attach(field); + src.field_value_set(field, "X-Nonexistent"sv); + + HttpTransactHeaders::copy_header_fields(&src, &dst, false, 0); + + // Host should remain + auto *h = dst.field_find(static_cast(MIME_FIELD_HOST)); + REQUIRE(h != nullptr); + CHECK(std::string_view{h->value_get()} == "example.com"sv); + } + + // 5) Whitespace around tokens: tokenized names removed correctly + SECTION("whitespace-trimmed token handling") + { + HTTPHdr src; + HTTPHdr dst; + ts::PostScript src_defer([&]() -> void { src.destroy(); }); + ts::PostScript dst_defer([&]() -> void { dst.destroy(); }); + + MIMEField *field; + + src.create(HTTPType::REQUEST); + field = src.field_create(static_cast(MIME_FIELD_HOST)); + src.field_attach(field); + src.field_value_set(field, "example.com"sv); + + field = src.field_create(static_cast(MIME_FIELD_CONNECTION)); + src.field_attach(field); + src.field_value_set(field, " X-Foo , X-Bar "sv); + + field = src.field_create("X-Foo"sv); + src.field_attach(field); + src.field_value_set(field, "a"sv); + + field = src.field_create("X-Bar"sv); + src.field_attach(field); + src.field_value_set(field, "b"sv); + + HttpTransactHeaders::copy_header_fields(&src, &dst, false, 0); + + CHECK(dst.field_find("X-Foo"sv) == nullptr); + CHECK(dst.field_find("X-Bar"sv) == nullptr); + } + + // 6) @-prefix protection: @TCPInfo preserved while normal token removed + SECTION("@-prefix token protection while normal token is stripped") + { + HTTPHdr src; + HTTPHdr dst; + ts::PostScript src_defer([&]() -> void { src.destroy(); }); + ts::PostScript dst_defer([&]() -> void { dst.destroy(); }); + + MIMEField *field; + + src.create(HTTPType::REQUEST); + field = src.field_create(static_cast(MIME_FIELD_HOST)); + src.field_attach(field); + src.field_value_set(field, "example.com"sv); + + field = src.field_create(static_cast(MIME_FIELD_CONNECTION)); + src.field_attach(field); + src.field_value_set(field, "@TCPInfo, X-Custom"sv); + + field = src.field_create("@TCPInfo"sv); + src.field_attach(field); + src.field_value_set(field, "TS; data"sv); + + field = src.field_create("X-Custom"sv); + src.field_attach(field); + src.field_value_set(field, "secret"sv); + + HttpTransactHeaders::copy_header_fields(&src, &dst, false, 0); + + REQUIRE(dst.field_find("Host") != nullptr); + + // @TCPInfo should remain + auto *sfield = dst.field_find("@TCPInfo"sv); + REQUIRE(sfield != nullptr); + CHECK(std::string_view{sfield->value_get()} == "TS; data"sv); + + // X-Custom should be removed + CHECK(dst.field_find("X-Custom"sv) == nullptr); + } + + // 7) retain_proxy_auth_hdrs=true: Proxy-Authorization preserved when listed in Connection + SECTION("retain_proxy_auth_hdrs preserves Proxy-Authorization listed in Connection") + { + HTTPHdr src; + HTTPHdr dst; + ts::PostScript src_defer([&]() -> void { src.destroy(); }); + ts::PostScript dst_defer([&]() -> void { dst.destroy(); }); + + MIMEField *field; + + src.create(HTTPType::REQUEST); + field = src.field_create(static_cast(MIME_FIELD_HOST)); + src.field_attach(field); + src.field_value_set(field, "example.com"sv); + + field = src.field_create(static_cast(MIME_FIELD_CONNECTION)); + src.field_attach(field); + src.field_value_set(field, "Proxy-Authorization, X-Custom"sv); + + field = src.field_create(static_cast(MIME_FIELD_PROXY_AUTHORIZATION)); + src.field_attach(field); + src.field_value_set(field, "Basic dXNlcjpwYXNz"sv); + + field = src.field_create("X-Custom"sv); + src.field_attach(field); + src.field_value_set(field, "secret"sv); + + HttpTransactHeaders::copy_header_fields(&src, &dst, true, 0); + + // Proxy-Authorization preserved because retain_proxy_auth_hdrs=true + auto *pa = dst.field_find(static_cast(MIME_FIELD_PROXY_AUTHORIZATION)); + REQUIRE(pa != nullptr); + CHECK(std::string_view{pa->value_get()} == "Basic dXNlcjpwYXNz"sv); + + // X-Custom still stripped + CHECK(dst.field_find("X-Custom"sv) == nullptr); + } + + // 8) retain_proxy_auth_hdrs=false: Proxy-Authorization stripped when listed in Connection + SECTION("Proxy-Authorization stripped when retain_proxy_auth_hdrs is false") + { + HTTPHdr src; + HTTPHdr dst; + ts::PostScript src_defer([&]() -> void { src.destroy(); }); + ts::PostScript dst_defer([&]() -> void { dst.destroy(); }); + + MIMEField *field; + + src.create(HTTPType::REQUEST); + field = src.field_create(static_cast(MIME_FIELD_HOST)); + src.field_attach(field); + src.field_value_set(field, "example.com"sv); + + field = src.field_create(static_cast(MIME_FIELD_CONNECTION)); + src.field_attach(field); + src.field_value_set(field, "Proxy-Authorization"sv); + + field = src.field_create(static_cast(MIME_FIELD_PROXY_AUTHORIZATION)); + src.field_attach(field); + src.field_value_set(field, "Basic dXNlcjpwYXNz"sv); + + HttpTransactHeaders::copy_header_fields(&src, &dst, false, 0); + + // Proxy-Authorization stripped because retain_proxy_auth_hdrs=false + CHECK(dst.field_find(static_cast(MIME_FIELD_PROXY_AUTHORIZATION)) == nullptr); + } + + // 9) retain_proxy_auth_hdrs=true: Proxy-Authenticate preserved when listed in Connection + SECTION("retain_proxy_auth_hdrs preserves Proxy-Authenticate listed in Connection") + { + HTTPHdr src; + HTTPHdr dst; + ts::PostScript src_defer([&]() -> void { src.destroy(); }); + ts::PostScript dst_defer([&]() -> void { dst.destroy(); }); + + MIMEField *field; + + src.create(HTTPType::REQUEST); + field = src.field_create(static_cast(MIME_FIELD_HOST)); + src.field_attach(field); + src.field_value_set(field, "example.com"sv); + + field = src.field_create(static_cast(MIME_FIELD_CONNECTION)); + src.field_attach(field); + src.field_value_set(field, "Proxy-Authenticate"sv); + + field = src.field_create(static_cast(MIME_FIELD_PROXY_AUTHENTICATE)); + src.field_attach(field); + src.field_value_set(field, "Basic realm=\"proxy\""sv); + + HttpTransactHeaders::copy_header_fields(&src, &dst, true, 0); + + // Proxy-Authenticate preserved because retain_proxy_auth_hdrs=true + auto *pa = dst.field_find(static_cast(MIME_FIELD_PROXY_AUTHENTICATE)); + REQUIRE(pa != nullptr); + CHECK(std::string_view{pa->value_get()} == "Basic realm=\"proxy\""sv); + } +} diff --git a/src/proxy/http2/HPACK.cc b/src/proxy/http2/HPACK.cc index 7e4fd974f57..792fa1d3203 100644 --- a/src/proxy/http2/HPACK.cc +++ b/src/proxy/http2/HPACK.cc @@ -430,7 +430,6 @@ encode_literal_header_field_with_indexed_name(uint8_t *buf_start, const uint8_t switch (type) { case HpackField::INDEXED_LITERAL: - indexing_table.add_header_field(header); prefix = 6; flag = 0x40; break; @@ -467,6 +466,11 @@ encode_literal_header_field_with_indexed_name(uint8_t *buf_start, const uint8_t } p += len; + // Encoded successfully; update the dynamic table. + if (type == HpackField::INDEXED_LITERAL) { + indexing_table.add_header_field(header); + } + Dbg(dbg_ctl_hpack_encode, "Encoded field: %d: %.*s", index, static_cast(header.value.size()), header.value.data()); return p - buf_start; } @@ -483,7 +487,6 @@ encode_literal_header_field_with_new_name(uint8_t *buf_start, const uint8_t *buf switch (type) { case HpackField::INDEXED_LITERAL: - indexing_table.add_header_field(header); flag = 0x40; break; case HpackField::NOINDEX_LITERAL: @@ -515,6 +518,11 @@ encode_literal_header_field_with_new_name(uint8_t *buf_start, const uint8_t *buf p += len; + // Encoded successfully; update the dynamic table. + if (type == HpackField::INDEXED_LITERAL) { + indexing_table.add_header_field(header); + } + Dbg(dbg_ctl_hpack_encode, "Encoded field: %.*s: %.*s", static_cast(header.name.size()), header.name.data(), static_cast(header.value.size()), header.value.data()); @@ -548,7 +556,11 @@ decode_indexed_header_field(MIMEFieldWrapper &header, const uint8_t *buf_start, return HPACK_ERROR_COMPRESSION_ERROR; } - if (indexing_table.get_header_field(index, header) == HPACK_ERROR_COMPRESSION_ERROR) { + if (index > UINT32_MAX) { + return HPACK_ERROR_COMPRESSION_ERROR; + } + + if (indexing_table.get_header_field(static_cast(index), header) == HPACK_ERROR_COMPRESSION_ERROR) { return HPACK_ERROR_COMPRESSION_ERROR; } @@ -569,7 +581,7 @@ decode_indexed_header_field(MIMEFieldWrapper &header, const uint8_t *buf_start, // int64_t decode_literal_header_field(MIMEFieldWrapper &header, const uint8_t *buf_start, const uint8_t *buf_end, - HpackIndexingTable &indexing_table) + HpackIndexingTable &indexing_table, uint32_t header_field_max_size) { const uint8_t *p = buf_start; bool isIncremental = false; @@ -592,18 +604,22 @@ decode_literal_header_field(MIMEFieldWrapper &header, const uint8_t *buf_start, return HPACK_ERROR_COMPRESSION_ERROR; } + if (index > UINT32_MAX) { + return HPACK_ERROR_COMPRESSION_ERROR; + } + p += len; // Decode header field name if (index) { - if (indexing_table.get_header_field(index, header) == HPACK_ERROR_COMPRESSION_ERROR) { + if (indexing_table.get_header_field(static_cast(index), header) == HPACK_ERROR_COMPRESSION_ERROR) { return HPACK_ERROR_COMPRESSION_ERROR; } } else { char *name_str = nullptr; uint64_t name_str_len = 0; - len = xpack_decode_string(indexing_table.arena, &name_str, name_str_len, p, buf_end); + len = xpack_decode_string(indexing_table.arena, &name_str, name_str_len, p, buf_end, header_field_max_size); if (len == XPACK_ERROR_COMPRESSION_ERROR) { return HPACK_ERROR_COMPRESSION_ERROR; } @@ -617,23 +633,29 @@ decode_literal_header_field(MIMEFieldWrapper &header, const uint8_t *buf_start, } } - p += len; - header.name_set(name_str, name_str_len); + p += len; + bool name_stored = header.name_set(name_str, name_str_len); indexing_table.arena.str_free(name_str); + if (!name_stored) { + return HPACK_ERROR_COMPRESSION_ERROR; + } } // Decode header field value char *value_str = nullptr; uint64_t value_str_len = 0; - len = xpack_decode_string(indexing_table.arena, &value_str, value_str_len, p, buf_end); + len = xpack_decode_string(indexing_table.arena, &value_str, value_str_len, p, buf_end, header_field_max_size); if (len == XPACK_ERROR_COMPRESSION_ERROR) { return HPACK_ERROR_COMPRESSION_ERROR; } - p += len; - header.value_set(value_str, value_str_len); + p += len; + bool value_stored = header.value_set(value_str, value_str_len); indexing_table.arena.str_free(value_str); + if (!value_stored) { + return HPACK_ERROR_COMPRESSION_ERROR; + } // Incremental Indexing adds header to header table as new entry if (isIncremental) { @@ -687,7 +709,7 @@ update_dynamic_table_size(const uint8_t *buf_start, const uint8_t *buf_end, Hpac int64_t hpack_decode_header_block(HpackIndexingTable &indexing_table, HTTPHdr *hdr, const uint8_t *in_buf, const size_t in_buf_len, - uint32_t max_header_size, uint32_t maximum_table_size) + uint32_t max_header_size, uint32_t maximum_table_size, uint32_t header_field_max_size) { const uint8_t *cursor = in_buf; const uint8_t *const in_buf_end = in_buf + in_buf_len; @@ -717,7 +739,7 @@ hpack_decode_header_block(HpackIndexingTable &indexing_table, HTTPHdr *hdr, cons case HpackField::INDEXED_LITERAL: case HpackField::NOINDEX_LITERAL: case HpackField::NEVERINDEX_LITERAL: - read_bytes = decode_literal_header_field(header, cursor, in_buf_end, indexing_table); + read_bytes = decode_literal_header_field(header, cursor, in_buf_end, indexing_table, header_field_max_size); if (read_bytes == HPACK_ERROR_COMPRESSION_ERROR) { return HPACK_ERROR_COMPRESSION_ERROR; } @@ -774,12 +796,13 @@ hpack_encode_header_block(HpackIndexingTable &indexing_table, uint8_t *out_buf, // Update dynamic table size if (maximum_table_size >= 0) { - indexing_table.update_maximum_size(maximum_table_size); int64_t written = encode_dynamic_table_size_update(cursor, out_buf_end, maximum_table_size); if (written == HPACK_ERROR_COMPRESSION_ERROR) { return HPACK_ERROR_COMPRESSION_ERROR; } cursor += written; + // Encoded successfully; update the dynamic table. + indexing_table.update_maximum_size(maximum_table_size); } for (auto &field : *hdr) { diff --git a/src/proxy/http2/HTTP2.cc b/src/proxy/http2/HTTP2.cc index c4765d5aab3..e8ee95662cf 100644 --- a/src/proxy/http2/HTTP2.cc +++ b/src/proxy/http2/HTTP2.cc @@ -438,9 +438,10 @@ http2_encode_header_blocks(HTTPHdr *in, uint8_t *out, uint32_t out_len, uint32_t */ Http2ErrorCode http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_t buf_len, uint32_t *len_read, HpackHandle &handle, - bool is_trailing_header, uint32_t maximum_table_size, bool is_outbound) + bool is_trailing_header, uint32_t maximum_table_size, uint32_t header_field_max_size, bool is_outbound) { - int64_t result = hpack_decode_header_block(handle, hdr, buf_start, buf_len, Http2::max_header_list_size, maximum_table_size); + int64_t result = hpack_decode_header_block(handle, hdr, buf_start, buf_len, Http2::max_header_list_size, maximum_table_size, + header_field_max_size); if (result < 0) { if (result == HPACK_ERROR_COMPRESSION_ERROR) { diff --git a/src/proxy/http2/Http2ClientSession.cc b/src/proxy/http2/Http2ClientSession.cc index da42c2ef9b2..53955cb2a66 100644 --- a/src/proxy/http2/Http2ClientSession.cc +++ b/src/proxy/http2/Http2ClientSession.cc @@ -80,6 +80,8 @@ Http2ClientSession::start() SET_HANDLER(&Http2ClientSession::main_event_handler); HTTP2_SET_SESSION_HANDLER(&Http2ClientSession::state_read_connection_preface); + _vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); + VIO *read_vio = this->do_io_read(this, INT64_MAX, this->read_buffer); write_vio = this->do_io_write(this, INT64_MAX, this->_write_buffer_reader); @@ -100,9 +102,8 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB this->_milestones.mark(Http2SsnMilestone::OPEN); // Unique client session identifier. - this->con_id = ProxySession::next_connection_id(); - this->_vc = new_vc; - _vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); + this->con_id = ProxySession::next_connection_id(); + this->_vc = new_vc; this->schedule_event = nullptr; this->mutex = new_vc->mutex; @@ -136,6 +137,9 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB this->_handle_if_ssl(new_vc); + if (has_session_hook(TS_HTTP_SSN_START_HOOK)) { + _vc->cancel_inactivity_timeout(); + } do_api_callout(TS_HTTP_SSN_START_HOOK); } @@ -353,9 +357,8 @@ Http2ClientSession::is_protocol_framed() const uint64_t Http2ClientSession::get_received_frame_count(uint64_t type) const { - if (type == 999) { // TS_SSN_INFO_RECEIVED_FRAME_COUNT_H2_UNKNOWN in apidefs.h.in - return this->_frame_counts_in[HTTP2_FRAME_TYPE_MAX]; - } else { - return this->_frame_counts_in[type]; + if (type > HTTP2_FRAME_TYPE_MAX) { + type = HTTP2_FRAME_TYPE_MAX; } + return this->_frame_counts_in[type]; } diff --git a/src/proxy/http2/Http2ConnectionState.cc b/src/proxy/http2/Http2ConnectionState.cc index ca7b0eeb707..3eaa2f464a6 100644 --- a/src/proxy/http2/Http2ConnectionState.cc +++ b/src/proxy/http2/Http2ConnectionState.cc @@ -34,6 +34,7 @@ #include "proxy/http2/Http2DebugNames.h" #include "proxy/http/HttpDebugNames.h" #include "proxy/http/HttpSM.h" +#include "proxy/http/HttpConfig.h" #include "iocore/net/TLSSNISupport.h" @@ -43,9 +44,11 @@ #include "tsutil/PostScript.h" #include "tsutil/LocalBuffer.h" +#include #include -#include +#include #include +#include namespace { @@ -475,8 +478,8 @@ Http2ConnectionState::rcv_headers_frame(const Http2Frame &frame) } else { stream->mark_milestone(Http2StreamMilestone::START_DECODE_HEADERS); } - Http2ErrorCode result = stream->decode_header_blocks(*this->local_hpack_handle, - this->acknowledged_local_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); + Http2ErrorCode result = stream->decode_header_blocks( + *this->local_hpack_handle, this->acknowledged_local_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE), _header_field_max_size); // If this was an outbound connection and the state was already closed, just clear the // headers after processing. We just processed the heaer blocks to keep the dynamic table in @@ -753,8 +756,7 @@ Http2ConnectionState::rcv_settings_frame(const Http2Frame &frame) // error of type FRAME_SIZE_ERROR. if (frame.header().flags & HTTP2_FLAGS_SETTINGS_ACK) { if (frame.header().length == 0) { - this->_process_incoming_settings_ack_frame(); - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + return this->_process_incoming_settings_ack_frame(); } else { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_FRAME_SIZE_ERROR, "recv settings ACK header length not 0"); @@ -812,7 +814,7 @@ Http2ConnectionState::rcv_settings_frame(const Http2Frame &frame) // Update settings count per minute this->increment_received_settings_count(n_settings); // Close this connection if its settings count received exceeds a limit - if (Http2::max_settings_per_frame >= 0 && + if (Http2::max_settings_per_minute >= 0 && this->get_received_settings_count() > static_cast(Http2::max_settings_per_minute)) { Metrics::Counter::increment(http2_rsb.max_settings_per_minute_exceeded); Http2StreamDebug(this->session, stream_id, "Observed too frequent setting changes: %u settings within a last minute", @@ -1091,8 +1093,12 @@ Http2ConnectionState::rcv_continuation_frame(const Http2Frame &frame) "reset too frequent CONTINUATION frames"); } - uint32_t header_blocks_offset = stream->header_blocks_length; - stream->header_blocks_length += payload_length; + uint32_t header_blocks_offset = stream->header_blocks_length; + if (http2_continuation_length_would_overflow(stream->header_blocks_length, payload_length)) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM, + "header blocks length overflow"); + } + stream->header_blocks_length += payload_length; // ATS advertises SETTINGS_MAX_HEADER_LIST_SIZE as a limit of total header blocks length. (Details in [RFC 7560] 10.5.1.) // Make it double to relax the limit in cases of 1) HPACK is used naively, or 2) Huffman Encoding generates large header blocks. @@ -1116,8 +1122,8 @@ Http2ConnectionState::rcv_continuation_frame(const Http2Frame &frame) "continuation no state change"); } - Http2ErrorCode result = stream->decode_header_blocks(*this->local_hpack_handle, - this->acknowledged_local_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); + Http2ErrorCode result = stream->decode_header_blocks( + *this->local_hpack_handle, this->acknowledged_local_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE), _header_field_max_size); if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { if (result == Http2ErrorCode::HTTP2_ERROR_COMPRESSION_ERROR) { @@ -1353,6 +1359,12 @@ Http2ConnectionState::init(Http2CommonSession *ssn) configured_max_continuation_frames_per_minute = Http2::max_continuation_frames_per_minute; configured_max_empty_frames_per_minute = Http2::max_empty_frames_per_minute; + HttpConfigParams *http_config = HttpConfig::acquire(); + if (http_config) { + _header_field_max_size = http_config->http_hdr_field_max_size; + HttpConfig::release(http_config); + } + if (auto snis = session->get_netvc()->get_service(); snis) { if (snis->hints_from_sni.http2_max_settings_frames_per_minute.has_value()) { configured_max_settings_frames_per_minute = snis->hints_from_sni.http2_max_settings_frames_per_minute.value(); @@ -1406,7 +1418,11 @@ Http2ConnectionState::send_connection_preface() configured_settings.set(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE, configured_initial_window_size); } - send_settings_frame(configured_settings); + Http2Error error = send_settings_frame(configured_settings, SEND_EMPTY); + if (error.cls != Http2ErrorClass::HTTP2_ERROR_CLASS_NONE) { + this->_close_connection(error.code); + return; + } // If the session window size is non-default, send a WINDOW_UPDATE right // away. Note that there is no session window size setting in HTTP/2. The @@ -1511,11 +1527,7 @@ Http2ConnectionState::rcv_frame(const Http2Frame *frame) Error("HTTP/2 connection error code=0x%02x client_ip=%s session_id=%" PRId64 " stream_id=%u %s", static_cast(error.code), client_ip, session->get_connection_id(), stream_id, error.msg); } - this->send_goaway_frame(this->latest_streamid_in, error.code); - this->session->set_half_close_local_flag(true); - if (fini_event == nullptr) { - fini_event = this_ethread()->schedule_imm_local(static_cast(this), HTTP2_SESSION_EVENT_FINI); - } + this->_close_connection(error.code); // The streams will be cleaned up by the HTTP2_SESSION_EVENT_FINI event // The Http2ClientSession will shutdown because connection_state.is_state_closed() will be true @@ -1597,11 +1609,7 @@ Http2ConnectionState::main_event_handler(int event, void *edata) } SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); - this->send_goaway_frame(this->latest_streamid_in, error_code); - this->session->set_half_close_local_flag(true); - if (fini_event == nullptr) { - this->fini_event = this_ethread()->schedule_imm_local(static_cast(this), HTTP2_SESSION_EVENT_FINI); - } + this->_close_connection(error_code); } break; // Initiate a graceful shutdown @@ -1746,8 +1754,25 @@ Http2ConnectionState::create_initiating_stream(Http2Error &error) ink_assert(dynamic_cast(this->session->get_proxy_session())); ink_assert(this->session->is_outbound() == true); - uint32_t const initial_stream_window = this->acknowledged_local_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); - Http2Stream *new_stream = + uint32_t const initial_stream_window = this->acknowledged_local_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); + uint32_t initial_stream_window_target = initial_stream_window; + bool const update_dynamic_stream_window = session->is_outbound() && this->_has_dynamic_stream_window(); + + if (update_dynamic_stream_window) { + // See the comment in create_stream() concerning the difference between the + // initial window size and the target window size for dynamic stream window + // sizes. + Http2ConnectionSettings new_settings = local_settings; + initial_stream_window_target = this->_get_configured_receive_session_window_size() / (peer_streams_count_in.load() + 1); + new_settings.set(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE, initial_stream_window_target); + error = this->_check_outgoing_settings_frame(new_settings, !SEND_EMPTY); + if (error.cls != Http2ErrorClass::HTTP2_ERROR_CLASS_NONE) { + this->_close_connection(error.code); + return nullptr; + } + } + + Http2Stream *new_stream = THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), session->get_proxy_session(), -1, peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE), initial_stream_window, STREAM_IS_REGISTERED); @@ -1771,15 +1796,11 @@ Http2ConnectionState::create_initiating_stream(Http2Error &error) // Clear the session timeout. Let the transaction timeouts reign session->get_proxy_session()->cancel_inactivity_timeout(); - if (session->is_outbound() && this->_has_dynamic_stream_window()) { - // See the comment in create_stream() concerning the difference between the - // initial window size and the target window size for dynamic stream window - // sizes. + if (update_dynamic_stream_window) { Http2ConnectionSettings new_settings = local_settings; - uint32_t const initial_stream_window_target = - this->_get_configured_receive_session_window_size() / (peer_streams_count_in.load()); new_settings.set(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE, initial_stream_window_target); - send_settings_frame(new_settings); + Http2Error settings_error = this->send_settings_frame(new_settings, !SEND_EMPTY); + ink_assert(settings_error.cls == Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } return new_stream; @@ -1872,7 +1893,15 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) // The situation of dynamic stream window sizes is described in [RFC 9113] // 6.9.3. initial_stream_window_target = this->_get_configured_receive_session_window_size() / (peer_streams_count_in.load() + 1); + + Http2ConnectionSettings new_settings = local_settings; + new_settings.set(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE, initial_stream_window_target); + error = this->_check_outgoing_settings_frame(new_settings, !SEND_EMPTY); + if (error.cls != Http2ErrorClass::HTTP2_ERROR_CLASS_NONE) { + return nullptr; + } } + Http2Stream *new_stream = THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), session->get_proxy_session(), new_id, peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE), initial_stream_window, STREAM_IS_REGISTERED); @@ -1891,7 +1920,8 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) if (this->_has_dynamic_stream_window()) { Http2ConnectionSettings new_settings = local_settings; new_settings.set(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE, initial_stream_window_target); - send_settings_frame(new_settings); + Http2Error settings_error = this->send_settings_frame(new_settings, !SEND_EMPTY); + ink_assert(settings_error.cls == Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } } else { latest_streamid_out = new_id; @@ -2480,6 +2510,11 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) Http2ErrorCode result = http2_encode_header_blocks(send_hdr, buf, buf_len, &header_blocks_size, *(this->peer_hpack_handle), peer_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { + // The encoder may have mutated the dynamic table for fields written before + // the failure. Rolling that back to keep other streams alive would be + // ideal but is non-trivial; close the connection instead so peer state + // cannot diverge. + this->_close_connection(result); return; } @@ -2531,11 +2566,7 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) // Change stream state if (!stream->change_state(HTTP2_FRAME_TYPE_HEADERS, flags)) { - this->send_goaway_frame(this->latest_streamid_in, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR); - this->session->set_half_close_local_flag(true); - if (fini_event == nullptr) { - fini_event = this_ethread()->schedule_imm_local(static_cast(this), HTTP2_SESSION_EVENT_FINI); - } + this->_close_connection(Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR); return; } @@ -2602,6 +2633,10 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con Http2ErrorCode result = http2_encode_header_blocks(&hdr, buf, buf_len, &header_blocks_size, *(this->peer_hpack_handle), peer_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { + // See send_headers_frame: a partial encode can leave the dynamic table + // out of sync with the peer. Close the connection rather than risk + // desync; rollback would be cleaner but is non-trivial. + this->_close_connection(result); return false; } @@ -2683,11 +2718,7 @@ Http2ConnectionState::send_rst_stream_frame(Http2StreamId id, Http2ErrorCode ec) if (stream != nullptr) { stream->set_tx_error_code({ProxyErrorClass::TXN, static_cast(ec)}); if (!stream->change_state(HTTP2_FRAME_TYPE_RST_STREAM, 0)) { - this->send_goaway_frame(this->latest_streamid_in, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR); - this->session->set_half_close_local_flag(true); - if (fini_event == nullptr) { - fini_event = this_ethread()->schedule_imm_local(static_cast(this), HTTP2_SESSION_EVENT_FINI); - } + this->_close_connection(Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR); return; } @@ -2697,13 +2728,24 @@ Http2ConnectionState::send_rst_stream_frame(Http2StreamId id, Http2ErrorCode ec) this->session->xmit(rst_stream); } -void -Http2ConnectionState::send_settings_frame(const Http2ConnectionSettings &new_settings) +Http2Error +Http2ConnectionState::send_settings_frame(const Http2ConnectionSettings &new_settings, bool send_empty) { constexpr Http2StreamId stream_id = HTTP2_CONNECTION_CONTROL_STREAM; Http2StreamDebug(session, stream_id, "Send SETTINGS frame"); + if (!this->_settings_have_changes(new_settings) && !send_empty) { + Http2StreamDebug(session, stream_id, "Skip SETTINGS frame with no changes"); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } + + Http2Error error = this->_check_outgoing_settings_frame(new_settings, send_empty); + if (error.cls != Http2ErrorClass::HTTP2_ERROR_CLASS_NONE) { + Http2StreamDebug(session, stream_id, "Too many outstanding SETTINGS frames: %zu", this->_outstanding_settings_frames.size()); + return error; + } + Http2SettingsParameter params[HTTP2_SETTINGS_MAX]; size_t params_size = 0; @@ -2717,25 +2759,33 @@ Http2ConnectionState::send_settings_frame(const Http2ConnectionSettings &new_set Http2StreamDebug(session, stream_id, " %s : %u -> %u", Http2DebugNames::get_settings_param_name(id), old_value, new_value); params[params_size++] = {static_cast(id), new_value}; - - // Update current settings - local_settings.set(id, new_settings.get(id)); } } + for (size_t i = 0; i < params_size; ++i) { + local_settings.set(static_cast(params[i].id), params[i].value); + } + Http2SettingsFrame settings(stream_id, HTTP2_FRAME_NO_FLAG, params, params_size); this->_outstanding_settings_frames.emplace(new_settings); this->session->xmit(settings, true); + + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } -void +Http2Error Http2ConnectionState::_process_incoming_settings_ack_frame() { constexpr Http2StreamId stream_id = HTTP2_CONNECTION_CONTROL_STREAM; Http2StreamDebug(session, stream_id, "Processing SETTINGS ACK frame with a queue size of %zu", this->_outstanding_settings_frames.size()); + if (this->_outstanding_settings_frames.empty()) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, + "recv settings ACK with no outstanding SETTINGS frame"); + } + // Do not update this->acknowledged_local_settings yet as // update_initial_local_rwnd relies upon it still pointing to the old value. Http2ConnectionSettings const &old_settings = this->acknowledged_local_settings; @@ -2760,6 +2810,58 @@ Http2ConnectionState::_process_incoming_settings_ack_frame() } this->acknowledged_local_settings = new_settings; this->_outstanding_settings_frames.pop(); + + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); +} + +bool +Http2ConnectionState::_settings_have_changes(const Http2ConnectionSettings &new_settings) const +{ + for (int i = HTTP2_SETTINGS_HEADER_TABLE_SIZE; i < HTTP2_SETTINGS_MAX; ++i) { + Http2SettingsIdentifier id = static_cast(i); + + if (new_settings.get(id) != local_settings.get(id)) { + return true; + } + } + + return false; +} + +Http2Error +Http2ConnectionState::_check_outgoing_settings_frame(const Http2ConnectionSettings &new_settings, bool send_empty) const +{ + bool const send_frame = send_empty || this->_settings_have_changes(new_settings); + + if (send_frame && this->_outstanding_settings_frames.size() >= this->_get_outstanding_settings_frame_limit()) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_SETTINGS_TIMEOUT, + "send settings too many outstanding SETTINGS frames"); + } + + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); +} + +size_t +Http2ConnectionState::_get_outstanding_settings_frame_limit() const +{ + uint32_t const stream_limit = this->_get_configured_max_concurrent_streams(); + + if (stream_limit == std::numeric_limits::max()) { + return stream_limit; + } + return std::max(1, static_cast(stream_limit) + 1); +} + +void +Http2ConnectionState::_close_connection(Http2ErrorCode error_code) +{ + if (!this->session->get_half_close_local_flag()) { + this->send_goaway_frame(this->latest_streamid_in, error_code); + this->session->set_half_close_local_flag(true); + } + if (fini_event == nullptr) { + fini_event = this_ethread()->schedule_imm_local(static_cast(this), HTTP2_SESSION_EVENT_FINI); + } } void diff --git a/src/proxy/http2/Http2ServerSession.cc b/src/proxy/http2/Http2ServerSession.cc index e62dfc853e1..e3e550e1bf9 100644 --- a/src/proxy/http2/Http2ServerSession.cc +++ b/src/proxy/http2/Http2ServerSession.cc @@ -76,6 +76,8 @@ Http2ServerSession::start() SET_HANDLER(&Http2ServerSession::main_event_handler); HTTP2_SET_SESSION_HANDLER(&Http2ServerSession::state_start_frame_read); + _vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); + VIO *read_vio = this->do_io_read(this, INT64_MAX, this->read_buffer); write_vio = this->do_io_write(this, INT64_MAX, this->_write_buffer_reader); @@ -103,9 +105,8 @@ Http2ServerSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB this->_milestones.mark(Http2SsnMilestone::OPEN); // Unique client session identifier. - this->con_id = ProxySession::next_connection_id(); - this->_vc = new_vc; - _vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); + this->con_id = ProxySession::next_connection_id(); + this->_vc = new_vc; this->schedule_event = nullptr; this->mutex = new_vc->mutex; @@ -139,6 +140,9 @@ Http2ServerSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB this->_handle_if_ssl(new_vc); + if (has_session_hook(TS_HTTP_SSN_START_HOOK)) { + _vc->cancel_inactivity_timeout(); + } do_api_callout(TS_HTTP_SSN_START_HOOK); this->add_session(); @@ -422,11 +426,10 @@ Http2ServerSession::is_protocol_framed() const uint64_t Http2ServerSession::get_received_frame_count(uint64_t type) const { - if (type == 999) { // TS_SSN_INFO_RECEIVED_FRAME_COUNT_H2_UNKNOWN in apidefs.h.in - return this->_frame_counts_in[HTTP2_FRAME_TYPE_MAX]; - } else { - return this->_frame_counts_in[type]; + if (type > HTTP2_FRAME_TYPE_MAX) { + type = HTTP2_FRAME_TYPE_MAX; } + return this->_frame_counts_in[type]; } std::function create_h2_server_session = []() -> PoolableSession * { diff --git a/src/proxy/http2/Http2SessionAccept.cc b/src/proxy/http2/Http2SessionAccept.cc index a7a5ad0fb93..0b52df11d2a 100644 --- a/src/proxy/http2/Http2SessionAccept.cc +++ b/src/proxy/http2/Http2SessionAccept.cc @@ -50,10 +50,15 @@ Http2SessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferRead break; } else if (IpAllow::Subject::PROXY == IpAllow::subjects[i] && netvc->get_proxy_protocol_version() != ProxyProtocolVersion::UNDEFINED) { - client_ip = netvc->get_proxy_protocol_src_addr(); - break; + if (sockaddr const *proxy_ip = netvc->get_proxy_protocol_src_addr(); proxy_ip != nullptr) { + client_ip = proxy_ip; + break; + } } } + if (client_ip == nullptr) { + client_ip = netvc->get_remote_addr(); + } IpAllow::ACL session_acl = IpAllow::match(client_ip, IpAllow::match_key_t::SRC_ADDR); if (!session_acl.isValid()) { diff --git a/src/proxy/http2/Http2Stream.cc b/src/proxy/http2/Http2Stream.cc index 6c1f385afe9..6be67db03b9 100644 --- a/src/proxy/http2/Http2Stream.cc +++ b/src/proxy/http2/Http2Stream.cc @@ -248,7 +248,9 @@ Http2Stream::main_event_handler(int event, void *edata) } break; case VC_EVENT_READ_COMPLETE: - read_vio.nbytes = read_vio.ndone; + if (!this->_read_event_paused) { + read_vio.nbytes = read_vio.ndone; + } /* fall through */ case VC_EVENT_READ_READY: _timeout.update_inactivity(); @@ -282,11 +284,11 @@ Http2Stream::main_event_handler(int event, void *edata) } Http2ErrorCode -Http2Stream::decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size) +Http2Stream::decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size, uint32_t header_field_max_size) { - Http2ErrorCode error = - http2_decode_header_blocks(&_receive_header, (const uint8_t *)header_blocks, header_blocks_length, nullptr, hpack_handle, - _trailing_header_is_possible, maximum_table_size, this->is_outbound_connection()); + Http2ErrorCode error = http2_decode_header_blocks(&_receive_header, (const uint8_t *)header_blocks, header_blocks_length, nullptr, + hpack_handle, _trailing_header_is_possible, maximum_table_size, + header_field_max_size, this->is_outbound_connection()); if (error != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { Http2StreamDebug("Error decoding header blocks: %u", static_cast(error)); } @@ -356,6 +358,10 @@ Http2Stream::send_headers(Http2ConnectionState & /* cstate ATS_UNUSED */) return; } + if (!this->receive_end_stream && this->_receive_header.type_get() == HTTPType::REQUEST) { + this->has_body = true; + } + // Is the _sm ready to process the header? if (this->read_vio.nbytes > 0) { if (this->receive_end_stream) { @@ -392,7 +398,6 @@ Http2Stream::send_headers(Http2ConnectionState & /* cstate ATS_UNUSED */) } } else { // End of header but not end of stream, must have some body frames coming - this->has_body = true; this->signal_read_event(VC_EVENT_READ_READY); } } @@ -506,18 +511,34 @@ Http2Stream::change_state(uint8_t type, uint8_t flags) VIO * Http2Stream::do_io_read(Continuation *c, int64_t nbytes, MIOBuffer *buf) { + // DATA can arrive while HttpSM has installed a zero-byte read to pause TXN_START. + // Keep that byte count when the state machine resumes reading the buffered request. + int64_t const gated_ndone = (this->_read_event_paused && nbytes != 0) ? read_vio.ndone : 0; + if (buf) { read_vio.set_writer(buf); } else { read_vio.buffer.clear(); } - read_vio.mutex = c ? c->mutex : this->mutex; - read_vio.cont = c; - read_vio.nbytes = nbytes; - read_vio.ndone = 0; - read_vio.vc_server = this; - read_vio.op = VIO::READ; + read_vio.mutex = c ? c->mutex : this->mutex; + read_vio.cont = c; + read_vio.nbytes = nbytes; + read_vio.ndone = gated_ndone; + read_vio.vc_server = this; + read_vio.op = VIO::READ; + this->_read_event_paused = nbytes == 0; + + if (this->_read_event_paused) { + if (this->_read_vio_event) { + this->_read_vio_event->cancel(); + this->_read_vio_event = nullptr; + } + if (this->read_event) { + this->read_event->cancel(); + this->read_event = nullptr; + } + } // TODO: re-enable read_vio @@ -741,7 +762,7 @@ Http2Stream::update_read_request(bool call_update) ink_release_assert(this->_thread == this_ethread()); SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); - if (read_vio.nbytes == 0 || read_vio.is_disabled()) { + if (this->_read_event_paused || read_vio.nbytes == 0 || read_vio.is_disabled()) { return; } @@ -906,7 +927,8 @@ void Http2Stream::signal_read_event(int event) { if (this->_sm == nullptr || this->read_vio.cont == nullptr || this->read_vio.cont->mutex == nullptr || - this->read_vio.op == VIO::NONE || this->terminate_stream) { + this->read_vio.op == VIO::NONE || this->_read_event_paused || this->read_vio.nbytes == 0 || this->read_vio.is_disabled() || + this->terminate_stream) { return; } @@ -1254,6 +1276,12 @@ Http2Stream::expect_send_trailer() const return this->_expect_send_trailer; } +bool +Http2Stream::can_send_h2_trailer() const +{ + return !send_end_stream; +} + void Http2Stream::set_expect_send_trailer() { diff --git a/src/proxy/http2/test_HPACK.cc b/src/proxy/http2/test_HPACK.cc index 16bfccdfb3c..cde31b7c13f 100644 --- a/src/proxy/http2/test_HPACK.cc +++ b/src/proxy/http2/test_HPACK.cc @@ -35,6 +35,7 @@ const static int MAX_REQUEST_HEADER_SIZE = 131072; const static int MAX_TABLE_SIZE = 4096; +const static int MAX_FIELD_SIZE = 32768; using namespace std; @@ -194,7 +195,8 @@ test_decoding(const string &filename) case 'w': parse_line(line, 6, name, value); unpacked_len = unpack(value, unpacked); - hpack_decode_header_block(indexing_table, &decoded, unpacked, unpacked_len, MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE); + hpack_decode_header_block(indexing_table, &decoded, unpacked, unpacked_len, MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE, + MAX_FIELD_SIZE); break; } break; @@ -250,7 +252,7 @@ test_encoding(const string &filename_in, const string &filename_out) break; } hpack_decode_header_block(indexing_table_for_decoding, &decoded, encoded, written, MAX_REQUEST_HEADER_SIZE, - MAX_TABLE_SIZE); + MAX_TABLE_SIZE, MAX_FIELD_SIZE); if (compare_header_fields(&decoded, &original) != 0) { result = seqnum; break; @@ -295,7 +297,8 @@ test_encoding(const string &filename_in, const string &filename_out) result = seqnum; return result; } - hpack_decode_header_block(indexing_table_for_decoding, &decoded, encoded, written, MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE); + hpack_decode_header_block(indexing_table_for_decoding, &decoded, encoded, written, MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE, + MAX_FIELD_SIZE); if (compare_header_fields(&decoded, &original) != 0) { result = seqnum; return result; diff --git a/src/proxy/http2/unit_tests/test_HTTP2.cc b/src/proxy/http2/unit_tests/test_HTTP2.cc index 7b0a1bbb982..23d8a870960 100644 --- a/src/proxy/http2/unit_tests/test_HTTP2.cc +++ b/src/proxy/http2/unit_tests/test_HTTP2.cc @@ -143,6 +143,115 @@ TEST_CASE("Convert HTTPHdr", "[HTTP2]") "\r\n")); } + SECTION("reject CRLF in header value") + { + const char request[] = "GET /index.html HTTP/1.1\r\n" + "Host: trafficserver.apache.org\r\n" + "User-Agent: foobar\r\n" + "\r\n"; + + HTTPHdr hdr; + ts::PostScript hdr_defer([&]() -> void { hdr.destroy(); }); + hdr.create(HTTPType::REQUEST, HTTP_2_0); + + const char *start = request; + const char *end = request + sizeof(request) - 1; + hdr.parse_req(&parser, &start, end, true); + http2_convert_header_from_1_1_to_2(&hdr); + + MIMEField *evil = hdr.field_create("x-injected"); + hdr.field_attach(evil); + evil->value_set(hdr.m_heap, hdr.m_mime, std::string_view{"safe\r\ninjected: evil"}); + + HTTPHdr hdr_out; + ts::PostScript hdr_out_defer([&]() -> void { hdr_out.destroy(); }); + hdr_out.create(HTTPType::REQUEST); + hdr_out.copy(&hdr); + + CHECK(http2_convert_header_from_2_to_1_1(&hdr_out) == ParseResult::ERROR); + } + + SECTION("reject bare CR in header value") + { + const char request[] = "GET /index.html HTTP/1.1\r\n" + "Host: trafficserver.apache.org\r\n" + "\r\n"; + + HTTPHdr hdr; + ts::PostScript hdr_defer([&]() -> void { hdr.destroy(); }); + hdr.create(HTTPType::REQUEST, HTTP_2_0); + + const char *start = request; + const char *end = request + sizeof(request) - 1; + hdr.parse_req(&parser, &start, end, true); + http2_convert_header_from_1_1_to_2(&hdr); + + MIMEField *evil = hdr.field_create("x-injected"); + hdr.field_attach(evil); + evil->value_set(hdr.m_heap, hdr.m_mime, std::string_view{"before\rafter"}); + + HTTPHdr hdr_out; + ts::PostScript hdr_out_defer([&]() -> void { hdr_out.destroy(); }); + hdr_out.create(HTTPType::REQUEST); + hdr_out.copy(&hdr); + + CHECK(http2_convert_header_from_2_to_1_1(&hdr_out) == ParseResult::ERROR); + } + + SECTION("reject bare LF in header value") + { + const char request[] = "GET /index.html HTTP/1.1\r\n" + "Host: trafficserver.apache.org\r\n" + "\r\n"; + + HTTPHdr hdr; + ts::PostScript hdr_defer([&]() -> void { hdr.destroy(); }); + hdr.create(HTTPType::REQUEST, HTTP_2_0); + + const char *start = request; + const char *end = request + sizeof(request) - 1; + hdr.parse_req(&parser, &start, end, true); + http2_convert_header_from_1_1_to_2(&hdr); + + MIMEField *evil = hdr.field_create("x-injected"); + hdr.field_attach(evil); + evil->value_set(hdr.m_heap, hdr.m_mime, std::string_view{"before\nafter"}); + + HTTPHdr hdr_out; + ts::PostScript hdr_out_defer([&]() -> void { hdr_out.destroy(); }); + hdr_out.create(HTTPType::REQUEST); + hdr_out.copy(&hdr); + + CHECK(http2_convert_header_from_2_to_1_1(&hdr_out) == ParseResult::ERROR); + } + + SECTION("accept clean header value") + { + const char request[] = "GET /index.html HTTP/1.1\r\n" + "Host: trafficserver.apache.org\r\n" + "\r\n"; + + HTTPHdr hdr; + ts::PostScript hdr_defer([&]() -> void { hdr.destroy(); }); + hdr.create(HTTPType::REQUEST, HTTP_2_0); + + const char *start = request; + const char *end = request + sizeof(request) - 1; + hdr.parse_req(&parser, &start, end, true); + http2_convert_header_from_1_1_to_2(&hdr); + + MIMEField *clean = hdr.field_create("x-clean"); + hdr.field_attach(clean); + clean->value_set(hdr.m_heap, hdr.m_mime, std::string_view{"perfectly-fine-value"}); + + HTTPHdr hdr_out; + ts::PostScript hdr_out_defer([&]() -> void { hdr_out.destroy(); }); + hdr_out.create(HTTPType::REQUEST); + hdr_out.copy(&hdr); + + CHECK(http2_convert_header_from_2_to_1_1(&hdr_out) == ParseResult::DONE); + } + SECTION("response") { const char response[] = "HTTP/1.1 200 OK\r\n" @@ -196,3 +305,45 @@ TEST_CASE("Convert HTTPHdr", "[HTTP2]") CHECK_THAT(buf, Catch::Matchers::StartsWith("HTTP/1.1 200 OK\r\n\r\n")); } } + +// Regression: Http2ConnectionState::rcv_continuation_frame accumulates +// the size of every CONTINUATION payload into stream->header_blocks_length, a uint32_t. +// Before the fix, the increment was performed without overflow checking, so a crafted +// sequence of CONTINUATION frames whose payloads sum to more than UINT32_MAX would +// wrap the accumulator, and the subsequent ats_realloc would allocate a buffer smaller +// than the pre-wrap offset that memcpy then writes to. +TEST_CASE("CONTINUATION header_blocks_length overflow guard", "[HTTP2]") +{ + SECTION("zero accumulator and zero payload do not overflow") + { + CHECK_FALSE(http2_continuation_length_would_overflow(0u, 0u)); + } + + SECTION("small additions do not overflow") + { + CHECK_FALSE(http2_continuation_length_would_overflow(0u, 16384u)); + CHECK_FALSE(http2_continuation_length_would_overflow(16384u, 16384u)); + CHECK_FALSE(http2_continuation_length_would_overflow(1u << 20, 1u << 20)); + } + + SECTION("sum that exactly fills uint32_t is allowed") + { + CHECK_FALSE(http2_continuation_length_would_overflow(UINT32_MAX, 0u)); + CHECK_FALSE(http2_continuation_length_would_overflow(0u, UINT32_MAX)); + CHECK_FALSE(http2_continuation_length_would_overflow(UINT32_MAX - 1u, 1u)); + CHECK_FALSE(http2_continuation_length_would_overflow(1u, UINT32_MAX - 1u)); + } + + SECTION("sum exceeding uint32_t by one wraps and must be rejected") + { + CHECK(http2_continuation_length_would_overflow(UINT32_MAX, 1u)); + CHECK(http2_continuation_length_would_overflow(1u, UINT32_MAX)); + } + + SECTION("realistic attack shape: prior bytes plus a max HTTP/2 frame payload") + { + constexpr uint32_t max_frame_payload = (1u << 24) - 1u; + CHECK(http2_continuation_length_would_overflow(UINT32_MAX - max_frame_payload + 1u, max_frame_payload)); + CHECK_FALSE(http2_continuation_length_would_overflow(UINT32_MAX - max_frame_payload, max_frame_payload)); + } +} diff --git a/src/proxy/http2/unit_tests/test_HpackIndexingTable.cc b/src/proxy/http2/unit_tests/test_HpackIndexingTable.cc index ad373211fb8..0e7385113e8 100644 --- a/src/proxy/http2/unit_tests/test_HpackIndexingTable.cc +++ b/src/proxy/http2/unit_tests/test_HpackIndexingTable.cc @@ -26,16 +26,20 @@ #include #include +#include #include #include "proxy/http2/HPACK.h" +#include "proxy/hdrs/XPACK.h" -static constexpr int DYNAMIC_TABLE_SIZE_FOR_REGRESSION_TEST = 256; -static constexpr int BUFSIZE_FOR_REGRESSION_TEST = 128; -static constexpr int MAX_TEST_FIELD_NUM = 8; -static constexpr int MAX_REQUEST_HEADER_SIZE = 131072; -static constexpr int MAX_TABLE_SIZE = 4096; +static constexpr int DYNAMIC_TABLE_SIZE_FOR_REGRESSION_TEST = 256; +static constexpr int BUFSIZE_FOR_REGRESSION_TEST = 128; +static constexpr int MAX_TEST_FIELD_NUM = 8; +static constexpr int MAX_REQUEST_HEADER_SIZE = 131072; +static constexpr int MAX_TABLE_SIZE = 4096; +static constexpr int MAX_FIELD_SIZE = 32768; +static constexpr uint64_t OVERSIZED_HPACK_INDEX = (static_cast(1) << 32) + 58; namespace { @@ -49,6 +53,18 @@ destroy_http_hdr(HTTPHdr *hdr) hdr->destroy(); delete hdr; } + +int64_t +encode_oversized_hpack_index(uint8_t *buf, size_t buf_len, uint8_t prefix, uint8_t flag) +{ + memset(buf, 0, buf_len); + buf[0] = flag; + + int64_t len = xpack_encode_integer(buf, buf + buf_len, OVERSIZED_HPACK_INDEX, prefix); + REQUIRE(len > 0); + + return len; +} } // namespace TEST_CASE("HPACK low level APIs", "[hpack]") @@ -103,6 +119,22 @@ TEST_CASE("HPACK low level APIs", "[hpack]") REQUIRE(actual_value == std::string_view{i.raw_value}); } } + + SECTION("rejects oversized indexed header field index") + { + uint8_t buf[BUFSIZE_FOR_REGRESSION_TEST]; + int64_t encoded_len = encode_oversized_hpack_index(buf, sizeof(buf), 7, 0x80); + + HpackIndexingTable indexing_table(4096); + std::unique_ptr headers(new HTTPHdr, destroy_http_hdr); + headers->create(HTTPType::REQUEST); + MIMEField *field = mime_field_create(headers->m_heap, headers->m_http->m_fields_impl); + MIMEFieldWrapper header(field, headers->m_heap, headers->m_http->m_fields_impl); + + int64_t len = decode_indexed_header_field(header, buf, buf + encoded_len, indexing_table); + + REQUIRE(len == HPACK_ERROR_COMPRESSION_ERROR); + } } SECTION("literal_header_field") @@ -225,7 +257,8 @@ TEST_CASE("HPACK low level APIs", "[hpack]") MIMEField *field = mime_field_create(headers->m_heap, headers->m_http->m_fields_impl); MIMEFieldWrapper header(field, headers->m_heap, headers->m_http->m_fields_impl); - int len = decode_literal_header_field(header, i.encoded_field, i.encoded_field + i.encoded_field_len, indexing_table); + int len = decode_literal_header_field(header, i.encoded_field, i.encoded_field + i.encoded_field_len, indexing_table, + MAX_FIELD_SIZE); REQUIRE(len == i.encoded_field_len); auto name{header.name_get()}; @@ -238,6 +271,76 @@ TEST_CASE("HPACK low level APIs", "[hpack]") } } } + + SECTION("rejects oversized literal header field name index") + { + const static struct { + uint8_t prefix; + uint8_t flag; + } oversized_literal_index_cases[] = { + {6, 0x40}, // INDEXED_LITERAL + {4, 0x00}, // NOINDEX_LITERAL + {4, 0x10}, // NEVERINDEX_LITERAL + }; + + for (const auto &i : oversized_literal_index_cases) { + uint8_t buf[BUFSIZE_FOR_REGRESSION_TEST]; + int64_t encoded_len = encode_oversized_hpack_index(buf, sizeof(buf), i.prefix, i.flag); + uint8_t value[] = {0x05, 'v', 'a', 'l', 'u', 'e'}; + memcpy(buf + encoded_len, value, sizeof(value)); + encoded_len += sizeof(value); + + HpackIndexingTable indexing_table(4096); + std::unique_ptr headers(new HTTPHdr, destroy_http_hdr); + headers->create(HTTPType::REQUEST); + MIMEField *field = mime_field_create(headers->m_heap, headers->m_http->m_fields_impl); + MIMEFieldWrapper header(field, headers->m_heap, headers->m_http->m_fields_impl); + + int64_t len = decode_literal_header_field(header, buf, buf + encoded_len, indexing_table, MAX_FIELD_SIZE); + + REQUIRE(len == HPACK_ERROR_COMPRESSION_ERROR); + } + } + + SECTION("dynamic table is not mutated when encoding fails") + { + HpackHeaderField const header{"custom-key", "custom-header"}; + uint8_t buf[1]; + + // A zero-length output buffer guarantees the first xpack_encode_* + // call fails. The dynamic table must remain untouched. + { + HpackIndexingTable indexing_table(4096); + uint32_t const baseline_size = indexing_table.size(); + + int64_t len = + encode_literal_header_field_with_indexed_name(buf, buf, header, 4, indexing_table, HpackField::INDEXED_LITERAL); + REQUIRE(len == HPACK_ERROR_COMPRESSION_ERROR); + REQUIRE(indexing_table.size() == baseline_size); + } + { + HpackIndexingTable indexing_table(4096); + uint32_t const baseline_size = indexing_table.size(); + + int64_t len = encode_literal_header_field_with_new_name(buf, buf, header, indexing_table, HpackField::INDEXED_LITERAL); + REQUIRE(len == HPACK_ERROR_COMPRESSION_ERROR); + REQUIRE(indexing_table.size() == baseline_size); + } + + // The size-update branch in hpack_encode_header_block must likewise + // leave maximum_size untouched on encode failure. + { + std::unique_ptr headers(new HTTPHdr, destroy_http_hdr); + headers->create(HTTPType::REQUEST); + + HpackIndexingTable indexing_table(4096); + uint32_t const baseline_max = indexing_table.maximum_size(); + + int64_t len = hpack_encode_header_block(indexing_table, buf, 0, headers.get(), 256); + REQUIRE(len == HPACK_ERROR_COMPRESSION_ERROR); + REQUIRE(indexing_table.maximum_size() == baseline_max); + } + } } } @@ -448,7 +551,8 @@ TEST_CASE("HPACK high level APIs", "[hpack]") headers->create(HTTPType::REQUEST); hpack_decode_header_block(indexing_table, headers.get(), encoded_field_request_test_case[i].encoded_field, - encoded_field_request_test_case[i].encoded_field_len, MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE); + encoded_field_request_test_case[i].encoded_field_len, MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE, + MAX_FIELD_SIZE); for (unsigned int j = 0; j < sizeof(raw_field_request_test_case[i]) / sizeof(raw_field_request_test_case[i][0]); j++) { const char *expected_name = raw_field_request_test_case[i][j].raw_name; @@ -483,8 +587,8 @@ TEST_CASE("HPACK high level APIs", "[hpack]") uint8_t data[] = {0x82, 0x86, 0x84, 0x41, 0x0f, 0x77, 0x77, 0x77, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d}; - int64_t len = - hpack_decode_header_block(indexing_table, headers.get(), data, sizeof(data), MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE); + int64_t len = hpack_decode_header_block(indexing_table, headers.get(), data, sizeof(data), MAX_REQUEST_HEADER_SIZE, + MAX_TABLE_SIZE, MAX_FIELD_SIZE); CHECK(len == sizeof(data)); CHECK(indexing_table.maximum_size() == 4096); CHECK(indexing_table.size() == 57); @@ -497,8 +601,8 @@ TEST_CASE("HPACK high level APIs", "[hpack]") uint8_t data[] = {0x20}; - int64_t len = - hpack_decode_header_block(indexing_table, headers.get(), data, sizeof(data), MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE); + int64_t len = hpack_decode_header_block(indexing_table, headers.get(), data, sizeof(data), MAX_REQUEST_HEADER_SIZE, + MAX_TABLE_SIZE, MAX_FIELD_SIZE); CHECK(len == sizeof(data)); CHECK(indexing_table.maximum_size() == 0); CHECK(indexing_table.size() == 0); @@ -511,8 +615,8 @@ TEST_CASE("HPACK high level APIs", "[hpack]") uint8_t data[] = {0x3f, 0xe1, 0x1f}; - int64_t len = - hpack_decode_header_block(indexing_table, headers.get(), data, sizeof(data), MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE); + int64_t len = hpack_decode_header_block(indexing_table, headers.get(), data, sizeof(data), MAX_REQUEST_HEADER_SIZE, + MAX_TABLE_SIZE, MAX_FIELD_SIZE); CHECK(len == sizeof(data)); CHECK(indexing_table.maximum_size() == 4096); CHECK(indexing_table.size() == 0); @@ -525,9 +629,146 @@ TEST_CASE("HPACK high level APIs", "[hpack]") uint8_t data[] = {0x3f, 0xe2, 0x1f}; - int64_t len = - hpack_decode_header_block(indexing_table, headers.get(), data, sizeof(data), MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE); + int64_t len = hpack_decode_header_block(indexing_table, headers.get(), data, sizeof(data), MAX_REQUEST_HEADER_SIZE, + MAX_TABLE_SIZE, MAX_FIELD_SIZE); CHECK(len == HPACK_ERROR_COMPRESSION_ERROR); } } } + +namespace +{ +// Build a single literal-with-incremental-indexing header field (RFC 7541 6.2.1) +// with a non-Huffman, new (index 0) name and value. The name and value lengths +// are written using HPACK integer encoding (7-bit prefix, high bit reserved for +// the Huffman flag, which is left clear here). +std::vector +build_literal_incremental_field(std::string_view name, std::string_view value) +{ + std::vector block; + + // Pattern '01' in the top two bits selects "Literal Header Field with + // Incremental Indexing", and a 6-bit index of 0 means a new name follows. + block.push_back(0x40); + + uint8_t prefix[16]; + + // Header field name: length prefix (n = 7) followed by the raw octets. + prefix[0] = 0x00; // clear the Huffman flag in the prefix octet + const int64_t name_len_bytes = xpack_encode_integer(prefix, prefix + sizeof(prefix), name.length(), 7); + block.insert(block.end(), prefix, prefix + name_len_bytes); + block.insert(block.end(), name.begin(), name.end()); + + // Header field value: length prefix (n = 7) followed by the raw octets. + prefix[0] = 0x00; // clear the Huffman flag in the prefix octet + const int64_t value_len_bytes = xpack_encode_integer(prefix, prefix + sizeof(prefix), value.length(), 7); + block.insert(block.end(), prefix, prefix + value_len_bytes); + block.insert(block.end(), value.begin(), value.end()); + + return block; +} + +// Build a single literal-with-incremental-indexing header field (RFC 7541 6.2.1) +// using an INDEXED name (the name is taken from the static/dynamic table at the +// given index) and a new, non-Huffman value. The 6-bit prefix carries the table +// index. Used to exercise the oversized-value path when the name does not come +// from the wire. +std::vector +build_literal_incremental_field_indexed_name(uint32_t name_index, std::string_view value) +{ + std::vector block; + + uint8_t prefix[16]; + + // Pattern '01' in the top two bits selects "Literal Header Field with + // Incremental Indexing". The remaining 6 bits encode the name's table index. + prefix[0] = 0x40; + const int64_t name_idx_bytes = xpack_encode_integer(prefix, prefix + sizeof(prefix), name_index, 6); + block.insert(block.end(), prefix, prefix + name_idx_bytes); + + // Header field value: length prefix (n = 7) followed by the raw octets. + prefix[0] = 0x00; // clear the Huffman flag in the prefix octet + const int64_t value_len_bytes = xpack_encode_integer(prefix, prefix + sizeof(prefix), value.length(), 7); + block.insert(block.end(), prefix, prefix + value_len_bytes); + block.insert(block.end(), value.begin(), value.end()); + + return block; +} +} // namespace + +// The header field-size limit caps both the name and the value at UINT16_MAX +// (65535) octets. An HPACK-decoded literal header field whose name or value is +// longer than UINT16_MAX cannot be stored under that cap, so the decoder rejects +// such a field with HPACK_ERROR_COMPRESSION_ERROR (a connection error) BEFORE +// inserting anything into the dynamic indexing table. +TEST_CASE("HPACK oversized literal field is rejected", "[hpack]") +{ + // Use a generous header-size and per-field budget so the failure is + // attributable to the uint16_t field-length limit, not to the max_header_size + // or header_field_max_size decoder limits. + constexpr uint32_t LARGE_MAX_HEADER_SIZE = 1 * 1024 * 1024; + constexpr size_t OVERSIZED_LEN = 70000; // > UINT16_MAX (65535) + + SECTION("oversized value") + { + HpackIndexingTable indexing_table(4096); + REQUIRE(indexing_table.size() == 0); + + std::string value(OVERSIZED_LEN, 'A'); + std::vector block = build_literal_incremental_field("x-big-value", value); + + std::unique_ptr headers(new HTTPHdr, destroy_http_hdr); + headers->create(HTTPType::REQUEST); + + int64_t len = hpack_decode_header_block(indexing_table, headers.get(), block.data(), block.size(), LARGE_MAX_HEADER_SIZE, + MAX_TABLE_SIZE, LARGE_MAX_HEADER_SIZE); + + CHECK(len == HPACK_ERROR_COMPRESSION_ERROR); + + // The unstorable field must not have been inserted into the dynamic table. + CHECK(indexing_table.size() == 0); + } + + SECTION("oversized name") + { + HpackIndexingTable indexing_table(4096); + REQUIRE(indexing_table.size() == 0); + + std::string name(OVERSIZED_LEN, 'a'); + std::vector block = build_literal_incremental_field(name, "small-value"); + + std::unique_ptr headers(new HTTPHdr, destroy_http_hdr); + headers->create(HTTPType::REQUEST); + + int64_t len = hpack_decode_header_block(indexing_table, headers.get(), block.data(), block.size(), LARGE_MAX_HEADER_SIZE, + MAX_TABLE_SIZE, LARGE_MAX_HEADER_SIZE); + + CHECK(len == HPACK_ERROR_COMPRESSION_ERROR); + + // The unstorable field must not have been inserted into the dynamic table. + CHECK(indexing_table.size() == 0); + } + + SECTION("indexed name with oversized value") + { + HpackIndexingTable indexing_table(4096); + REQUIRE(indexing_table.size() == 0); + + // Static-table index 4 is ":path", so only the value comes from the wire. + constexpr uint32_t STATIC_INDEX_PATH = 4; + + std::string value(OVERSIZED_LEN, 'A'); + std::vector block = build_literal_incremental_field_indexed_name(STATIC_INDEX_PATH, value); + + std::unique_ptr headers(new HTTPHdr, destroy_http_hdr); + headers->create(HTTPType::REQUEST); + + int64_t len = hpack_decode_header_block(indexing_table, headers.get(), block.data(), block.size(), LARGE_MAX_HEADER_SIZE, + MAX_TABLE_SIZE, LARGE_MAX_HEADER_SIZE); + + CHECK(len == HPACK_ERROR_COMPRESSION_ERROR); + + // The unstorable field must not have been inserted into the dynamic table. + CHECK(indexing_table.size() == 0); + } +} diff --git a/src/proxy/http3/Http3Session.cc b/src/proxy/http3/Http3Session.cc index a68f61cf003..9a001306bf7 100644 --- a/src/proxy/http3/Http3Session.cc +++ b/src/proxy/http3/Http3Session.cc @@ -26,6 +26,7 @@ #include "proxy/http3/Http3.h" #include "proxy/http3/Http3Types.h" +#include "proxy/http/HttpConfig.h" // // HQSession @@ -177,11 +178,19 @@ HQSession::main_event_handler(int event, void *edata) // Http3Session::Http3Session(NetVConnection *vc) : HQSession(vc) { - QUICConnection *qc = vc->get_service()->get_quic_connection(); - this->_local_qpack = - new QPACK(qc, HTTP3_DEFAULT_MAX_FIELD_SECTION_SIZE, HTTP3_DEFAULT_HEADER_TABLE_SIZE, HTTP3_DEFAULT_QPACK_BLOCKED_STREAMS); - this->_remote_qpack = - new QPACK(qc, HTTP3_DEFAULT_MAX_FIELD_SECTION_SIZE, HTTP3_DEFAULT_HEADER_TABLE_SIZE, HTTP3_DEFAULT_QPACK_BLOCKED_STREAMS); + QUICConnection *qc = vc->get_service()->get_quic_connection(); + uint32_t header_field_max_size = 32768; + HttpConfigParams *http_config = HttpConfig::acquire(); + + if (http_config) { + header_field_max_size = http_config->http_hdr_field_max_size; + HttpConfig::release(http_config); + } + + this->_local_qpack = new QPACK(qc, HTTP3_DEFAULT_MAX_FIELD_SECTION_SIZE, HTTP3_DEFAULT_HEADER_TABLE_SIZE, + HTTP3_DEFAULT_QPACK_BLOCKED_STREAMS, header_field_max_size); + this->_remote_qpack = new QPACK(qc, HTTP3_DEFAULT_MAX_FIELD_SECTION_SIZE, HTTP3_DEFAULT_HEADER_TABLE_SIZE, + HTTP3_DEFAULT_QPACK_BLOCKED_STREAMS, header_field_max_size); } Http3Session::~Http3Session() diff --git a/src/proxy/http3/QPACK.cc b/src/proxy/http3/QPACK.cc index 1469d69de12..6e915334dfc 100644 --- a/src/proxy/http3/QPACK.cc +++ b/src/proxy/http3/QPACK.cc @@ -140,10 +140,12 @@ const QPACK::Header QPACK::StaticTable::STATIC_HEADER_FIELDS[] = { {"x-frame-options", "sameorigin" } }; -QPACK::QPACK(QUICConnection *qc, uint32_t max_field_section_size, uint16_t max_table_size, uint16_t max_blocking_streams) +QPACK::QPACK(QUICConnection *qc, uint32_t max_field_section_size, uint16_t max_table_size, uint16_t max_blocking_streams, + uint32_t header_field_max_size) : QUICApplication(qc), _dynamic_table(max_table_size), _max_field_section_size(max_field_section_size), + _header_field_max_size(header_field_max_size), _max_table_size(max_table_size), _max_blocking_streams(max_blocking_streams) { @@ -764,7 +766,7 @@ QPACK::_decode_literal_header_field_with_name_ref(int16_t base_index, const uint // Read value char *value; uint64_t value_len; - if ((ret = xpack_decode_string(this->_arena, &value, value_len, buf + read_len, buf + buf_len, 7)) < 0) { + if ((ret = xpack_decode_string(this->_arena, &value, value_len, buf + read_len, buf + buf_len, _header_field_max_size, 7)) < 0) { return -1; } read_len += ret; @@ -796,14 +798,14 @@ QPACK::_decode_literal_header_field_without_name_ref(const uint8_t *buf, size_t int64_t ret; char *name; uint64_t name_len; - if ((ret = xpack_decode_string(this->_arena, &name, name_len, buf, buf + buf_len, 3)) < 0) { + if ((ret = xpack_decode_string(this->_arena, &name, name_len, buf, buf + buf_len, _header_field_max_size, 3)) < 0) { return -1; } read_len += ret; char *value; uint64_t value_len; - if ((ret = xpack_decode_string(this->_arena, &value, value_len, buf + read_len, buf + buf_len, 7)) < 0) { + if ((ret = xpack_decode_string(this->_arena, &value, value_len, buf + read_len, buf + buf_len, _header_field_max_size, 7)) < 0) { return -1; } read_len += ret; @@ -893,7 +895,7 @@ QPACK::_decode_literal_header_field_with_postbase_name_ref(int16_t base_index, c // Read value char *value; uint64_t value_len; - if ((ret = xpack_decode_string(this->_arena, &value, value_len, buf + read_len, buf + buf_len, 7)) < 0) { + if ((ret = xpack_decode_string(this->_arena, &value, value_len, buf + read_len, buf + buf_len, _header_field_max_size, 7)) < 0) { return -1; } read_len += ret; @@ -1518,7 +1520,8 @@ QPACK::_read_insert_with_name_ref(IOBufferReader &reader, bool &is_static, uint1 read_len += ret; // Value - if ((ret = xpack_decode_string(arena, value, tmp, input + read_len, input + input_len, 7)) < 0 && tmp > 0xFF) { + if ((ret = xpack_decode_string(arena, value, tmp, input + read_len, input + input_len, _header_field_max_size, 7)) < 0 && + tmp > 0xFF) { return -1; } value_len = tmp; @@ -1541,14 +1544,15 @@ QPACK::_read_insert_without_name_ref(IOBufferReader &reader, Arena &arena, char // Name uint64_t tmp; - if ((ret = xpack_decode_string(arena, name, tmp, input, input + input_len, 5)) < 0 && tmp > 0xFFFF) { + if ((ret = xpack_decode_string(arena, name, tmp, input, input + input_len, _header_field_max_size, 5)) < 0 && tmp > 0xFFFF) { return -1; } name_len = tmp; read_len += ret; // Value - if ((ret = xpack_decode_string(arena, value, tmp, input + read_len, input + input_len, 7)) < 0 && tmp > 0xFFFF) { + if ((ret = xpack_decode_string(arena, value, tmp, input + read_len, input + input_len, _header_field_max_size, 7)) < 0 && + tmp > 0xFFFF) { return -1; } value_len = tmp; diff --git a/src/proxy/http3/test/test_QPACK.cc b/src/proxy/http3/test/test_QPACK.cc index 438571d1149..df4d5495f1d 100644 --- a/src/proxy/http3/test/test_QPACK.cc +++ b/src/proxy/http3/test/test_QPACK.cc @@ -44,7 +44,8 @@ extern char pattern[256]; constexpr int ACK_MODE_IMMEDIATE = 1; // constexpr int ACK_MODE_NONE = 0; -constexpr int MAX_SEQUENCE = 1024; +constexpr int MAX_SEQUENCE = 1024; +constexpr uint64_t MAX_FIELD_SIZE = 32768; class TestQUICConnection : public MockQUICConnection { @@ -291,7 +292,7 @@ test_encode(const char *qif_file, const char *out_file, int dts, int mbs, int am int n_requests = load_qif_file(qif_file, requests); QUICApplicationDriver driver; - QPACK *qpack = new QPACK(driver.get_connection(), UINT32_MAX, dts, mbs); + QPACK *qpack = new QPACK(driver.get_connection(), UINT32_MAX, dts, mbs, MAX_FIELD_SIZE); TestQUICStream *encoder_stream = new TestQUICStream(0); TestQUICStream *decoder_stream = new TestQUICStream(10); qpack->on_stream_open(*encoder_stream); @@ -352,7 +353,7 @@ test_decode(const char *enc_file, const char *out_file, int dts, int mbs) TestQPACKEventHandler *event_handler = new TestQPACKEventHandler(); QUICApplicationDriver driver; - QPACK *qpack = new QPACK(driver.get_connection(), UINT32_MAX, dts, mbs); + QPACK *qpack = new QPACK(driver.get_connection(), UINT32_MAX, dts, mbs, MAX_FIELD_SIZE); TestQUICStream *encoder_stream = new TestQUICStream(0); qpack->on_stream_open(*encoder_stream); diff --git a/src/proxy/logging/LogFilter.cc b/src/proxy/logging/LogFilter.cc index 4bff98fd47c..e7de132fd07 100644 --- a/src/proxy/logging/LogFilter.cc +++ b/src/proxy/logging/LogFilter.cc @@ -27,6 +27,7 @@ ***************************************************************************/ +#include #include #include "swoc/BufferWriter.h" @@ -34,6 +35,7 @@ #include "swoc/bwf_ip.h" #include "tscore/ink_platform.h" +#include "tsutil/LocalBuffer.h" #include "tsutil/ts_errata.h" #include "proxy/logging/LogUtils.h" @@ -55,6 +57,8 @@ namespace DbgCtl dbg_ctl_log{"log"}; DbgCtl dbg_ctl_log_filter_compare{"log-filter-compare"}; +static constexpr size_t FIELD_VALUE_LOCAL_BUFFER_SIZE = 8192; + } // end anonymous namespace /*------------------------------------------------------------------------- @@ -273,30 +277,39 @@ findPatternFromParamName(const char *lookup_query_param, const char *pattern, bo static void updatePatternForFieldValue(char *field, const char *pattern_str, int /* field_pos ATS_UNUSED */, char *buf_dest) { - int buf_dest_len = strlen(buf_dest); - char buf_dest_to_field[buf_dest_len + 1]; - char *temp_text = buf_dest_to_field; + size_t const buf_dest_size = strlen(buf_dest); + if (buf_dest_size > static_cast(std::numeric_limits::max())) { + return; + } - memcpy(temp_text, buf_dest, (pattern_str - buf_dest)); - temp_text += (pattern_str - buf_dest); + int buf_dest_len = static_cast(buf_dest_size); + ts::LocalBuffer buf_dest_to_field(buf_dest_len + 1); + char *temp_text = buf_dest_to_field.data(); + + const int prefix_len = pattern_str - buf_dest; + memcpy(temp_text, buf_dest, prefix_len); + temp_text += prefix_len; const char *value_str = strchr(pattern_str, '='); if (value_str) { value_str++; - memcpy(temp_text, pattern_str, (value_str - pattern_str)); - temp_text += (value_str - pattern_str); + const int param_name_len = value_str - pattern_str; + memcpy(temp_text, pattern_str, param_name_len); + temp_text += param_name_len; const char *next_param_str = strchr(value_str, '&'); + const char *buf_dest_end = buf_dest + buf_dest_len; if (next_param_str) { - for (int i = 0; i < (next_param_str - value_str); i++) { + const int value_len = next_param_str - value_str; + for (int i = 0; i < value_len; i++) { temp_text[i] = 'X'; } - temp_text += (next_param_str - value_str); - memcpy(temp_text, next_param_str, ((buf_dest + buf_dest_len) - next_param_str)); + temp_text += value_len; + memcpy(temp_text, next_param_str, buf_dest_end - next_param_str); } else { - for (int i = 0; i < ((buf_dest + buf_dest_len) - value_str); i++) { + for (int i = 0; i < buf_dest_end - value_str; i++) { temp_text[i] = 'X'; } } @@ -304,8 +317,8 @@ updatePatternForFieldValue(char *field, const char *pattern_str, int /* field_po return; } - buf_dest_to_field[buf_dest_len] = '\0'; - strcpy(field, buf_dest_to_field); + buf_dest_to_field.data()[buf_dest_len] = '\0'; + strcpy(field, buf_dest_to_field.data()); } /*--------------------------------------------------------------------------- @@ -476,19 +489,17 @@ LogFilterString::operator==(LogFilterString &rhs) bool LogFilterString::toss_this_entry(LogAccess *lad) { - static const unsigned BUFSIZE = 8192; - if (m_num_values == 0 || m_field == nullptr || lad == nullptr) { return false; } - char small_buf[BUFSIZE]; + char small_buf[FIELD_VALUE_LOCAL_BUFFER_SIZE]; char *big_buf = nullptr; char *buf = small_buf; size_t marsh_len = m_field->marshal_len(lad); // includes null termination bool cond_satisfied = false; - if (marsh_len > BUFSIZE) { + if (marsh_len > FIELD_VALUE_LOCAL_BUFFER_SIZE) { big_buf = static_cast(ats_malloc(static_cast(marsh_len))); ink_assert(big_buf != nullptr); buf = big_buf; diff --git a/src/proxy/unit_tests/CMakeLists.txt b/src/proxy/unit_tests/CMakeLists.txt index b21fb327155..0396792cd2d 100644 --- a/src/proxy/unit_tests/CMakeLists.txt +++ b/src/proxy/unit_tests/CMakeLists.txt @@ -15,9 +15,7 @@ # ####################### -add_executable( - test_proxy main.cc test_ParentHashConfig.cc "${PROJECT_SOURCE_DIR}/src/iocore/net/libinknet_stub.cc" stub.cc -) +add_executable(test_proxy main.cc test_ControlBase.cc test_FetchSM.cc test_ParentHashConfig.cc stub.cc) target_link_libraries(test_proxy PRIVATE Catch2::Catch2WithMain ts::http ts::proxy ts::tscore ts::records ts::inkevent) diff --git a/src/proxy/unit_tests/stub.cc b/src/proxy/unit_tests/stub.cc index a1a95fe8ccb..35c526147be 100644 --- a/src/proxy/unit_tests/stub.cc +++ b/src/proxy/unit_tests/stub.cc @@ -21,6 +21,14 @@ limitations under the License. */ -#include "proxy/IPAllow.h" +#include "tscore/Version.h" -uint8_t IpAllow::subjects[IpAllow::Subject::MAX_SUBJECTS]; +// libinknet.a references appVersionInfo (declared in tscore/Version.h) but +// the tscore definition is file-static, so nobody exports a global instance +// for the linker. Provide one here. +// +// IpAllow::subjects used to live here too, but it duplicates the real +// definition in libproxy.a (IPAllow.cc). The test_proxy link line now pulls +// IPAllow.cc.o in transitively (via FetchSM in libproxy.a), which trips +// Apple ld on duplicate symbols. +AppVersionInfo appVersionInfo; diff --git a/src/proxy/unit_tests/test_ControlBase.cc b/src/proxy/unit_tests/test_ControlBase.cc new file mode 100644 index 00000000000..aee0c6d2099 --- /dev/null +++ b/src/proxy/unit_tests/test_ControlBase.cc @@ -0,0 +1,99 @@ +/** @file + + Unit tests for ControlBase. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include +#include + +#include "proxy/ControlBase.h" +#include "proxy/ControlMatcher.h" +#include "proxy/hdrs/HTTP.h" +#include "proxy/hdrs/MIME.h" +#include "proxy/hdrs/URL.h" +#include "tscore/MatcherUtils.h" + +using namespace std::literals; + +extern int cmd_disable_pfreelist; + +namespace +{ +void +initialize_headers_once() +{ + static bool initialized = false; + if (!initialized) { + cmd_disable_pfreelist = true; + url_init(); + mime_init(); + http_init(); + initialized = true; + } +} + +// Apply "method=" to a ControlBase and check it against a +// request whose method is . +bool +method_matches(std::string_view config_method, std::string_view request_method) +{ + initialize_headers_once(); + + HTTPHdr hdr; + hdr.create(HTTPType::REQUEST); + hdr.method_set(request_method); + + HttpRequestData req; + req.hdr = &hdr; + + std::string label{"method"}; + std::string value{config_method}; + matcher_line line{}; + line.num_el = 1; + line.line[0][0] = label.data(); + line.line[1][0] = value.data(); + + ControlBase cb; + REQUIRE(cb.ProcessModifiers(&line) == nullptr); + bool matched = cb.CheckModifiers(&req); + + hdr.destroy(); + return matched; +} +} // namespace + +TEST_CASE("ControlBase MethodMod check", "[ControlBase]") +{ + // Exact match (case-insensitive). + CHECK(method_matches("GET", "GET")); + CHECK(method_matches("GET", "get")); + + // Different method. + CHECK_FALSE(method_matches("GET", "POST")); + + // Make sure it's not a prefix match + CHECK_FALSE(method_matches("GET", "GETT")); + CHECK_FALSE(method_matches("GET", "GETS")); + CHECK_FALSE(method_matches("POST", "POSTING")); + CHECK_FALSE(method_matches("PUT", "PUTS")); +} diff --git a/src/proxy/unit_tests/test_FetchSM.cc b/src/proxy/unit_tests/test_FetchSM.cc new file mode 100644 index 00000000000..d19466ad6bb --- /dev/null +++ b/src/proxy/unit_tests/test_FetchSM.cc @@ -0,0 +1,93 @@ +/** @file + + Unit tests for FetchSM. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include +#include + +#include "iocore/eventsystem/IOBuffer.h" +#include "proxy/FetchSM.h" +#include "proxy/hdrs/HTTP.h" +#include "proxy/hdrs/MIME.h" +#include "proxy/hdrs/URL.h" + +extern int cmd_disable_pfreelist; +extern ClassAllocator FetchSMAllocator; + +namespace +{ +void +initialize_fetch_sm_once() +{ + static bool initialized = false; + if (!initialized) { + cmd_disable_pfreelist = true; + init_buffer_allocators(0); + url_init(); + mime_init(); + http_init(); + initialized = true; + } +} +} // namespace + +TEST_CASE("FetchSM copies response headers across IOBufferBlocks", "[FetchSM]") +{ + initialize_fetch_sm_once(); + + constexpr int64_t block_size = BUFFER_SIZE_FOR_INDEX(BUFFER_SIZE_INDEX_128); + + // Make our X-Fill explicitly block_size in length to ensure we exceed + // block_size to force the parsed response header to span multiple + // IOBufferBlocks. + std::string const response = + "HTTP/1.1 200 OK\r\nX-Fill: " + std::string(static_cast(block_size), 'a') + "\r\nContent-Length: 4\r\n\r\nbody"; + size_t const header_length = response.find("\r\n\r\n") + 4; + + MIOBuffer *response_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_128); + IOBufferReader *reader = response_buffer->alloc_reader(); + response_buffer->write(response.data(), response.size()); + + // Verify our precondition: that we did in fact create a situation with more + // than one IOBufferBlocks. + REQUIRE(header_length > static_cast(block_size)); + REQUIRE(reader->block_count() > 1); + REQUIRE(reader->block_read_avail() < static_cast(header_length)); + + // The heart of the test: verify that FetchSMAllocator can parse the multiple + // blocks correctly. + FetchSM *fetch_sm = FetchSMAllocator.alloc(); + fetch_sm->init_comm(); + fetch_sm->get_info_from_buffer(reader); + + int copied_length = 0; + char *copied = fetch_sm->resp_get(&copied_length); + + REQUIRE(copied != nullptr); + REQUIRE(copied_length == static_cast(response.size())); + CHECK(std::string_view(copied, copied_length) == response); + + fetch_sm->cleanUp(); + free_MIOBuffer(response_buffer); +} diff --git a/src/records/RecHttp.cc b/src/records/RecHttp.cc index 6feade5dd1d..7f1dbd68f95 100644 --- a/src/records/RecHttp.cc +++ b/src/records/RecHttp.cc @@ -32,6 +32,8 @@ #include #include #include +#include +#include using swoc::TextView; @@ -181,6 +183,9 @@ const char *const HttpProxyPort::OPT_OUTBOUND_IP_PREFIX = "ip-out"; const char *const HttpProxyPort::OPT_INBOUND_IP_PREFIX = "ip-in"; const char *const HttpProxyPort::OPT_HOST_RES_PREFIX = "ip-resolve"; const char *const HttpProxyPort::OPT_PROTO_PREFIX = "proto"; +const char *const HttpProxyPort::OPT_UDS_PERM_PREFIX = "uds-perm"; +const char *const HttpProxyPort::OPT_UDS_USER_PREFIX = "uds-user"; +const char *const HttpProxyPort::OPT_UDS_GROUP_PREFIX = "uds-group"; const char *const HttpProxyPort::OPT_IPV6 = "ipv6"; const char *const HttpProxyPort::OPT_IPV4 = "ipv4"; @@ -207,12 +212,35 @@ size_t const OPT_OUTBOUND_IP_PREFIX_LEN = strlen(HttpProxyPort::OPT_OUTBOUND_IP_ size_t const OPT_INBOUND_IP_PREFIX_LEN = strlen(HttpProxyPort::OPT_INBOUND_IP_PREFIX); size_t const OPT_HOST_RES_PREFIX_LEN = strlen(HttpProxyPort::OPT_HOST_RES_PREFIX); size_t const OPT_PROTO_PREFIX_LEN = strlen(HttpProxyPort::OPT_PROTO_PREFIX); +size_t const OPT_UDS_PERM_PREFIX_LEN = strlen(HttpProxyPort::OPT_UDS_PERM_PREFIX); +size_t const OPT_UDS_USER_PREFIX_LEN = strlen(HttpProxyPort::OPT_UDS_USER_PREFIX); +size_t const OPT_UDS_GROUP_PREFIX_LEN = strlen(HttpProxyPort::OPT_UDS_GROUP_PREFIX); constexpr std::string_view TS_ALPN_PROTO_ID_OPENSSL_HTTP_0_9("\x8http/0.9"); constexpr std::string_view TS_ALPN_PROTO_ID_OPENSSL_HTTP_1_0("\x8http/1.0"); constexpr std::string_view TS_ALPN_PROTO_ID_OPENSSL_HTTP_1_1("\x8http/1.1"); constexpr std::string_view TS_ALPN_PROTO_ID_OPENSSL_HTTP_2("\x2h2"); constexpr std::string_view TS_ALPN_PROTO_ID_OPENSSL_HTTP_3("\x2h3"); + +bool +parse_octal_mode(const char *s, mode_t &out) +{ + if (*s == '\0') { + return false; + } + unsigned long mode = 0; + for (; *s != '\0'; ++s) { + if (*s < '0' || *s > '7') { + return false; + } + mode = (mode << 3) | static_cast(*s - '0'); + if (mode > 0777) { + return false; + } + } + out = static_cast(mode); + return true; +} } // namespace namespace @@ -481,6 +509,33 @@ HttpProxyPort::processOptions(const char *opts) } else if (nullptr != (value = this->checkPrefix(item, OPT_PROTO_PREFIX, OPT_PROTO_PREFIX_LEN))) { this->processSessionProtocolPreference(value); sp_set_p = true; + } else if (nullptr != (value = this->checkPrefix(item, OPT_UDS_PERM_PREFIX, OPT_UDS_PERM_PREFIX_LEN))) { + if (!parse_octal_mode(value, m_unix_perm)) { + Warning("Invalid uds-perm value '%s' in proxy port descriptor '%s'", value, opts); + zret = false; + } + } else if (nullptr != (value = this->checkPrefix(item, OPT_UDS_USER_PREFIX, OPT_UDS_USER_PREFIX_LEN))) { + struct passwd *pw = nullptr; + if (*value != '\0') { + pw = getpwnam(value); + } + if (pw == nullptr) { + Warning("Invalid uds-user '%s' in proxy port descriptor '%s'", value, opts); + zret = false; + } else { + m_unix_uid = pw->pw_uid; + } + } else if (nullptr != (value = this->checkPrefix(item, OPT_UDS_GROUP_PREFIX, OPT_UDS_GROUP_PREFIX_LEN))) { + struct group *gr = nullptr; + if (*value != '\0') { + gr = getgrnam(value); + } + if (gr == nullptr) { + Warning("Invalid uds-group '%s' in proxy port descriptor '%s'", value, opts); + zret = false; + } else { + m_unix_gid = gr->gr_gid; + } } else { Warning("Invalid option '%s' in proxy port descriptor '%s'", item, opts); } @@ -501,6 +556,13 @@ HttpProxyPort::processOptions(const char *opts) m_family = m_inbound_ip.family(); // set according to address. } + // uds-perm / uds-user / uds-group are only meaningful for unix domain sockets. + if (m_family != AF_UNIX && + (m_unix_perm != 0666 || m_unix_uid != static_cast(-1) || m_unix_gid != static_cast(-1))) { + Warning("uds-perm, uds-user and uds-group are only valid for unix domain socket ports in '%s'", opts); + zret = false; + } + // If the port is outbound transparent only CLIENT host resolution is possible. if (m_outbound_transparent_p) { if (host_res_set_p && diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index 5c0d0e46209..e3bc28f4e97 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -568,7 +568,7 @@ static constexpr RecordElement RecordsConfig[] = {RECT_CONFIG, "proxy.config.http.request_line_max_size", RECD_INT, "65535", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.http.header_field_max_size", RECD_INT, "32768", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.http.header_field_max_size", RECD_INT, "32768", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-65535]", RECA_NULL} , // ############ // # security # @@ -979,7 +979,7 @@ static constexpr RecordElement RecordsConfig[] = // # in entries, may not be changed while running {RECT_CONFIG, "proxy.config.hostdb.max_count", RECD_INT, "-1", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , - {RECT_CONFIG, "proxy.config.hostdb.round_robin_max_count", RECD_INT, "16", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + {RECT_CONFIG, "proxy.config.hostdb.round_robin_max_count", RECD_INT, "16", RECU_RESTART_TS, RR_NULL, RECC_INT, "[1-1024]", RECA_NULL} , {RECT_CONFIG, "proxy.config.hostdb.max_size", RECD_INT, "10M", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , @@ -1412,7 +1412,7 @@ static constexpr RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.write_time_threshold", RECD_INT, "100", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , - {RECT_CONFIG, "proxy.config.http2.default_buffer_water_mark", RECD_INT, "-1", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-9]+$", RECA_NULL} + {RECT_CONFIG, "proxy.config.http2.default_buffer_water_mark", RECD_INT, "32768", RECU_DYNAMIC, RR_NULL, RECC_INT, "[1024-4294967295]", RECA_NULL} , //############ diff --git a/src/traffic_cache_tool/CacheScan.cc b/src/traffic_cache_tool/CacheScan.cc index 4a5b29740fc..c39ffbfefdf 100644 --- a/src/traffic_cache_tool/CacheScan.cc +++ b/src/traffic_cache_tool/CacheScan.cc @@ -160,8 +160,12 @@ CacheScan::unmarshal(HdrHeap *hh, int buf_length, int obj_type, HdrHeapObjImpl * return zret; } - int unmarshal_size = hh->unmarshal_size(); - if (unmarshal_size > buf_length) { + if (hh->m_size < static_cast(HDR_HEAP_HDR_SIZE) || // heap too small for header + hh->m_size != (uintptr_t)hh->m_ronly_heap[0].m_heap_start || // string heap offset inconsistent + hh->m_ronly_heap[0].m_heap_len < 0 || // invalid string heap length + buf_length < 0 || // invalid buf_length + static_cast(hh->m_size) + static_cast(hh->m_ronly_heap[0].m_heap_len) > + static_cast(buf_length)) { ink_assert(!"HdrHeap::unmarshal truncated header"); return zret; } diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index 241c3850ca3..76e9c62cd95 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -92,6 +92,7 @@ extern "C" int plock(int); #include "records/RecordsConfig.h" #include "iocore/eventsystem/RecProcess.h" #include "proxy/Transform.h" +#include "proxy/ReverseProxy.h" #include "iocore/eventsystem/ConfigProcessor.h" #include "mgmt/config/ConfigContextDiags.h" #include "mgmt/config/ConfigRegistry.h" @@ -296,6 +297,8 @@ struct AutoStopCont : public Continuation { // Push buffered log entries into the preproc queue before shutdown. Log::flush_all_objects(); + shutdown_url_rewrite(); + TSSystemState::shut_down_event_system(); // Wake preproc threads to drain remaining log buffers before exit. diff --git a/src/tscore/CMakeLists.txt b/src/tscore/CMakeLists.txt index 3d70052b199..d3e70f7e5a5 100644 --- a/src/tscore/CMakeLists.txt +++ b/src/tscore/CMakeLists.txt @@ -150,6 +150,7 @@ if(BUILD_TESTING) unit_tests/test_List.cc unit_tests/test_MMH.cc unit_tests/test_ParseRules.cc + unit_tests/test_PendingAction.cc unit_tests/test_PluginUserArgs.cc unit_tests/test_PriorityQueue.cc unit_tests/test_Ptr.cc diff --git a/src/tscore/X509HostnameValidator.cc b/src/tscore/X509HostnameValidator.cc index 888e66ad32d..0efbcac99df 100644 --- a/src/tscore/X509HostnameValidator.cc +++ b/src/tscore/X509HostnameValidator.cc @@ -27,6 +27,7 @@ #include #include +#include "tscore/X509HostnameValidator.h" #include "tscore/ink_memory.h" using equal_fn = bool (*)(const unsigned char *, size_t, const unsigned char *, size_t); @@ -216,7 +217,7 @@ do_check_string(ASN1_STRING *a, int cmp_type, equal_fn equal, const unsigned cha } bool -validate_hostname(X509 *x, const unsigned char *hostname, bool is_ip, char **peername) +validate_hostname(X509 *x, std::string_view hostname, bool is_ip, char **peername) { GENERAL_NAMES *gens = nullptr; X509_NAME *name = nullptr; @@ -224,8 +225,13 @@ validate_hostname(X509 *x, const unsigned char *hostname, bool is_ip, char **pee int alt_type; bool retval = false; ; - equal_fn equal; - size_t hostname_len = strlen((char *)hostname); + equal_fn equal; + auto const *hostname_data = reinterpret_cast(hostname.data()); + size_t hostname_len = hostname.length(); + + if (hostname.empty()) { + return false; + } if (!is_ip) { alt_type = V_ASN1_IA5STRING; @@ -252,7 +258,7 @@ validate_hostname(X509 *x, const unsigned char *hostname, bool is_ip, char **pee continue; } - if ((retval = do_check_string(cstr, alt_type, equal, hostname, hostname_len, peername)) == true) { + if ((retval = do_check_string(cstr, alt_type, equal, hostname_data, hostname_len, peername)) == true) { // We got a match break; } @@ -277,7 +283,7 @@ validate_hostname(X509 *x, const unsigned char *hostname, bool is_ip, char **pee if (astrlen < 0) { return -1; } - retval = equal(astr, astrlen, hostname, hostname_len); + retval = equal(astr, astrlen, hostname_data, hostname_len); if (retval && peername) { *peername = ats_strndup((char *)astr, astrlen); } diff --git a/src/tscore/unit_tests/test_PendingAction.cc b/src/tscore/unit_tests/test_PendingAction.cc new file mode 100644 index 00000000000..99d36376f23 --- /dev/null +++ b/src/tscore/unit_tests/test_PendingAction.cc @@ -0,0 +1,91 @@ +/** @file + + Unit tests for PendingAction. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + See the NOTICE file distributed with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance with the License. You may obtain a + copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + */ + +#include "iocore/eventsystem/Action.h" +#include "tscore/PendingAction.h" + +#include + +namespace +{ +// Action subclass that records cancel() calls so the tests can assert +// whether the PendingAction operation cancelled the action or not. +class TestAction : public Action +{ +public: + void + cancel(Continuation *c = nullptr) override + { + Action::cancel(c); + cancel_count++; + } + int cancel_count = 0; +}; +} // namespace + +TEST_CASE("PendingAction::clear_if_action_is clears only the matching action", "[PendingAction]") +{ + // Declare the Action before PendingAction so it outlives pa during stack + // unwinding on a REQUIRE failure - PendingAction's destructor calls + // cancel() on whatever it still holds. + TestAction actionA; + PendingAction pa; + + pa = &actionA; + REQUIRE(pa.get() == &actionA); + + REQUIRE(pa.clear_if_action_is(&actionA)); + REQUIRE(pa.empty()); + REQUIRE(actionA.cancel_count == 0); // clear_if_action_is must not cancel +} + +TEST_CASE("PendingAction::clear_if_action_is does not touch a non-matching action", "[PendingAction]") +{ + // Regression for the CAS-race where compare_exchange_strong overwrote its + // expected argument on failure and the surrounding loop then cleared the + // *new* pending_action as if it were the one the caller asked about. + // Even in single-threaded use, calling clear_if_action_is with the wrong + // pointer must not touch what is currently pending. + TestAction actionA; + TestAction actionB; + PendingAction pa; + + pa = &actionA; + REQUIRE(pa.get() == &actionA); + + REQUIRE_FALSE(pa.clear_if_action_is(&actionB)); + REQUIRE(pa.get() == &actionA); // the unrelated action is still pending + REQUIRE(actionB.cancel_count == 0); + + // Clear the pending action so PendingAction's destructor does not call + // cancel() on it during teardown. + REQUIRE(pa.clear_if_action_is(&actionA)); +} + +TEST_CASE("PendingAction::clear_if_action_is on a null action is a no-op", "[PendingAction]") +{ + TestAction actionA; + PendingAction pa; + + pa = &actionA; + REQUIRE_FALSE(pa.clear_if_action_is(nullptr)); + REQUIRE(pa.get() == &actionA); + REQUIRE(pa.clear_if_action_is(&actionA)); +} diff --git a/src/tscore/unit_tests/test_X509HostnameValidator.cc b/src/tscore/unit_tests/test_X509HostnameValidator.cc index d937a194104..0f34f5d4d3e 100644 --- a/src/tscore/unit_tests/test_X509HostnameValidator.cc +++ b/src/tscore/unit_tests/test_X509HostnameValidator.cc @@ -112,9 +112,9 @@ TEST_CASE("CN_match", "[libts][X509HostnameValidator]") ts::PostScript x_defer([&]() -> void { X509_free(x); }); REQUIRE(x != nullptr); - REQUIRE(validate_hostname(x, (unsigned char *)test_certificate_cn_name, false, &matching) == true); + REQUIRE(validate_hostname(x, test_certificate_cn_name, false, &matching) == true); REQUIRE(strcmp(test_certificate_cn_name, matching) == 0); - REQUIRE(validate_hostname(x, (unsigned char *)test_certificate_cn_name + 1, false, nullptr) == false); + REQUIRE(validate_hostname(x, std::string_view{test_certificate_cn_name + 1}, false, nullptr) == false); ats_free(matching); } @@ -124,11 +124,11 @@ TEST_CASE("bad_wildcard_SANs", "[libts][X509HostnameValidator]") ts::PostScript x_defer([&]() -> void { X509_free(x); }); REQUIRE(x != nullptr); - REQUIRE(validate_hostname(x, (unsigned char *)"something.or.other", false, nullptr) == false); - REQUIRE(validate_hostname(x, (unsigned char *)"a.b.c", false, nullptr) == false); - REQUIRE(validate_hostname(x, (unsigned char *)"0.0.0.0", true, nullptr) == false); - REQUIRE(validate_hostname(x, (unsigned char *)"......", true, nullptr) == false); - REQUIRE(validate_hostname(x, (unsigned char *)"a.b", true, nullptr) == false); + REQUIRE(validate_hostname(x, "something.or.other", false, nullptr) == false); + REQUIRE(validate_hostname(x, "a.b.c", false, nullptr) == false); + REQUIRE(validate_hostname(x, "0.0.0.0", true, nullptr) == false); + REQUIRE(validate_hostname(x, "......", true, nullptr) == false); + REQUIRE(validate_hostname(x, "a.b", true, nullptr) == false); } TEST_CASE("wildcard_SAN_and_CN", "[libts][X509HostnameValidator]") @@ -138,14 +138,14 @@ TEST_CASE("wildcard_SAN_and_CN", "[libts][X509HostnameValidator]") ts::PostScript x_defer([&]() -> void { X509_free(x); }); REQUIRE(x != nullptr); - REQUIRE(validate_hostname(x, (unsigned char *)test_certificate_cn_name, false, &matching) == true); + REQUIRE(validate_hostname(x, test_certificate_cn_name, false, &matching) == true); REQUIRE(strcmp(test_certificate_cn_name, matching) == 0); ats_free(matching); - REQUIRE(validate_hostname(x, (unsigned char *)"a.trafficserver.org", false, &matching) == true); + REQUIRE(validate_hostname(x, "a.trafficserver.org", false, &matching) == true); REQUIRE(strcmp("*.trafficserver.org", matching) == 0); - REQUIRE(validate_hostname(x, (unsigned char *)"a.*.trafficserver.org", false, nullptr) == false); + REQUIRE(validate_hostname(x, "a.*.trafficserver.org", false, nullptr) == false); ats_free(matching); } @@ -156,12 +156,12 @@ TEST_CASE("IDNA_hostnames", "[libts][X509HostnameValidator]") ts::PostScript x_defer([&]() -> void { X509_free(x); }); REQUIRE(x != nullptr); - REQUIRE(validate_hostname(x, (unsigned char *)"xn--foobar.trafficserver.org", false, &matching) == true); + REQUIRE(validate_hostname(x, "xn--foobar.trafficserver.org", false, &matching) == true); REQUIRE(strcmp("*.trafficserver.org", matching) == 0); ats_free(matching); // IDNA means wildcard must match full label - REQUIRE(validate_hostname(x, (unsigned char *)"xn--foobar.trafficserver.net", false, &matching) == false); + REQUIRE(validate_hostname(x, "xn--foobar.trafficserver.net", false, &matching) == false); } TEST_CASE("middle_label_match", "[libts][X509HostnameValidator]") @@ -171,15 +171,15 @@ TEST_CASE("middle_label_match", "[libts][X509HostnameValidator]") ts::PostScript x_defer([&]() -> void { X509_free(x); }); REQUIRE(x != nullptr); - REQUIRE(validate_hostname(x, (unsigned char *)"foosomething.trafficserver.com", false, &matching) == true); + REQUIRE(validate_hostname(x, "foosomething.trafficserver.com", false, &matching) == true); REQUIRE(strcmp("foo*.trafficserver.com", matching) == 0); ats_free(matching); - REQUIRE(validate_hostname(x, (unsigned char *)"somethingbar.trafficserver.net", false, &matching) == true); + REQUIRE(validate_hostname(x, "somethingbar.trafficserver.net", false, &matching) == true); REQUIRE(strcmp("*bar.trafficserver.net", matching) == 0); ats_free(matching); - REQUIRE(validate_hostname(x, (unsigned char *)"a.bar.trafficserver.net", false, nullptr) == false); - REQUIRE(validate_hostname(x, (unsigned char *)"foo.bar.trafficserver.net", false, nullptr) == false); + REQUIRE(validate_hostname(x, "a.bar.trafficserver.net", false, nullptr) == false); + REQUIRE(validate_hostname(x, "foo.bar.trafficserver.net", false, nullptr) == false); } int diff --git a/src/tscpp/api/InterceptPlugin.cc b/src/tscpp/api/InterceptPlugin.cc index 7b52acfc4af..c04397b49e2 100644 --- a/src/tscpp/api/InterceptPlugin.cc +++ b/src/tscpp/api/InterceptPlugin.cc @@ -330,10 +330,21 @@ InterceptPlugin::handleEvent(int abstract_event, void *edata) namespace { +/** RAII try-lock helper used by the @c InterceptPlugin continuation. + * + * The guard takes shared ownership of the mutex for its entire scope. The + * locked region below may destroy the @c InterceptPlugin::State that owns + * the only other @c std::shared_ptr to the mutex. + */ class TryLockGuard { public: - TryLockGuard(Mutex &m) : _m(m), _isLocked(m.try_lock()) {} + TryLockGuard(std::shared_ptr m) : _m(std::move(m)), _isLocked(_m && _m->try_lock()) {} + + TryLockGuard(const TryLockGuard &) = delete; + TryLockGuard &operator=(const TryLockGuard &) = delete; + TryLockGuard(TryLockGuard &&) = delete; + TryLockGuard &operator=(TryLockGuard &&) = delete; bool isLocked() const @@ -344,13 +355,13 @@ class TryLockGuard ~TryLockGuard() { if (_isLocked) { - _m.unlock(); + _m->unlock(); } } private: - std::recursive_mutex &_m; - const bool _isLocked; + std::shared_ptr _m; ///< Shared ownership of the protected mutex. + const bool _isLocked; ///< Whether @c _m->try_lock() succeeded. }; int @@ -365,7 +376,7 @@ handleEvents(TSCont cont, TSEvent pristine_event, void *pristine_edata) return 0; } - TryLockGuard scopedTryLock(*(state->plugin_mutex_)); + TryLockGuard scopedTryLock(state->plugin_mutex_); if (!scopedTryLock.isLocked()) { LOG_ERROR("Couldn't get plugin lock. Will retry"); if (event != TS_EVENT_TIMEOUT) { // save only "non-retry" info diff --git a/src/tscpp/api/TransformationPlugin.cc b/src/tscpp/api/TransformationPlugin.cc index 35a2afd6d9e..2b4f41b9358 100644 --- a/src/tscpp/api/TransformationPlugin.cc +++ b/src/tscpp/api/TransformationPlugin.cc @@ -299,7 +299,7 @@ TransformationPlugin::pause() state_->vconn_, state_->txn_); } else { state_->paused_ = true; - if (!static_cast(static_cast(state_.get()))) { + if (!static_cast(*static_cast(state_.get()))) { *static_cast(state_.get()) = ResumeAfterPauseCont(TSContMutexGet(reinterpret_cast(state_->txn_))); } diff --git a/src/tsutil/Regex.cc b/src/tsutil/Regex.cc index 0e76c50ce18..34bfc5447d6 100644 --- a/src/tsutil/Regex.cc +++ b/src/tsutil/Regex.cc @@ -40,6 +40,29 @@ static_assert(RE_NOTEMPTY == PCRE2_NOTEMPTY, "Update RE_NOTEMPTY for current PCR static_assert(RE_ERROR_NOMATCH == PCRE2_ERROR_NOMATCH, "Update RE_ERROR_NOMATCH for current PCRE2 version."); static_assert(RE_ERROR_NULL == PCRE2_ERROR_NULL, "Update RE_ERROR_NULL for current PCRE2 version."); +// PCRE2 10.30 added PCRE2_ENDANCHORED. Older PCRE2 (e.g., CentOS 7 ships 10.23) lacks it. +// On modern PCRE2 we pass RE_ENDANCHORED through natively (zero overhead); on old PCRE2 the +// bit is not a valid pcre2_compile option and would be rejected with PCRE2_ERROR_BADOPTION, +// so we transparently rewrite the pattern to "(?:pattern)\z" and strip the bit. See +// Regex::compile() for the rewrite. This preserves alternation-with-backtracking semantics +// (unlike a post-match length check, which stops at the first successful alternative). +#ifdef PCRE2_ENDANCHORED +static constexpr bool ATS_PCRE2_HAS_ENDANCHORED = true; +static_assert(RE_ENDANCHORED == PCRE2_ENDANCHORED, "Update RE_ENDANCHORED for current PCRE2 version."); +static_assert((RE_FULL_MATCH & PCRE2_ENDANCHORED) == 0, "RE_FULL_MATCH bit collides with PCRE2_ENDANCHORED"); +#else +static constexpr bool ATS_PCRE2_HAS_ENDANCHORED = false; +#endif + +// RE_FULL_MATCH is an ATS-only flag; it must not collide with any PCRE2 compile or match flag. +// We strip it before forwarding to pcre2_match, but a collision would cause spurious behavior +// if someone OR'd it into a flag word that's also passed elsewhere. +static_assert((RE_FULL_MATCH & PCRE2_ANCHORED) == 0, "RE_FULL_MATCH bit collides with PCRE2_ANCHORED"); +static_assert((RE_FULL_MATCH & PCRE2_NO_UTF_CHECK) == 0, "RE_FULL_MATCH bit collides with PCRE2_NO_UTF_CHECK"); +static_assert((RE_FULL_MATCH & PCRE2_CASELESS) == 0, "RE_FULL_MATCH bit collides with PCRE2_CASELESS"); +static_assert((RE_FULL_MATCH & PCRE2_MULTILINE) == 0, "RE_FULL_MATCH bit collides with PCRE2_MULTILINE"); +static_assert((RE_FULL_MATCH & PCRE2_NOTEMPTY) == 0, "RE_FULL_MATCH bit collides with PCRE2_NOTEMPTY"); + //---------------------------------------------------------------------------- namespace { @@ -373,12 +396,35 @@ Regex::compile(std::string_view pattern, std::string &error, int &erroroffset, u return false; } + // On PCRE2 < 10.30 the ENDANCHORED bit is not a valid pcre2_compile option. Rewrite + // the pattern to "(?:pattern)\z" and strip the bit so pcre2 enforces end-of-subject + // natively (including proper alternation backtracking). Zero overhead on modern PCRE2 + // where the bit is passed through unchanged. + std::string rewritten_pattern; + std::string_view effective_pattern = pattern; + bool pattern_wrapped = false; + if constexpr (!ATS_PCRE2_HAS_ENDANCHORED) { + if ((flags & RE_ENDANCHORED) != 0) { + rewritten_pattern.reserve(pattern.size() + 6); + rewritten_pattern.append("(?:").append(pattern).append(")\\z"); + effective_pattern = rewritten_pattern; + flags &= ~static_cast(RE_ENDANCHORED); + pattern_wrapped = true; + } + } + PCRE2_SIZE error_offset; int error_code; - auto code = pcre2_compile(reinterpret_cast(pattern.data()), pattern.size(), flags, &error_code, &error_offset, - regex_context->get_compile_context()); + auto code = pcre2_compile(reinterpret_cast(effective_pattern.data()), effective_pattern.size(), flags, &error_code, + &error_offset, regex_context->get_compile_context()); if (!code) { - erroroffset = error_offset; + // Compensate for the "(?:" prefix so callers see offsets into their pattern, not ours. + // If the offset is inside the prefix itself, clamp to 0. + if (pattern_wrapped && error_offset >= 3) { + erroroffset = static_cast(error_offset - 3); + } else { + erroroffset = static_cast(error_offset); + } // get pcre2 error message PCRE2_UCHAR buffer[256]; @@ -441,8 +487,11 @@ Regex::exec(std::string_view subject, RegexMatches &matches, uint32_t flags, Reg match_context = RegexMatchContext::_MatchContext::get(matchContext->_match_context); } - int const rc = pcre2_match(code, reinterpret_cast(subject.data()), subject.size(), 0, flags, - RegexMatches::_MatchData::get(matches._match_data), match_context); + bool const full_match = (flags & RE_FULL_MATCH) != 0; + uint32_t const pcre2_flags = flags & ~RE_FULL_MATCH; + + int rc = pcre2_match(code, reinterpret_cast(subject.data()), subject.size(), 0, pcre2_flags, + RegexMatches::_MatchData::get(matches._match_data), match_context); matches._size = rc; @@ -454,6 +503,12 @@ Regex::exec(std::string_view subject, RegexMatches &matches, uint32_t flags, Reg if (rc == 0) { matches._size = pcre2_get_ovector_count(RegexMatches::_MatchData::get(matches._match_data)); } + + // Enforce full-subject consumption when requested. + if (full_match && matches[0].size() != subject.size()) { + matches._size = PCRE2_ERROR_NOMATCH; + rc = PCRE2_ERROR_NOMATCH; + } } return rc; @@ -514,6 +569,11 @@ DFA::build(const std::string_view pattern, unsigned flags) Regex rxp; std::string string{pattern}; + if (flags & RE_FULL_MATCH) { + _full_match = true; + flags &= ~RE_FULL_MATCH; + } + if (!(flags & RE_UNANCHORED)) { flags |= RE_ANCHORED; } @@ -560,8 +620,10 @@ DFA::compile(const char *const *patterns, int npatterns, unsigned flags) int32_t DFA::match(std::string_view str) const { + uint32_t const exec_flags = _full_match ? static_cast(RE_FULL_MATCH) : 0u; + for (auto spot = _patterns.begin(), limit = _patterns.end(); spot != limit; ++spot) { - if (spot->_re.exec(str)) { + if (spot->_re.exec(str, exec_flags)) { return spot - _patterns.begin(); } } diff --git a/src/tsutil/unit_tests/test_Regex.cc b/src/tsutil/unit_tests/test_Regex.cc index 8312146060f..f5cddd47a00 100644 --- a/src/tsutil/unit_tests/test_Regex.cc +++ b/src/tsutil/unit_tests/test_Regex.cc @@ -883,3 +883,115 @@ TEST_CASE("RegexMatchContext", "[libts][Regex][RegexMatchContext]") REQUIRE(r.compile(item.regex) == item.valid); REQUIRE(r.exec(item.str, matches, 0, &match_context) == item.rcode); } + +TEST_CASE("Regex RE_FULL_MATCH rejects trailing content", "[libts][Regex][full_match]") +{ + Regex re; + REQUIRE(re.compile(R"(example\.com)", REFlags::RE_ANCHORED)); + + RegexMatches matches; + + SECTION("exact input matches with RE_FULL_MATCH") + { + REQUIRE(re.exec("example.com", matches, REFlags::RE_FULL_MATCH) > 0); + } + + SECTION("trailing content rejected with RE_FULL_MATCH") + { + int rc = re.exec("example.com.evil", matches, REFlags::RE_FULL_MATCH); + REQUIRE(rc == RE_ERROR_NOMATCH); + } + + SECTION("trailing content still matches without RE_FULL_MATCH") + { + REQUIRE(re.exec("example.com.evil", matches) > 0); + } + + SECTION("bool exec overload honors RE_FULL_MATCH") + { + REQUIRE(re.exec("example.com", REFlags::RE_FULL_MATCH)); + REQUIRE_FALSE(re.exec("example.com.evil", REFlags::RE_FULL_MATCH)); + } +} + +TEST_CASE("Regex RE_FULL_MATCH preserves capture groups and combines with other flags", "[libts][Regex][full_match]") +{ + RegexMatches matches; + + SECTION("captures available on full match") + { + Regex re; + REQUIRE(re.compile(R"(^([a-z]+)\.([a-z]+)$)")); + REQUIRE(re.exec("foo.bar", matches, REFlags::RE_FULL_MATCH) == 3); + REQUIRE(matches[1] == "foo"); + REQUIRE(matches[2] == "bar"); + } + + SECTION("RE_FULL_MATCH composes with RE_CASE_INSENSITIVE") + { + Regex re; + REQUIRE(re.compile(R"(example\.com)", REFlags::RE_CASE_INSENSITIVE | REFlags::RE_ANCHORED)); + REQUIRE(re.exec("EXAMPLE.COM", matches, REFlags::RE_FULL_MATCH) > 0); + REQUIRE(re.exec("EXAMPLE.COM.evil", matches, REFlags::RE_FULL_MATCH) == RE_ERROR_NOMATCH); + } + + SECTION("matches._size reflects RE_ERROR_NOMATCH on length-rejected match") + { + Regex re; + REQUIRE(re.compile(R"(foo)")); + REQUIRE(re.exec("foobar", matches, REFlags::RE_FULL_MATCH) == RE_ERROR_NOMATCH); + REQUIRE(matches.size() == RE_ERROR_NOMATCH); + } +} + +TEST_CASE("DFA RE_FULL_MATCH applied at compile time", "[libts][DFA][full_match]") +{ + SECTION("trailing content rejected when DFA compiled with RE_FULL_MATCH") + { + DFA dfa; + std::string_view pattern = R"(example\.com)"; + REQUIRE(dfa.compile(pattern, REFlags::RE_FULL_MATCH) == 1); + + REQUIRE(dfa.match("example.com") == 0); + REQUIRE(dfa.match("example.com.evil") == -1); + } + + SECTION("multi-pattern DFA: RE_FULL_MATCH applies to all patterns") + { + std::vector patterns = {R"(foo)", R"(bar)"}; + DFA dfa; + REQUIRE(dfa.compile(patterns.data(), patterns.size(), REFlags::RE_FULL_MATCH) == 2); + + REQUIRE(dfa.match("foo") == 0); + REQUIRE(dfa.match("bar") == 1); + REQUIRE(dfa.match("foobar") == -1); + REQUIRE(dfa.match("barbaz") == -1); + } + + SECTION("DFA without RE_FULL_MATCH still permits trailing content (existing behavior)") + { + DFA dfa; + REQUIRE(dfa.compile(R"(foo)") == 1); + REQUIRE(dfa.match("foobar") == 0); + } +} + +// Regression: RE_ANCHORED | RE_ENDANCHORED on an alternation where a shorter +// alternative is a prefix of a longer one must backtrack to the longer alt +// when only the longer alt spans the full subject. On modern PCRE2 this is +// handled by the native PCRE2_ENDANCHORED flag; on PCRE2 < 10.30 the pattern +// is rewritten to "(?:pattern)\z" so pcre2 does the same backtracking. A +// naive post-match length check on the first successful pcre2_match would +// stop at the shorter alt and incorrectly report no match. +TEST_CASE("Regex end-anchor with alternation", "[libts][Regex]") +{ + Regex r; + REQUIRE(r.compile(R"(cdn\.example\.com|cdn\.example\.com\.edge)", RE_ANCHORED | RE_ENDANCHORED) == true); + + RegexMatches matches; + CHECK(r.exec("cdn.example.com", matches) > 0); // shorter alt spans fully + CHECK(r.exec("cdn.example.com.edge", matches) > 0); // longer alt spans fully -- requires backtracking + CHECK(r.exec("cdn.example.com.evil", matches) == RE_ERROR_NOMATCH); + CHECK(r.exec("cdn.example.com.evil.com", matches) == RE_ERROR_NOMATCH); + CHECK(r.exec("prefix.cdn.example.com", matches) == RE_ERROR_NOMATCH); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d61f530ab37..aa09cd1cf83 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -37,6 +37,7 @@ endfunction() add_subdirectory(tools/plugins) add_subdirectory(gold_tests/chunked_encoding) +add_subdirectory(gold_tests/at_headers/plugins) add_subdirectory(gold_tests/continuations/plugins) add_subdirectory(gold_tests/jsonrpc/plugins) add_subdirectory(gold_tests/pluginTest/crash_test) diff --git a/tests/fuzzing/fuzz_hpack.cc b/tests/fuzzing/fuzz_hpack.cc index 1a068ec5a2f..6903c35663e 100644 --- a/tests/fuzzing/fuzz_hpack.cc +++ b/tests/fuzzing/fuzz_hpack.cc @@ -30,6 +30,7 @@ #define INITIAL_TABLE_SIZE 4096 #define MAX_REQUEST_HEADER_SIZE 131072 #define MAX_TABLE_SIZE 4096 +#define MAX_FIELD_SIZE 32768 extern int cmd_disable_pfreelist; @@ -46,7 +47,8 @@ LLVMFuzzerTestOneInput(const uint8_t *input_data, size_t size_data) std::unique_ptr headers(new HTTPHdr); headers->create(HTTPType::REQUEST); - hpack_decode_header_block(indexing_table, headers.get(), input_data, size_data, MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE); + hpack_decode_header_block(indexing_table, headers.get(), input_data, size_data, MAX_REQUEST_HEADER_SIZE, MAX_TABLE_SIZE, + MAX_FIELD_SIZE); headers->destroy(); diff --git a/tests/gold_tests/at_headers/at_headers.test.py b/tests/gold_tests/at_headers/at_headers.test.py new file mode 100644 index 00000000000..e92c6038042 --- /dev/null +++ b/tests/gold_tests/at_headers/at_headers.test.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify that @ headers are stripped before global plugin and remap hooks run. +''' + +Test.ATSReplayTest(replay_file="replay/at_headers.replay.yaml") diff --git a/tests/gold_tests/at_headers/plugins/CMakeLists.txt b/tests/gold_tests/at_headers/plugins/CMakeLists.txt new file mode 100644 index 00000000000..67501089475 --- /dev/null +++ b/tests/gold_tests/at_headers/plugins/CMakeLists.txt @@ -0,0 +1,18 @@ +####################### +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor license +# agreements. See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# +####################### + +add_autest_plugin(at_header_probe at_header_probe.cc) diff --git a/tests/gold_tests/at_headers/plugins/at_header_probe.cc b/tests/gold_tests/at_headers/plugins/at_header_probe.cc new file mode 100644 index 00000000000..7cf2c74d5b4 --- /dev/null +++ b/tests/gold_tests/at_headers/plugins/at_header_probe.cc @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include + +namespace +{ +constexpr char PLUGIN_NAME[] = "at_header_probe"; +constexpr char REQUEST_ADDED_VALUE[] = "request-added"; +constexpr char RESPONSE_ADDED_VALUE[] = "response-added"; +constexpr char REQUEST_ERROR_PREFIX[] = "saw unexpected request header"; +constexpr char RESPONSE_ERROR_PREFIX[] = "saw unexpected response header"; +constexpr char REMAP_REQUEST_ERROR_PREFIX[] = "saw unexpected remap request header"; + +std::string request_probe_header; +std::string response_probe_header; +std::string request_added_header; +std::string response_added_header; + +struct RemapConfig { + std::string request_probe_header; +}; + +bool +set_header(TSMBuffer bufp, TSMLoc hdr_loc, std::string const &name, char const *value) +{ + int const name_len = static_cast(name.size()); + int const value_len = static_cast(std::strlen(value)); + TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, name.data(), name_len); + bool created = false; + bool ok = false; + + if (field_loc == TS_NULL_MLOC) { + if (TSMimeHdrFieldCreateNamed(bufp, hdr_loc, name.data(), name_len, &field_loc) != TS_SUCCESS) { + return false; + } + created = true; + } + + if (TSMimeHdrFieldValuesClear(bufp, hdr_loc, field_loc) == TS_SUCCESS && + TSMimeHdrFieldValueStringSet(bufp, hdr_loc, field_loc, -1, value, value_len) == TS_SUCCESS) { + if (!created || TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc) == TS_SUCCESS) { + ok = true; + } + } + + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + return ok; +} + +bool +has_header(TSMBuffer bufp, TSMLoc hdr_loc, std::string const &name) +{ + TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, name.data(), static_cast(name.size())); + if (field_loc == TS_NULL_MLOC) { + return false; + } + + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + return true; +} + +void +log_unexpected(char const *prefix, std::string const &name) +{ + TSError("[%s] %s %s", PLUGIN_NAME, prefix, name.c_str()); +} + +bool +parse_arguments(int argc, char const *argv[]) +{ + if (argc != 5) { + TSError("[%s] Expected request-probe, response-probe, request-added, and response-added header names", PLUGIN_NAME); + return false; + } + + request_probe_header = argv[1]; + response_probe_header = argv[2]; + request_added_header = argv[3]; + response_added_header = argv[4]; + + return true; +} + +int +handle_event(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edata) +{ + TSHttpTxn txnp = static_cast(edata); + + if (event == TS_EVENT_HTTP_READ_REQUEST_HDR) { + TSMBuffer req_bufp = nullptr; + TSMLoc req_hdr = TS_NULL_MLOC; + + if (TSHttpTxnClientReqGet(txnp, &req_bufp, &req_hdr) == TS_SUCCESS) { + if (has_header(req_bufp, req_hdr, request_probe_header)) { + log_unexpected(REQUEST_ERROR_PREFIX, request_probe_header); + } + TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_hdr); + } + } else if (event == TS_EVENT_HTTP_SEND_REQUEST_HDR) { + TSMBuffer req_bufp = nullptr; + TSMLoc req_hdr = TS_NULL_MLOC; + + if (TSHttpTxnServerReqGet(txnp, &req_bufp, &req_hdr) == TS_SUCCESS) { + set_header(req_bufp, req_hdr, request_added_header, REQUEST_ADDED_VALUE); + TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_hdr); + } + } else if (event == TS_EVENT_HTTP_READ_RESPONSE_HDR) { + TSMBuffer resp_bufp = nullptr; + TSMLoc resp_hdr = TS_NULL_MLOC; + + if (TSHttpTxnServerRespGet(txnp, &resp_bufp, &resp_hdr) == TS_SUCCESS) { + if (has_header(resp_bufp, resp_hdr, response_probe_header)) { + log_unexpected(RESPONSE_ERROR_PREFIX, response_probe_header); + } + TSHandleMLocRelease(resp_bufp, TS_NULL_MLOC, resp_hdr); + } + } else if (event == TS_EVENT_HTTP_SEND_RESPONSE_HDR) { + TSMBuffer resp_bufp = nullptr; + TSMLoc resp_hdr = TS_NULL_MLOC; + + if (TSHttpTxnClientRespGet(txnp, &resp_bufp, &resp_hdr) == TS_SUCCESS) { + set_header(resp_bufp, resp_hdr, response_added_header, RESPONSE_ADDED_VALUE); + TSHandleMLocRelease(resp_bufp, TS_NULL_MLOC, resp_hdr); + } + } + + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; +} + +} // namespace + +void +TSPluginInit(int argc, char const *argv[]) +{ + TSPluginRegistrationInfo info; + info.plugin_name = const_cast(PLUGIN_NAME); + info.vendor_name = const_cast("Apache"); + info.support_email = const_cast("dev@trafficserver.apache.org"); + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("[%s] Plugin registration failed", PLUGIN_NAME); + return; + } + + if (!parse_arguments(argc, argv)) { + return; + } + + TSCont contp = TSContCreate(handle_event, nullptr); + + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, contp); + TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, contp); + TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, contp); + TSHttpHookAdd(TS_HTTP_SEND_RESPONSE_HDR_HOOK, contp); +} + +TSReturnCode +TSRemapInit(TSRemapInterface * /* api_info ATS_UNUSED */, char * /* errbuf ATS_UNUSED */, int /* errbuf_size ATS_UNUSED */) +{ + return TS_SUCCESS; +} + +TSReturnCode +TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size) +{ + if (argc != 3) { + TSstrlcpy(errbuf, "expected from, to, and request-probe header arguments", errbuf_size); + return TS_ERROR; + } + + auto *config = new RemapConfig; + config->request_probe_header = argv[2]; + *ih = config; + + return TS_SUCCESS; +} + +void +TSRemapDeleteInstance(void *ih) +{ + delete static_cast(ih); +} + +TSRemapStatus +TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo * /* rri ATS_UNUSED */) +{ + auto const *config = static_cast(ih); + TSMBuffer req_bufp = nullptr; + TSMLoc req_hdr = TS_NULL_MLOC; + + if (TSHttpTxnClientReqGet(txnp, &req_bufp, &req_hdr) == TS_SUCCESS) { + if (has_header(req_bufp, req_hdr, config->request_probe_header)) { + log_unexpected(REMAP_REQUEST_ERROR_PREFIX, config->request_probe_header); + } + TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_hdr); + } + + return TSREMAP_NO_REMAP; +} diff --git a/tests/gold_tests/at_headers/replay/at_headers.replay.yaml b/tests/gold_tests/at_headers/replay/at_headers.replay.yaml new file mode 100644 index 00000000000..095406c4994 --- /dev/null +++ b/tests/gold_tests/at_headers/replay/at_headers.replay.yaml @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: "Verify client-supplied @ headers are stripped before global plugin and remap hooks run" + + at_headers: + global_path: &global_path "/global" + remap_path: &remap_path "/remap" + global_request_probe: &global_request_probe "@Client-Test" + response_probe: &response_probe "@Origin-Test" + request_added: &request_added "@Plugin-Request" + response_added: &response_added "@Plugin-Response" + remap_request_probe: &remap_request_probe "@Client-Remap-Test" + + server: + name: "server" + + client: + name: "client" + + ats: + name: "ts" + + process_config: + enable_cache: false + # There will be errors concerning the external @ heaers. + disable_log_checks: true + + plugin_config: + - name: "at_header_probe.so" + args: + - *global_request_probe + - *response_probe + - *request_added + - *response_added + + copy_custom_plugin: + - "plugins/.libs/at_header_probe.so" + + remap_config: + - "map /global http://127.0.0.1:{SERVER_HTTP_PORT}/global" + - from: "/remap" + to: "http://127.0.0.1:{SERVER_HTTP_PORT}/remap" + plugins: + - name: "at_header_probe.so" + args: + - *remap_request_probe + + metric_checks: + - metric: "proxy.process.http.client_request_at_headers_stripped" + value: 2 + - metric: "proxy.process.http.origin_response_at_headers_stripped" + value: 2 + + log_validation: + diags_log: + contains: + - expression: "stripped internal @ header from client request: @client-test" + description: "Client-side @ header removals should be logged." + - expression: "stripped internal @ header from origin response: @origin-test" + description: "Origin-side @ header removals should be logged." + - expression: "stripped internal @ header from client request: @client-remap-test" + description: "Client-side @ header removals should be logged before remap plugins run." + excludes: + - expression: "saw unexpected request header" + description: "Plugins should not see client-supplied @ headers." + - expression: "saw unexpected response header" + description: "Plugins should not see origin-supplied @ headers." + - expression: "saw unexpected remap request header" + description: "Remap plugins should not see client-supplied @ headers." + - expression: "FATAL:" + description: "ATS should not log fatal errors while stripping @ headers." + +sessions: + - transactions: + - client-request: + method: "GET" + url: *global_path + version: "1.1" + headers: + fields: + - [Host, example.com] + - [*global_request_probe, spoofed-value] + - [X-Normal, keep-me] + - [uuid, at-header-global-client] + + proxy-request: + method: "GET" + url: *global_path + headers: + fields: + - [*global_request_probe, {as: absent}] + - [*request_added, {as: absent}] + - [X-Normal, {value: keep-me, as: equal}] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [*response_probe, spoofed-response] + - [Content-Length, "0"] + + proxy-response: + status: 200 + headers: + fields: + - [*response_probe, {as: absent}] + - [*response_added, {as: absent}] + + - client-request: + method: "GET" + url: *remap_path + version: "1.1" + headers: + fields: + - [Host, example.com] + - [*remap_request_probe, spoofed-value] + - [X-Normal, keep-me] + - [uuid, at-header-remap-client] + + proxy-request: + method: "GET" + url: *remap_path + headers: + fields: + - [*remap_request_probe, {as: absent}] + - [*request_added, {as: absent}] + - [X-Normal, {value: keep-me, as: equal}] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [*response_probe, spoofed-remap-response] + - [Content-Length, "0"] + + proxy-response: + status: 200 + headers: + fields: + - [*response_probe, {as: absent}] + - [*response_added, {as: absent}] diff --git a/tests/gold_tests/autest-site/ats_replay.test.ext b/tests/gold_tests/autest-site/ats_replay.test.ext index 440f9ae22d4..04196e8711f 100644 --- a/tests/gold_tests/autest-site/ats_replay.test.ext +++ b/tests/gold_tests/autest-site/ats_replay.test.ext @@ -22,6 +22,53 @@ import os import re import yaml +_PATH_VARIABLE_RE = re.compile(r'\{([A-Za-z_][A-Za-z0-9_]*)\}') + + +def _resolve_test_artifact_path(obj, path: str) -> str: + '''Resolve a test artifact path for copy_custom_plugin. + + Resolution order after variable expansion: + 1. Absolute paths. + 2. Paths relative to the current test directory. + 3. Paths relative to the current test's build-tree mirror under + Variables.AtsBuildGoldTestsDir. + 4. Paths relative to Variables.AtsBuildGoldTestsDir. + ''' + + # Expand any specified obj.Variables into the path. + resolved_path = _PATH_VARIABLE_RE.sub(lambda match: str(getattr(obj.Variables, match.group(1), match.group(0))), path) + if os.path.isabs(resolved_path): + return resolved_path + + test_relative_path = os.path.join(obj.TestDirectory, resolved_path) + if os.path.exists(test_relative_path): + return test_relative_path + + gold_tests_source_dir = os.path.join(obj.Variables.RepoDir, 'tests', 'gold_tests') + if os.path.commonpath([gold_tests_source_dir, obj.TestDirectory]) == gold_tests_source_dir: + test_relpath = os.path.relpath(obj.TestDirectory, gold_tests_source_dir) + build_relative_path = os.path.join(obj.Variables.AtsBuildGoldTestsDir, test_relpath, resolved_path) + if os.path.exists(build_relative_path): + return build_relative_path + + # Otherwise, resolve relative to Variables.AtsBuildGoldTestsDir. + return os.path.join(obj.Variables.AtsBuildGoldTestsDir, resolved_path) + + +def _format_plugin_config_entry(plugin_entry) -> str: + '''Format a plugin.config entry from a string or structured config.''' + if isinstance(plugin_entry, str): + return plugin_entry + + if isinstance(plugin_entry, dict): + line = plugin_entry['name'] + for arg in plugin_entry.get('args', []): + line += f' {arg}' + return line + + raise TypeError(f'Unsupported plugin_config entry type: {type(plugin_entry)}') + def _contains_expression(contains_entry: dict, default_description: str): '''Create a ContainsExpression tester from a log validation entry.''' @@ -45,6 +92,11 @@ def configure_ats(obj: 'TestRun', server: 'Process', ats_config: dict, dns: Opti process_config = ats_config.get('process_config', {}) ts = obj.MakeATSProcess(name, **process_config) + # Cripts are compiled with clang at TS startup (during remap load), which is + # slow and scales with the number of cripts. + if process_config.get('enable_cripts', False): + ts.StartupTimeout = 60 + # Configure records_config if specified. records_config = ats_config.get('records_config', {}) ts.Disk.records_config.update(records_config) @@ -78,11 +130,17 @@ def configure_ats(obj: 'TestRun', server: 'Process', ats_config: dict, dns: Opti # Configure plugin_config if specified. plugin_config = ats_config.get('plugin_config', []) for plugin_line in plugin_config: - ts.Disk.plugin_config.AddLine(plugin_line) + ts.Disk.plugin_config.AddLine(_format_plugin_config_entry(plugin_line)) + + # Stage custom plugin artifacts into the ATS sandbox plugin directory. + for plugin_path in ats_config.get('copy_custom_plugin', []): + ts.Setup.Copy(_resolve_test_artifact_path(obj, plugin_path), ts.Env['PROXY_CONFIG_PLUGIN_PLUGIN_DIR']) # Configure parent_config if specified. parent_config = ats_config.get('parent_config', []) for parent_line in parent_config: + parent_line = parent_line.replace('{SERVER_HTTP_PORT}', str(server.Variables.http_port)) + parent_line = parent_line.replace('{SERVER_HTTPS_PORT}', str(server.Variables.https_port)) ts.Disk.parent_config.AddLine(parent_line) # Configure logging.yaml if specified. @@ -102,6 +160,8 @@ def configure_ats(obj: 'TestRun', server: 'Process', ats_config: dict, dns: Opti remap_config = ats_config.get('remap_config', []) for remap_entry in remap_config: if isinstance(remap_entry, str): + remap_entry = remap_entry.replace('{SERVER_HTTP_PORT}', str(server.Variables.http_port)) + remap_entry = remap_entry.replace('{SERVER_HTTPS_PORT}', str(server.Variables.https_port)) ts.Disk.remap_config.AddLine(remap_entry) elif isinstance(remap_entry, dict): from_url = remap_entry['from'] diff --git a/tests/gold_tests/autest-site/min_cfg/ip_allow.yaml b/tests/gold_tests/autest-site/min_cfg/ip_allow.yaml index 841286477cb..0ab41c18d6a 100644 --- a/tests/gold_tests/autest-site/min_cfg/ip_allow.yaml +++ b/tests/gold_tests/autest-site/min_cfg/ip_allow.yaml @@ -14,6 +14,10 @@ # the License. # Allow anything on localhost, limit destructive and debug methods elsewhere. +# Outbound CONNECT to unspecified, loopback, private, link-local, and +# IPv4-mapped IPv6 destinations is denied by default. Tests that intentionally +# use those tunnels should add an explicit outbound allow rule before the +# default deny rule. ip_allow: - apply: in ip_addrs: 127.0.0.1 @@ -43,3 +47,19 @@ ip_allow: - PUSH - DELETE - TRACE + - apply: out + ip_addrs: + - 0.0.0.0/8 + - 127.0.0.0/8 + - "::" + - ::1 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 169.254.0.0/16 + - ::/96 + - fc00::/7 + - fe80::/10 + - ::ffff:0:0/96 + action: deny + methods: CONNECT diff --git a/tests/gold_tests/autest-site/trafficserver.test.ext b/tests/gold_tests/autest-site/trafficserver.test.ext index 221aa16be1e..9b516247f8e 100755 --- a/tests/gold_tests/autest-site/trafficserver.test.ext +++ b/tests/gold_tests/autest-site/trafficserver.test.ext @@ -278,6 +278,7 @@ def MakeATSProcess( fname = "ip_allow.yaml" tmpname = os.path.join(config_dir, fname) p.Disk.File(tmpname, id=make_id(fname), typename="ats:config") + AddMethodToInstance(p, addPrivateConnectAllowYaml) fname = "logging.yaml" tmpname = os.path.join(config_dir, fname) @@ -679,6 +680,46 @@ def addSSLFileFromDefaultTestFolder(self, filename): addSSLfile(self, os.path.join(self.Variables.AtsTestToolsDir, "ssl", filename)) +CONNECT_DENY_RULE_MARKER = """ - apply: out + ip_addrs: + - 0.0.0.0/8 +""" + + +def _addPrivateConnectAllowRule(content, methods="CONNECT"): + rule = "\n".join([ + " - apply: out", + " ip_addrs: 127.0.0.1", + " action: allow", + f" methods: {methods}", + ]) + "\n" + + if rule in content: + return content + + marker_index = content.find(CONNECT_DENY_RULE_MARKER) + if marker_index >= 0: + return content[:marker_index] + rule + content[marker_index:] + + raise RuntimeError("Could not find default outbound CONNECT deny rule in ip_allow.yaml") + + +def addPrivateConnectAllowYaml(self, methods="CONNECT"): + """Allow tests to intentionally CONNECT to a loopback endpoint.""" + + def update_ip_allow_yaml(name): + with open(name) as f: + content = f.read() + with open(name, "w") as f: + f.write(_addPrivateConnectAllowRule(content, methods)) + return (True, "Updated file {0}".format(self.Disk.ip_allow_yaml.Name), "Success") + + if self.Disk.ip_allow_yaml.content is not None: + self.Disk.ip_allow_yaml.content = _addPrivateConnectAllowRule(self.Disk.ip_allow_yaml.content, methods) + else: + self.Disk.ip_allow_yaml.WriteCustomOn(update_ip_allow_yaml) + + RegisterFileType(Config, "ats:config") RegisterFileType(YAMLFile, "ats:config:yaml") RegisterFileType(RecordsYAML, "ats:config:records") diff --git a/tests/gold_tests/cache/cache_corrupt_recursive_uaf.test.py b/tests/gold_tests/cache/cache_corrupt_recursive_uaf.test.py new file mode 100644 index 00000000000..2dc20fec9bb --- /dev/null +++ b/tests/gold_tests/cache/cache_corrupt_recursive_uaf.test.py @@ -0,0 +1,127 @@ +''' +Recursive cache-read tracking must not use-after-free on failure. + +This autest sets ATS_TEST_FORCE_CORRUPT_DOC=1 in the ATS environment, which +makes openReadStartEarliest treat every doc magic as corrupt. Reading a +multi-fragment object then drives the recursive earliest-read path: each level +re-enters openReadStartEarliest via do_read_call, and when an inner level falls +into free_CacheVC the outer frame -- on the *unfixed* code -- decrements the +recursion counter through the now-freed CacheVC member. A sanitizer build +catches that as a use-after-free. Note the depth limit ("Too many recursive +calls") is NOT required to trigger the bug: the dangling decrement happens at +whatever level the inner call frees `this`. + +With the fix in place the counter lives in a thread_local that outlives any +freed CacheVC, so the recursive run completes cleanly and the proxy stays up. + +NOTE ON COVERAGE: the use-after-free is only *caught* when ATS is built with a +memory sanitizer (e.g. the ASan preset the security CI uses). On a non-sanitized +build the dangling decrement is typically a silent no-op, so this test would +pass even on the unfixed sources. What the test verifies unconditionally is that +the forced-corrupt earliest-read path actually runs ("Doc magic does not match") +and that the proxy never crashes (no FATAL). Run it under a sanitizer build to +get the regression-catching guarantee. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = 'Recursive cache read must not use-after-free when every doc magic is corrupt' + +# The corrupt-doc env hook is compiled in only when TS_HAS_TESTS is enabled; +# without it the hook is a constexpr false and this test would silently exercise +# a normal cache hit instead of the recursive corrupt-doc path. +Test.SkipUnless(Condition.HasATSFeature('TS_HAS_TESTS')) + +Test.ContinueOnFail = True + +server = Test.MakeOriginServer("server") + +# The corrupt-doc hook lives in openReadStartEarliest, which is only reached when +# the cached object spans more than one fragment (a single-fragment object is +# served entirely from openReadStartHead and never enters the recursive earliest +# read path). Use a multi-fragment body together with a small target_fragment_size +# below so the second read actually drives openReadStartEarliest. 64 KiB over an +# 8 KiB fragment size is several fragments while keeping the test lightweight. +body = "x" * (64 * 1024) + +server.addResponse( + "sessionlog.json", { + "headers": "GET /obj HTTP/1.1\r\nHost: *\r\n\r\n", + "timestamp": "1", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\nContent-Length: {0}\r\nCache-Control: max-age=3600\r\n\r\n".format(len(body)), + "timestamp": "1", + "body": body + }) + +ts = Test.MakeATSProcess("ts", enable_cache=True) +# Force the corrupt-doc path inside openReadStartEarliest for every cache read. +ts.Env['ATS_TEST_FORCE_CORRUPT_DOC'] = '1' + +ts.Disk.records_config.update( + { + 'proxy.config.http.wait_for_cache': 1, + # Force the object to be written as several small fragments so the read + # path enters openReadStartEarliest (sizeof(Doc) < value <= cap). Debug + # logging is intentionally left off: the corruption/recursion diagnostics + # asserted below are Warning/Error level and always reach diags.log, and + # cache_read debug over many fragments would bloat the run's disk use. + 'proxy.config.cache.target_fragment_size': 8192, + }) + +ts.Disk.remap_config.AddLine('map http://example.com/ http://127.0.0.1:{0}/'.format(server.Variables.Port)) + +# Assert the forced-corrupt earliest-read path actually ran. The depth-limit +# diagnostic ("Too many recursive calls") is deliberately NOT required: the +# recursion frees `this` at whatever level the inner read fails, which is the +# use-after-free locus, and it does not need to reach MAX_READ_RECURSION_DEPTH. +# So the reliable, sanitizer-independent signal that the vulnerable path was +# exercised is the corruption warning itself. The real regression guard is a +# sanitizer build catching the UAF on the unfixed sources (see module docstring). +# Assigning Content with `=` replaces the framework's default ERROR:/FATAL: +# guards, so re-add a FATAL: guard to keep asserting the proxy never crashed. +ts.Disk.diags_log.Content = Testers.ContainsExpression("Doc magic does not match", "the forced-corrupt earliest-read path must run") +ts.Disk.diags_log.Content += Testers.ExcludesExpression("FATAL:", "ATS must not crash on the recursive corrupt-doc path") + +# First request: cache miss, populate cache from origin. +tr1 = Test.AddTestRun() +tr1.MakeCurlCommandMulti( + '{curl} -sS -i -x 127.0.0.1:TSPORT http://example.com/obj'.replace('TSPORT', str(ts.Variables.port)), ts=ts) +tr1.Processes.Default.StartBefore(ts) +tr1.Processes.Default.StartBefore(server) +tr1.Processes.Default.ReturnCode = 0 +# The populate read also walks the forced-corrupt earliest path; assert ATS and +# origin survive it (a UAF crash here would otherwise look like a later failure). +tr1.StillRunningAfter = ts +tr1.StillRunningAfter = server + +# Second request: cache hit; openReadStartEarliest treats the doc as corrupt +# (env var) and drives the recursive earliest-read path. With the fix ATS +# survives; the unfixed code would be caught by a sanitizer as UAF on the +# recursion counter. +tr2 = Test.AddTestRun() +tr2.MakeCurlCommandMulti( + '{curl} -sS -i -x 127.0.0.1:TSPORT http://example.com/obj'.replace('TSPORT', str(ts.Variables.port)), ts=ts) +tr2.Processes.Default.ReturnCode = 0 +# Distinguishing assertion: ATS must remain up and respond. Body content +# doesn't matter; what matters is the proxy didn't crash. +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("HTTP/1.1", "ATS must respond, not crash") +# Belt-and-suspenders: the proxy (and origin) must still be alive after serving +# the recursive corrupt-doc read -- catches a crash that lands right after the +# response is written. +tr2.StillRunningAfter = ts +tr2.StillRunningAfter = server diff --git a/tests/gold_tests/cache/host_down_range_recursion.test.py b/tests/gold_tests/cache/host_down_range_recursion.test.py new file mode 100644 index 00000000000..db3711fdaf1 --- /dev/null +++ b/tests/gold_tests/cache/host_down_range_recursion.test.py @@ -0,0 +1,81 @@ +''' +Verify that a Range request to a DOWN host backed by a cache HIT does not +trigger unbounded recursion. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from jsonrpc import Request + +Test.Summary = ''' +Verify that a Range request the cache cannot satisfy against a host marked +DOWN does not trigger unbounded recursion in HttpTransact. +''' + + +class HostDownRangeRecursionTest: + prime_replay = "replay/host_down_range_recursion_prime.replay.yaml" + range_replay = "replay/host_down_range_recursion_range.replay.yaml" + + def __init__(self): + # Same server process serves both replay files. Only the prime phase + # actually reaches the server; the range phase is expected to be + # short-circuited by the DOWN host check. + self._server = Test.MakeVerifierServerProcess("server", self.prime_replay) + self._configure_ts() + + def _configure_ts(self): + self._ts = Test.MakeATSProcess("ts", enable_cache=True) + + self._ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|host_statuses', + 'proxy.config.http.cache.range.write': 1, + 'proxy.config.http.insert_response_via_str': 3, + }) + + self._ts.Disk.remap_config.AddLine(f'map http://backend.example.com/ http://127.0.0.1:{self._server.Variables.http_port}/') + + def run(self): + # Phase 1: prime the cache with the full object. + prime = Test.AddTestRun("Prime cache with full object") + prime.AddVerifierClientProcess("prime-client", self.prime_replay, http_ports=[self._ts.Variables.port]) + prime.Processes.Default.StartBefore(self._server) + prime.Processes.Default.StartBefore(self._ts) + prime.StillRunningAfter = self._server + prime.StillRunningAfter = self._ts + + # Phase 2: mark the origin host DOWN via JSON-RPC. + mark_down = Test.AddTestRun("Mark host DOWN") + mark_down.AddJsonRPCClientRequest( + self._ts, Request.admin_host_set_status(operation='down', host=['127.0.0.1'], reason='manual', time='0')) + mark_down.StillRunningAfter = self._server + mark_down.StillRunningAfter = self._ts + + # Phase 3: send an out-of-order multi-range request. The cache cannot + # satisfy the range, so ATS must fall back to the origin; with the host + # marked DOWN it should return 502 Bad Gateway. This is a regression + # test for a stack-overflow crash where this scenario instead drove an + # unbounded recursion between the cache-fallback and DNS-lookup paths. + range_run = Test.AddTestRun("Out-of-order Range against DOWN host") + range_run.AddVerifierClientProcess("range-client", self.range_replay, http_ports=[self._ts.Variables.port]) + range_run.Processes.Default.TimeOut = 10 + range_run.StillRunningAfter = self._server + range_run.StillRunningAfter = self._ts + + +HostDownRangeRecursionTest().run() diff --git a/tests/gold_tests/cache/replay/host_down_range_recursion_prime.replay.yaml b/tests/gold_tests/cache/replay/host_down_range_recursion_prime.replay.yaml new file mode 100644 index 00000000000..a5fba7ea9a6 --- /dev/null +++ b/tests/gold_tests/cache/replay/host_down_range_recursion_prime.replay.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Phase 1: prime the cache with a full 11-byte response to /cached. +# +meta: + version: "1.0" + +sessions: +- transactions: + - client-request: + method: GET + version: "1.1" + url: /cached + headers: + fields: + - [ Host, backend.example.com ] + - [ uuid, prime ] + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 11 ] + - [ Cache-Control, "max-age=300" ] + - [ Last-Modified, "Thu, 10 Feb 2022 00:00:00 GMT" ] + - [ ETag, range ] + content: + encoding: plain + data: "0123456789\n" + proxy-response: + status: 200 diff --git a/tests/gold_tests/cache/replay/host_down_range_recursion_range.replay.yaml b/tests/gold_tests/cache/replay/host_down_range_recursion_range.replay.yaml new file mode 100644 index 00000000000..5cc9e7bde60 --- /dev/null +++ b/tests/gold_tests/cache/replay/host_down_range_recursion_range.replay.yaml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Phase 2: after the origin host has been marked DOWN, send an out-of-order +# multi-range request against the cached resource. The Range parser marks +# this as NOT_HANDLED, so the cache cannot satisfy the request and ATS must +# fall back to the origin; with the host DOWN it should return 502 Bad +# Gateway. This is a regression test for an unbounded recursion between +# build_response_from_cache, CallOSDNSLookup, and +# handle_server_connection_not_open that crashed the proxy via stack +# overflow. +# +meta: + version: "1.0" + +sessions: +- transactions: + - client-request: + method: GET + version: "1.1" + url: /cached + headers: + fields: + - [ Host, backend.example.com ] + - [ uuid, range-down ] + # Out-of-order multi-range: second range starts before first ends. + - [ Range, "bytes=5-9,0-4" ] + proxy-request: + # Host is DOWN; ATS must not forward this to origin. + expect: absent + server-response: + # Unused; included because replay format requires it. + status: 500 + reason: "Internal Server Error" + proxy-response: + # Cache cannot satisfy the range and the host is DOWN, so ATS returns + # 502 Bad Gateway rather than re-entering the cache fallback path. + status: 502 diff --git a/tests/gold_tests/chunked_encoding/chunk_extension_client.py b/tests/gold_tests/chunked_encoding/chunk_extension_client.py new file mode 100644 index 00000000000..96f2ba996f9 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/chunk_extension_client.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +''' +Send a single chunked POST whose chunk extension carries a quoted-string value. +The quoted value embeds octets that look like a second HTTP request. Per RFC 9110 +Section 5.6.4 the quoted-string runs until its closing DQUOTE, so a correct proxy +treats the whole thing as one request and the embedded GET is never parsed as a +request of its own. + +The script prints the number of HTTP responses received on the connection and the +raw response, so the test can assert that exactly one response came back and that +the embedded endpoint was not reached. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +import sys +import time + +# A chunked POST whose first chunk opens a quoted-string extension value with a +# CR/LF inside it. A correct parser rejects the request (the quoted-string cannot +# contain CR/LF). A parser that ignores the quoted-string ends the chunk size +# line at the first CRLF, reads "X" as the one-byte chunk, sees the 0-chunk, and +# then treats the trailing GET as a separate pipelined request, forwarding it to +# the origin. The origin answers that GET /second with a distinctive body, so if +# it ever reaches the client the request was smuggled. +PAYLOAD = ( + b"POST / HTTP/1.1\r\n" + b"Host: localhost\r\n" + b"uuid: 1\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + b'1;a="\r\n' + b"X\r\n" + b"0\r\n" + b"\r\n" + b"GET /second HTTP/1.1\r\n" + b"Host: localhost\r\n" + b"uuid: 2\r\n" + b"Connection: close\r\n" + b"\r\n") + +# Byte offset at which to split the payload across two sends, right after the +# opening DQUOTE and before the CR inside it, so the proxy must suspend parsing +# the extension on the first read and resume it (and reject) on the second. +SPLIT_AT = PAYLOAD.index(b'1;a="') + len(b'1;a="') + + +def main() -> int: + if len(sys.argv) < 3: + print(f"usage: {sys.argv[0]} [--split]", file=sys.stderr) + return 2 + + host, port = sys.argv[1], int(sys.argv[2]) + split = "--split" in sys.argv[3:] + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(5) + response = b"" + try: + sock.connect((host, port)) + if split: + sock.sendall(PAYLOAD[:SPLIT_AT]) + time.sleep(0.3) # Force the remainder into a separate read on the proxy. + try: + sock.sendall(PAYLOAD[SPLIT_AT:]) + except OSError: + pass # The proxy may have already rejected and closed the connection. + else: + sock.sendall(PAYLOAD) + while True: + try: + chunk = sock.recv(4096) + except socket.timeout: + break + if not chunk: + break + response += chunk + finally: + sock.close() + + responses = response.count(b"HTTP/1.1 ") + status = response.split(b"\r\n", 1)[0].decode(errors="replace") if response else "(no response)" + print(f"responses={responses}") + print(f"status={status}") + print("=== response ===") + print(response.decode(errors="replace")) + print("=== end ===") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/gold_tests/chunked_encoding/chunk_extension_quoted_string.test.py b/tests/gold_tests/chunked_encoding/chunk_extension_quoted_string.test.py new file mode 100644 index 00000000000..a670805d367 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/chunk_extension_quoted_string.test.py @@ -0,0 +1,84 @@ +''' +Verify that a chunk extension whose quoted-string value contains CR or LF is +rejected. Per RFC 9110 Section 5.6.4 a quoted-string cannot contain a bare CR or +LF, so such a chunk size line is malformed. ATS must reject the request rather +than interpreting the embedded octets as a second, smuggled request: the client +gets a single error response and the embedded GET never reaches the origin. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +A chunk extension quoted-string value containing CR/LF is rejected with 400. +''' + + +class ChunkExtensionQuotedStringTest: + + def __init__(self): + self._setup_origin() + self._setup_ts() + + def _setup_origin(self): + # A proxy-verifier origin parses chunked requests correctly and serves the + # legitimate POST with 200 (keep-alive). If the proxy forwards the embedded + # GET /second, the origin answers it with SECOND-ENDPOINT, so a smuggled + # request shows up as a second response on the client. + self._server = Test.MakeVerifierServerProcess("verifier-server", "replays/chunk_extension_quoted_string.replay.yaml") + + def _setup_ts(self): + self._ts = Test.MakeATSProcess("ts", enable_cache=False) + self._ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.http.strict_chunk_parsing': 1, + }) + self._ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.http_port}') + + def _check(self, tr): + tr.Processes.Default.ReturnCode = 0 + # The malformed chunk size line is rejected, so the client gets exactly one + # error response and the embedded GET /second is never forwarded. A parser + # that ignores the quoted-string would forward that GET as a second request + # and the origin's SECOND-ENDPOINT body would reach the client (two + # responses). Keying on the smuggled body is deterministic; the exact error + # status is not (ATS may relay the origin's response to the forwarded POST + # headers, or generate its own 400). + tr.Processes.Default.Streams.stdout += Testers.ContainsExpression("responses=1", "the client must get exactly one response") + tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "SECOND-ENDPOINT", "the embedded GET must not be smuggled to the origin") + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + def run(self): + # Case 1: the whole request arrives in one read. + tr = Test.AddTestRun("Quoted extension value containing CRLF") + tr.Setup.Copy("chunk_extension_client.py") + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.Command = f'python3 chunk_extension_client.py 127.0.0.1 {self._ts.Variables.port}' + self._check(tr) + + # Case 2: the same request split across two writes at a read boundary + # inside the extension. The proxy must resume parsing and still reject it. + tr = Test.AddTestRun("Quoted extension split across reads") + tr.Setup.Copy("chunk_extension_client.py") + tr.Processes.Default.Command = f'python3 chunk_extension_client.py 127.0.0.1 {self._ts.Variables.port} --split' + self._check(tr) + + +ChunkExtensionQuotedStringTest().run() diff --git a/tests/gold_tests/chunked_encoding/chunk_trailer_bare_lf.test.py b/tests/gold_tests/chunked_encoding/chunk_trailer_bare_lf.test.py new file mode 100644 index 00000000000..ab56a721638 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/chunk_trailer_bare_lf.test.py @@ -0,0 +1,86 @@ +''' +Verify that a chunked request whose trailer section is terminated by a bare LF +instead of CRLF is rejected. Per RFC 9112 Section 7.1 the trailer section ends +with an empty line, "CRLF"; a bare LF blank line is not a valid terminator. ATS +must reject the request rather than ending the body one byte early and +interpreting the trailing octets as a second, smuggled request: the client gets a +single response and the embedded GET never reaches the origin. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +A chunked trailer terminated by a bare LF instead of CRLF is rejected. +''' + + +class ChunkTrailerBareLfTest: + + def __init__(self): + self._setup_origin() + self._setup_ts() + + def _setup_origin(self): + # A proxy-verifier origin parses chunked requests correctly and serves the + # legitimate POST with 200 (keep-alive). If the proxy forwards the embedded + # GET /smuggled, the origin answers it with SECOND-ENDPOINT, so a smuggled + # request shows up as a second response on the client. + self._server = Test.MakeVerifierServerProcess("verifier-server", "replays/chunk_trailer_bare_lf.replay.yaml") + + def _setup_ts(self): + self._ts = Test.MakeATSProcess("ts", enable_cache=False) + self._ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.http.strict_chunk_parsing': 1, + }) + self._ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.http_port}') + + def _check(self, tr): + tr.Processes.Default.ReturnCode = 0 + # The bare-LF trailer terminator is rejected, so the client gets exactly one + # response and the embedded GET /smuggled is never forwarded. A parser that + # accepts the bare LF would forward that GET as a second request and the + # origin's SECOND-ENDPOINT body would reach the client (two responses). + # Keying on the smuggled body is deterministic; the exact error status is + # not (ATS may relay the origin's response to the forwarded POST headers, or + # generate its own 400). + tr.Processes.Default.Streams.stdout += Testers.ContainsExpression("responses=1", "the client must get exactly one response") + tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "SECOND-ENDPOINT", "the embedded GET must not be smuggled to the origin") + tr.StillRunningAfter = self._server + tr.StillRunningAfter = self._ts + + def run(self): + # Case 1: the whole request arrives in one read. + tr = Test.AddTestRun("Chunked trailer terminated by a bare LF") + tr.Setup.Copy("chunk_trailer_client.py") + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.Command = f'python3 chunk_trailer_client.py 127.0.0.1 {self._ts.Variables.port}' + self._check(tr) + + # Case 2: the same request split across two writes at a read boundary right + # after the final "0\r\n". The proxy must resume parsing the trailer and + # still reject the bare-LF terminator. + tr = Test.AddTestRun("Chunked trailer bare LF split across reads") + tr.Setup.Copy("chunk_trailer_client.py") + tr.Processes.Default.Command = f'python3 chunk_trailer_client.py 127.0.0.1 {self._ts.Variables.port} --split' + self._check(tr) + + +ChunkTrailerBareLfTest().run() diff --git a/tests/gold_tests/chunked_encoding/chunk_trailer_client.py b/tests/gold_tests/chunked_encoding/chunk_trailer_client.py new file mode 100644 index 00000000000..94c03c0257a --- /dev/null +++ b/tests/gold_tests/chunked_encoding/chunk_trailer_client.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +''' +Send a single chunked POST whose body ends with a bare-LF trailer terminator. Per +RFC 9112 Section 7.1 the trailer section ends with an empty line, "CRLF". A parser +that accepts a bare LF blank line ends the body one byte early, so the bytes that +follow are framed as a separate, smuggled request. A parser that requires CRLF +rejects the request instead and never forwards the embedded GET. + +The script prints the number of HTTP responses received on the connection and the +raw response, so the test can assert that exactly one response came back and that +the embedded endpoint was not reached. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +import sys +import time + +# A chunked POST whose final (zero-size) chunk is followed by a bare LF instead of +# CRLF as the trailer terminator. A correct parser requires CRLF and rejects the +# request. A parser that accepts the bare LF ends the body at "0\r\n\n", reads the +# trailing GET as a separate pipelined request, and forwards it to the origin. The +# origin answers that GET /smuggled with a distinctive body, so if it ever reaches +# the client the request was smuggled. The body is empty (the bug is in the +# trailer terminator, not the body) so the origin drains it cleanly. +PAYLOAD = ( + b"POST /legit HTTP/1.1\r\n" + b"Host: localhost\r\n" + b"uuid: 1\r\n" + b"Transfer-Encoding: chunked\r\n" + b"\r\n" + b"0\r\n" + b"\n" + b"GET /smuggled HTTP/1.1\r\n" + b"Host: localhost\r\n" + b"uuid: 2\r\n" + b"Connection: close\r\n" + b"\r\n") + +# Byte offset at which to split the payload across two sends, right after the +# final "0\r\n" and before the bare LF terminator, so the proxy must suspend +# parsing the trailer on the first read and resume it (and reject) on the second. +SPLIT_AT = PAYLOAD.index(b"0\r\n") + len(b"0\r\n") + + +def main() -> int: + if len(sys.argv) < 3: + print(f"usage: {sys.argv[0]} [--split]", file=sys.stderr) + return 2 + + host, port = sys.argv[1], int(sys.argv[2]) + split = "--split" in sys.argv[3:] + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(5) + response = b"" + try: + sock.connect((host, port)) + if split: + sock.sendall(PAYLOAD[:SPLIT_AT]) + time.sleep(0.3) # Force the remainder into a separate read on the proxy. + try: + sock.sendall(PAYLOAD[SPLIT_AT:]) + except OSError: + pass # The proxy may have already rejected and closed the connection. + else: + sock.sendall(PAYLOAD) + while True: + try: + chunk = sock.recv(4096) + except socket.timeout: + break + if not chunk: + break + response += chunk + finally: + sock.close() + + responses = response.count(b"HTTP/1.1 ") + status = response.split(b"\r\n", 1)[0].decode(errors="replace") if response else "(no response)" + print(f"responses={responses}") + print(f"status={status}") + print("=== response ===") + print(response.decode(errors="replace")) + print("=== end ===") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/gold_tests/chunked_encoding/chunked_flow_control.test.py b/tests/gold_tests/chunked_encoding/chunked_flow_control.test.py new file mode 100644 index 00000000000..c1ee966174e --- /dev/null +++ b/tests/gold_tests/chunked_encoding/chunked_flow_control.test.py @@ -0,0 +1,24 @@ +''' +Regression test: chunked-origin -> HTTP/2 client dechunk flow-control throttle. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = __doc__ + +Test.ATSReplayTest(replay_file="replays/chunked_h1_flow_control.replay.yaml") +Test.ATSReplayTest(replay_file="replays/chunked_h2_flow_control.replay.yaml") +Test.ATSReplayTest(replay_file="replays/chunked_passthru_flow_control.replay.yaml") diff --git a/tests/gold_tests/chunked_encoding/chunked_not_last.test.py b/tests/gold_tests/chunked_encoding/chunked_not_last.test.py new file mode 100644 index 00000000000..c45088c0181 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/chunked_not_last.test.py @@ -0,0 +1,25 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +RFC 9112 requires chunked to be the final transfer coding. Verify that ATS +rejects requests where chunked is not the last Transfer-Encoding value, and +handles responses correctly when chunked is not terminal. +''' + +Test.ATSReplayTest(replay_file='replays/chunked_not_last.replay.yaml') diff --git a/tests/gold_tests/chunked_encoding/replays/chunk_extension_quoted_string.replay.yaml b/tests/gold_tests/chunked_encoding/replays/chunk_extension_quoted_string.replay.yaml new file mode 100644 index 00000000000..323b9c01019 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/replays/chunk_extension_quoted_string.replay.yaml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Origin-side replay. The verifier server parses chunked requests correctly and +# serves the legitimate POST with 200, keeping the connection alive. If the proxy +# ever forwards the embedded GET /second (the smuggle), the server answers it with +# a distinctive body so the client side can detect it. + +meta: + version: "1.0" + +sessions: +- transactions: + - client-request: + method: "POST" + version: "1.1" + url: / + headers: + fields: + - [ Host, localhost ] + - [ uuid, 1 ] + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 2 ] + + - client-request: + method: "GET" + version: "1.1" + url: /second + headers: + fields: + - [ Host, localhost ] + - [ uuid, 2 ] + server-response: + status: 200 + reason: OK + content: + data: "SECOND-ENDPOINT" + headers: + fields: + - [ Content-Length, 15 ] diff --git a/tests/gold_tests/chunked_encoding/replays/chunk_trailer_bare_lf.replay.yaml b/tests/gold_tests/chunked_encoding/replays/chunk_trailer_bare_lf.replay.yaml new file mode 100644 index 00000000000..b97aaf1c98d --- /dev/null +++ b/tests/gold_tests/chunked_encoding/replays/chunk_trailer_bare_lf.replay.yaml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Origin-side replay. The verifier server parses chunked requests correctly and +# serves the legitimate POST with 200, keeping the connection alive. If the proxy +# ever forwards the second request that is hidden after a bare-LF trailer +# terminator, the server answers it with a distinctive body so the client side can +# detect the smuggle. + +meta: + version: "1.0" + +sessions: +- transactions: + - client-request: + method: "POST" + version: "1.1" + url: /legit + headers: + fields: + - [ Host, localhost ] + - [ uuid, 1 ] + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 2 ] + + - client-request: + method: "GET" + version: "1.1" + url: /smuggled + headers: + fields: + - [ Host, localhost ] + - [ uuid, 2 ] + server-response: + status: 200 + reason: OK + content: + data: "SECOND-ENDPOINT" + headers: + fields: + - [ Content-Length, 15 ] diff --git a/tests/gold_tests/chunked_encoding/replays/chunked_h1_flow_control.replay.yaml b/tests/gold_tests/chunked_encoding/replays/chunked_h1_flow_control.replay.yaml new file mode 100644 index 00000000000..8257559ac85 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/replays/chunked_h1_flow_control.replay.yaml @@ -0,0 +1,138 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +# Regression test for the chunked-tunnel high-water throttle on the DOCHUNK +# (re-chunking) producer path in HttpTunnel. +# +# An HTTP/1.0 origin returns a body without Content-Length (close-delimited) +# and an HTTP/1.1 client requests it. ATS must re-chunk the body for the +# client, which exercises ChunkedHandler::Action::DOCHUNK and writes +# generated chunks into the chunked_buffer output. With a small +# default_buffer_water_mark, that output buffer crosses high_water before +# the consumer drains it, which must trigger producer_handler's +# "disable ... read_vio" throttle (HttpTunnel.cc) and a re-enable when the +# consumer catches up. The response must still complete with the full body. + +autest: + description: 'HTTP/1.0 origin -> HTTP/1.1 chunked client do-chunk flow-control throttle' + + dns: + name: 'dns-chunked-h1-fc' + + server: + name: 'server-chunked-h1-fc' + + client: + name: 'client-chunked-h1-fc' + + ats: + name: 'ts-chunked-h1-fc' + process_config: + enable_tls: false + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http_tunnel' + # Small water_mark so the chunked output buffer crosses high_water + # well before the user-agent socket can drain it. + proxy.config.http.default_buffer_water_mark: 4096 + + remap_config: + - from: "http://www.example.com/" + to: "http://backend.example.com:{SERVER_HTTP_PORT}/" + + log_validation: + diags_log: + excludes: + # Indicator of the tunnel getting stuck on a dropped terminal event. + - expression: 'ERROR.*inactivity timeout' + description: 'Tunnel must not stall to inactivity timeout.' + + metric_checks: + # The chunked-tunnel throttle must fire at least once: the chunked + # output buffer crosses high_water before the consumer drains it. + - metric: proxy.process.http.tunnel.chunked_throttle + value: '[1-9][0-9]*' + +sessions: + +- protocol: + stack: http + + transactions: + + # An HTTP/1.0 origin delivering a sizable, close-delimited body. ATS will + # re-chunk it for the HTTP/1.1 client, which is the DOCHUNK path. + - client-request: + method: GET + version: "1.1" + url: /large-rechunk + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, large-rechunk-1 ] + + server-response: + status: 200 + reason: OK + version: "1.0" + headers: + fields: + - [ "Content-Type", "text/plain" ] + content: + size: 524288 # 512 KB + + proxy-response: + status: 200 + headers: + fields: + - [ "Transfer-Encoding", { value: "chunked", as: equal } ] + content: + size: { value: 524288, as: equal } + + # A second transaction back-to-back, exercising the re-enable path + # after the first throttle cycle on a fresh connection. + - client-request: + delay: 100ms + method: GET + version: "1.1" + url: /large-rechunk-2 + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, large-rechunk-2 ] + + server-response: + status: 200 + reason: OK + version: "1.0" + headers: + fields: + - [ "Content-Type", "text/plain" ] + content: + size: 262144 # 256 KB + + proxy-response: + status: 200 + headers: + fields: + - [ "Transfer-Encoding", { value: "chunked", as: equal } ] + content: + size: { value: 262144, as: equal } diff --git a/tests/gold_tests/chunked_encoding/replays/chunked_h2_flow_control.replay.yaml b/tests/gold_tests/chunked_encoding/replays/chunked_h2_flow_control.replay.yaml new file mode 100644 index 00000000000..f771a2bb1e7 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/replays/chunked_h2_flow_control.replay.yaml @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +# Regression test for the chunked-dechunk high-water throttle in HttpTunnel. +# +# An HTTP/1.1 origin delivers a large chunked response; an HTTP/2 client +# consumes it. With a small default_buffer_water_mark, the dechunked buffer +# fills past high_water before the consumer drains it, which must trigger +# producer_handler's "disable ... read_vio" throttle (HttpTunnel.cc) and +# later a "re-enable ... read_vio" when the consumer catches up. The +# response must still complete with the full body intact. +# +# Without the throttle the dechunk buffer would grow unbounded (the +# original DoS vector). Without the terminal-event guard in producer_handler +# the transaction would stall because VC_EVENT_READ_COMPLETE / EOS / +# PRECOMPLETE would be dropped. + +autest: + description: 'HTTP/1.1 chunked origin -> HTTP/2 client dechunk flow-control throttle' + + dns: + name: 'dns-chunked-h2-fc' + + server: + name: 'server-chunked-h2-fc' + + client: + name: 'client-chunked-h2-fc' + + ats: + name: 'ts-chunked-h2-fc' + process_config: + enable_tls: true + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http_tunnel' + # Small water_mark so the dechunk output buffer crosses high_water + # well before the HTTP/2 consumer can drain it. + proxy.config.http.default_buffer_water_mark: 4096 + proxy.config.ssl.client.verify.server.policy: 'PERMISSIVE' + + remap_config: + - from: "https://www.example.com/" + to: "http://backend.example.com:{SERVER_HTTP_PORT}/" + + log_validation: + diags_log: + excludes: + # Indicator of the tunnel getting stuck on a dropped terminal event + # (the bug the producer_handler event guard fixes). + - expression: 'ERROR.*inactivity timeout' + description: 'Tunnel must not stall to inactivity timeout.' + + metric_checks: + # The chunked-tunnel throttle must fire at least once: the dechunked + # buffer crosses high_water before the H/2 client drains it. Combined + # with the proxy-response size assertion below (which only passes if + # the producer was re-enabled), this covers both the throttle and the + # unwind paths. + - metric: proxy.process.http.tunnel.chunked_throttle + value: '[1-9][0-9]*' + +sessions: + +- protocol: + stack: http2 + tls: + sni: www.example.com + transactions: + + # A sizable chunked response. The body is large enough to exceed + # water_mark many times over, and larger than the default HTTP/2 + # stream window (65535) so WINDOW_UPDATE round-trips naturally + # keep the consumer from draining instantly. + - client-request: + headers: + fields: + - [ ":method", "GET" ] + - [ ":scheme", "https" ] + - [ ":authority", "www.example.com" ] + - [ ":path", "/large-chunked" ] + - [ "uuid", "large-chunked-1" ] + + server-response: + status: 200 + reason: OK + version: '1.1' + headers: + fields: + - [ "Content-Type", "text/plain" ] + - [ "Transfer-Encoding", "chunked" ] + content: + size: 524288 # 512 KB + + proxy-response: + headers: + fields: + - [ ":status", { value: '200', as: equal } ] + content: + size: { value: 524288, as: equal } + + # A second transaction back-to-back, to exercise the re-enable path + # and confirm the tunnel recovers cleanly from the first throttle cycle. + - client-request: + delay: 100ms + headers: + fields: + - [ ":method", "GET" ] + - [ ":scheme", "https" ] + - [ ":authority", "www.example.com" ] + - [ ":path", "/large-chunked-2" ] + - [ "uuid", "large-chunked-2" ] + + server-response: + status: 200 + reason: OK + version: '1.1' + headers: + fields: + - [ "Content-Type", "text/plain" ] + - [ "Transfer-Encoding", "chunked" ] + content: + size: 262144 # 256 KB + + proxy-response: + headers: + fields: + - [ ":status", { value: '200', as: equal } ] + content: + size: { value: 262144, as: equal } diff --git a/tests/gold_tests/chunked_encoding/replays/chunked_not_last.replay.yaml b/tests/gold_tests/chunked_encoding/replays/chunked_not_last.replay.yaml new file mode 100644 index 00000000000..c051753f987 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/replays/chunked_not_last.replay.yaml @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# RFC 9112 requires chunked to be the final transfer coding. Verify that ATS +# rejects requests where chunked is not the last Transfer-Encoding value, and +# handles responses correctly. + +meta: + version: "1.0" + +autest: + description: 'Verify chunked must be the last Transfer-Encoding value' + + server: + name: 'server' + + client: + name: 'client' + + ats: + name: 'ts' + + process_config: + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http' + + remap_config: + - from: "/" + to: "http://127.0.0.1:{SERVER_HTTP_PORT}/" + +sessions: + +# --- Request-side tests --- + +# Test 1: chunked is NOT last in request Transfer-Encoding — reject with 501 +- transactions: + - client-request: + method: POST + version: '1.1' + url: /req/chunked-not-last + headers: + fields: + - [Host, example.com] + - [Transfer-Encoding, "chunked, gzip"] + - [uuid, req-chunked-not-last] + content: + size: 0 + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "0"] + + proxy-response: + status: 501 + +# Test 2: single chunked in request Transfer-Encoding — accept (baseline) +- transactions: + - client-request: + method: POST + version: '1.1' + url: /req/chunked-only + headers: + fields: + - [Host, example.com] + - [Transfer-Encoding, chunked] + - [uuid, req-chunked-only] + content: + transfer: chunked + data: hello + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "2"] + content: + encoding: plain + data: ok + + proxy-response: + status: 200 + +# --- Response-side tests --- + +# Test 3: chunked is NOT last in response Transfer-Encoding — origin violates RFC 9112. +# ATS should reject with 502 Bad Gateway. +- transactions: + - client-request: + method: GET + version: '1.1' + url: /resp/chunked-not-last + headers: + fields: + - [Host, example.com] + - [uuid, resp-chunked-not-last] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Transfer-Encoding, "chunked, gzip"] + - [Connection, close] + content: + size: 16 + + proxy-response: + status: 502 + +# Test 4: single chunked in response Transfer-Encoding — baseline +- transactions: + - client-request: + method: GET + version: '1.1' + url: /resp/chunked-only + headers: + fields: + - [Host, example.com] + - [uuid, resp-chunked-only] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Transfer-Encoding, chunked] + - [Connection, keep-alive] + content: + size: 16 + + proxy-response: + status: 200 + content: + size: 16 + +# --- Duplicate field tests --- + +# Test 5: chunked followed by gzip in separate request Transfer-Encoding fields +- transactions: + - client-request: + method: POST + version: '1.1' + url: /req/chunked-dup-not-last + headers: + fields: + - [Host, example.com] + - [Transfer-Encoding, chunked] + - [Transfer-Encoding, gzip] + - [uuid, req-chunked-dup-not-last] + content: + size: 0 + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "0"] + + proxy-response: + status: 501 + +# Test 6: chunked followed by gzip in separate response Transfer-Encoding fields +- transactions: + - client-request: + method: GET + version: '1.1' + url: /resp/chunked-dup-not-last + headers: + fields: + - [Host, example.com] + - [uuid, resp-chunked-dup-not-last] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Transfer-Encoding, chunked] + - [Transfer-Encoding, gzip] + - [Connection, close] + content: + size: 16 + + proxy-response: + status: 502 diff --git a/tests/gold_tests/chunked_encoding/replays/chunked_passthru_flow_control.replay.yaml b/tests/gold_tests/chunked_encoding/replays/chunked_passthru_flow_control.replay.yaml new file mode 100644 index 00000000000..e20d1ea3c37 --- /dev/null +++ b/tests/gold_tests/chunked_encoding/replays/chunked_passthru_flow_control.replay.yaml @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +# Regression test for the chunked-passthrough high-water flow control in +# HttpTunnel when chunked trailers are NOT dropped +# (proxy.config.http.drop_chunked_trailers = 0). +# +# A chunked HTTP/1.1 origin delivers a large response and an HTTP/1.1 client +# consumes it, so ATS proxies the body as-is on the +# TunnelChunkingAction_t::PASSTHRU_CHUNKED_CONTENT path. With trailers retained +# there is NO intermediate output buffer: the user-agent consumer reads +# p->read_buffer directly while the chunked parser's chunked_reader walks that +# same buffer only to track framing. +# +# With a small default_buffer_water_mark the read buffer crosses high_water +# before the consumer drains it. The buggy code threw the chunked high-water +# throttle on this path and returned WITHOUT walking the parser; because +# chunked_reader shares read_buffer with the consumer, the un-advanced +# chunked_reader pinned the buffer above high_water, the producer could never be +# re-enabled, and the transfer deadlocked until the inactivity timeout. The fix +# walks the parser before applying the throttle on this path, so chunked_reader +# stays drained and the high_water check reflects only the consumer's backlog. +# The response must complete with the full body and no inactivity-timeout stall. + +autest: + description: 'HTTP/1.1 chunked origin -> HTTP/1.1 chunked client passthru flow-control throttle (drop_chunked_trailers=0)' + + dns: + name: 'dns-chunked-passthru-fc' + + server: + name: 'server-chunked-passthru-fc' + + client: + name: 'client-chunked-passthru-fc' + + ats: + name: 'ts-chunked-passthru-fc' + process_config: + enable_tls: false + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http_tunnel' + # Retain chunked trailers so the chunked->chunked passthru has no + # intermediate buffer (the path that could deadlock). + proxy.config.http.drop_chunked_trailers: 0 + # Small water_mark so the read buffer crosses high_water well before the + # user-agent socket can drain it, forcing the throttle. + proxy.config.http.default_buffer_water_mark: 4096 + # Keep the transaction inactivity timeouts short so that, if the tunnel + # regresses to the deadlock, the failure surfaces quickly rather than + # hanging for the 30s default. A healthy transfer completes well under + # this bound. + proxy.config.http.transaction_no_activity_timeout_in: 10 + proxy.config.http.transaction_no_activity_timeout_out: 10 + + remap_config: + - from: "http://www.example.com/" + to: "http://backend.example.com:{SERVER_HTTP_PORT}/" + + log_validation: + diags_log: + excludes: + # Indicator of the tunnel deadlocking until the inactivity timeout. + - expression: 'ERROR.*inactivity timeout' + description: 'Tunnel must not stall to inactivity timeout.' + + metric_checks: + # The chunked-tunnel throttle must fire at least once: the read buffer + # crosses high_water before the consumer drains it. Combined with the + # proxy-response size assertion below (which only passes if the producer + # was re-enabled and the full body proxied), this covers both the throttle + # and the unwind paths. + - metric: proxy.process.http.tunnel.chunked_throttle + value: '[1-9][0-9]*' + +sessions: + +- protocol: + stack: http + + transactions: + + # A sizable chunked response proxied chunked->chunked. The body is large + # enough to exceed water_mark many times over. + - client-request: + method: GET + version: "1.1" + url: /large-passthru + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, large-passthru-1 ] + + server-response: + status: 200 + reason: OK + version: "1.1" + headers: + fields: + - [ "Content-Type", "text/plain" ] + - [ "Transfer-Encoding", "chunked" ] + content: + size: 524288 # 512 KB + + proxy-response: + status: 200 + headers: + fields: + - [ "Transfer-Encoding", { value: "chunked", as: equal } ] + content: + size: { value: 524288, as: equal } + + # A second transaction back-to-back, exercising the re-enable path after the + # first throttle cycle. + - client-request: + delay: 100ms + method: GET + version: "1.1" + url: /large-passthru-2 + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, large-passthru-2 ] + + server-response: + status: 200 + reason: OK + version: "1.1" + headers: + fields: + - [ "Content-Type", "text/plain" ] + - [ "Transfer-Encoding", "chunked" ] + content: + size: 262144 # 256 KB + + proxy-response: + status: 200 + headers: + fields: + - [ "Transfer-Encoding", { value: "chunked", as: equal } ] + content: + size: { value: 262144, as: equal } diff --git a/tests/gold_tests/connect/connect.test.py b/tests/gold_tests/connect/connect.test.py index 37ea4db84f5..633d3f351fb 100644 --- a/tests/gold_tests/connect/connect.test.py +++ b/tests/gold_tests/connect/connect.test.py @@ -56,6 +56,7 @@ def __setupTS(self): self.ts.Disk.remap_config.AddLines([ f"map http://foo.com/ http://127.0.0.1:{self.httpbin.Variables.Port}/", ]) + self.ts.addPrivateConnectAllowYaml() self.ts.Disk.logging_yaml.AddLines( ''' @@ -142,6 +143,7 @@ def setupTS(self): self.ts.Disk.remap_config.AddLines([ f"map / http://127.0.0.1:{self.server.Variables.http_port}/", ]) + self.ts.addPrivateConnectAllowYaml() # Verify ts logs self.ts.Disk.traffic_out.Content += Testers.ContainsExpression( f"Proxy's Request.*\n.*\nCONNECT 127.0.0.1:{self.server.Variables.http_port} HTTP/1.1", @@ -227,6 +229,7 @@ def setupTS(self): self.ts.Disk.remap_config.AddLines([ f"map / http://127.0.0.1:{self.server.Variables.http_port}/", ]) + self.ts.addPrivateConnectAllowYaml() # Verify ts logs self.ts.Disk.traffic_out.Content += Testers.ContainsExpression( f"Proxy's Request.*\n.*\nCONNECT 127.0.0.1:{self.server.Variables.http_port} HTTP/1.1", diff --git a/tests/gold_tests/connect/connect_parent_error_body.test.py b/tests/gold_tests/connect/connect_parent_error_body.test.py new file mode 100644 index 00000000000..8ac78d9158b --- /dev/null +++ b/tests/gold_tests/connect/connect_parent_error_body.test.py @@ -0,0 +1,72 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = 'Verify CONNECT error response bodies from parent proxies' +Test.ContinueOnFail = True + + +class ConnectParentErrorBodyTest: + replay_file = 'replays/connect_parent_error_body.replay.yaml' + + def __init__(self): + self._setupParentProxy() + self._setupTS() + + def _setupParentProxy(self): + self.parent = Test.MakeVerifierServerProcess('parent-proxy', self.replay_file) + self.parent.Streams.stdout += Testers.ContainsExpression( + 'CONNECT www.example.com:443 HTTP/1.1', 'Verify that ATS forwards the CONNECT request to the parent proxy.') + self.parent.Streams.stdout += Testers.ContainsExpression( + 'GET http://www.example.com/next HTTP/1.1', 'Verify that ATS reuses the parent connection for the next request.') + # Each no-DNS-forward request must reach the parent for that request's + # host. The parent receiving the third request with its own Host + # header confirms ATS did not short-circuit to a destination derived + # from earlier State (the dns_info.addr stale-data case). + self.parent.Streams.stdout += Testers.ContainsExpression( + 'GET http://other.example.org/page HTTP/1.1', + 'Verify that ATS forwards a different host on the same session to the parent.') + + def _setupTS(self): + self.ts = Test.MakeATSProcess('ts', enable_cache=False) + + self.ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|parent', + 'proxy.config.http.connect_ports': '443', + 'proxy.config.http.no_dns_just_forward_to_parent': 1, + 'proxy.config.http.parent_proxy.self_detect': 0, + 'proxy.config.http.server_ports': f'{self.ts.Variables.port}', + 'proxy.config.http.server_session_sharing.pool': 'global', + 'proxy.config.url_remap.remap_required': 0, + }) + + self.ts.Disk.parent_config.AddLine( + f'dest_domain=. parent="127.0.0.1:{self.parent.Variables.http_port}|1" go_direct=false parent_is_proxy=true') + self.ts.addPrivateConnectAllowYaml(methods='[ CONNECT, GET ]') + + def run(self): + tr = Test.AddTestRun('CONNECT error response body from parent proxy') + tr.AddVerifierClientProcess('client', self.replay_file, http_ports=[self.ts.Variables.port]) + tr.Processes.Default.StartBefore(self.parent) + tr.Processes.Default.StartBefore(self.ts) + tr.StillRunningAfter = self.parent + tr.StillRunningAfter = self.ts + + +ConnectParentErrorBodyTest().run() diff --git a/tests/gold_tests/connect/h2_malformed_request_logging.test.py b/tests/gold_tests/connect/h2_malformed_request_logging.test.py index 95ff7952210..188485b1bea 100644 --- a/tests/gold_tests/connect/h2_malformed_request_logging.test.py +++ b/tests/gold_tests/connect/h2_malformed_request_logging.test.py @@ -104,6 +104,7 @@ def _setup_ts(self): 'proxy.config.http.connect_ports': self._server.Variables.http_port, }) self._ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.http_port}/') + self._ts.addPrivateConnectAllowYaml(methods='[ CONNECT, GET ]') self._ts.Disk.logging_yaml.AddLines( """ logging: diff --git a/tests/gold_tests/connect/malformed_h2_request_client.py b/tests/gold_tests/connect/malformed_h2_request_client.py index c60178287c0..1266cf69f4d 100644 --- a/tests/gold_tests/connect/malformed_h2_request_client.py +++ b/tests/gold_tests/connect/malformed_h2_request_client.py @@ -118,23 +118,77 @@ def make_malformed_headers(scenario: str) -> bytes: ("user-agent", "Malformed/1.0"), ("uuid", "malformed-get-connection"), ] + elif scenario == "crlf-in-header-value": + headers = [ + (":method", "GET"), + (":scheme", "https"), + (":authority", "crlf-value.example"), + (":path", "/crlf-value"), + ("x-injected", "safe\r\ninjected: evil"), + ("uuid", "malformed-crlf-value"), + ] + elif scenario == "cr-in-header-value": + headers = [ + (":method", "GET"), + (":scheme", "https"), + (":authority", "cr-value.example"), + (":path", "/cr-value"), + ("x-injected", "before\rafter"), + ("uuid", "malformed-cr-value"), + ] + elif scenario == "lf-in-header-value": + headers = [ + (":method", "GET"), + (":scheme", "https"), + (":authority", "lf-value.example"), + (":path", "/lf-value"), + ("x-injected", "before\nafter"), + ("uuid", "malformed-lf-value"), + ] + elif scenario == "nul-in-header-value": + headers = [ + (":method", "GET"), + (":scheme", "https"), + (":authority", "nul-value.example"), + (":path", "/nul-value"), + ("x-injected", "before\0after"), + ("uuid", "malformed-nul-value"), + ] else: raise ValueError(f"unknown scenario: {scenario}") return encoder.encode(headers) +SCENARIOS_EXPECTING_ERROR_RESPONSE = { + "crlf-in-header-value", + "cr-in-header-value", + "lf-in-header-value", + "nul-in-header-value", +} + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("port", type=int, help="TLS port to connect to") parser.add_argument( "scenario", - choices=("connect-missing-authority", "get-missing-path", "get-connection-header"), + choices=( + "connect-missing-authority", + "get-missing-path", + "get-connection-header", + "crlf-in-header-value", + "cr-in-header-value", + "lf-in-header-value", + "nul-in-header-value", + ), help="Malformed request shape to send", ) args = parser.parse_args() tls_socket = connect_socket(args.port) + decoder = hpack.Decoder() + expect_error_response = args.scenario in SCENARIOS_EXPECTING_ERROR_RESPONSE try: payload = make_malformed_headers(args.scenario) tls_socket.sendall(H2_PREFACE) @@ -161,6 +215,13 @@ def main() -> int: error_code = int.from_bytes(frame["payload"][4:8], "big") print(f"Received GOAWAY with error code {error_code}") return 0 if error_code in (0, PROTOCOL_ERROR) else 1 + + if frame_type == TYPE_HEADERS and frame["stream_id"] == 1 and expect_error_response: + headers = decoder.decode(frame["payload"]) + status = dict(headers).get(":status", "") + status_code = int(status) if status else 0 + print(f"Received HTTP/2 response with status {status_code}") + return 0 if 400 <= status_code < 500 else 1 except socket.timeout: print(f"Timed out waiting for ATS to reject malformed request scenario {args.scenario}", file=sys.stderr) return 1 diff --git a/tests/gold_tests/connect/replays/connect_parent_error_body.replay.yaml b/tests/gold_tests/connect/replays/connect_parent_error_body.replay.yaml new file mode 100644 index 00000000000..8a6b436df1d --- /dev/null +++ b/tests/gold_tests/connect/replays/connect_parent_error_body.replay.yaml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# ATS sends these requests to a parent proxy. The CONNECT error response has a +# body and keeps the parent connection alive. ATS must consume that body before +# reusing the parent connection for the follow-up request. +# +meta: + version: '1.0' + +sessions: +- transactions: + - client-request: + method: CONNECT + version: '1.1' + url: www.example.com:443 + headers: + fields: + - [ Host, www.example.com:443 ] + - [ Proxy-Connection, keep-alive ] + - [ uuid, connect-error ] + + server-response: + status: 403 + reason: Forbidden + headers: + fields: + - [ Content-Length, 18 ] + - [ Connection, keep-alive ] + content: + encoding: plain + data: "connect forbidden\n" + + proxy-response: + status: 403 + content: + encoding: plain + data: "connect forbidden\n" + verify: { as: equal } + + - client-request: + delay: 100ms + method: GET + version: '1.1' + url: http://www.example.com/next + headers: + fields: + - [ Host, www.example.com ] + - [ Proxy-Connection, keep-alive ] + - [ uuid, follow-up ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 19 ] + - [ Connection, keep-alive ] + content: + encoding: plain + data: "follow-up response\n" + + proxy-response: + status: 200 + content: + encoding: plain + data: "follow-up response\n" + verify: { as: equal } + + # Third transaction with a different host. Each no-DNS-forward request + # must reach the parent for that request's host. If a stale dns_info.addr + # from an earlier transaction were reused, this request would either + # short-circuit to the wrong destination or fail to reach the parent. + - client-request: + delay: 100ms + method: GET + version: '1.1' + url: http://other.example.org/page + headers: + fields: + - [ Host, other.example.org ] + - [ Proxy-Connection, close ] + - [ uuid, third-host ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 20 ] + - [ Connection, close ] + content: + encoding: plain + data: "third host response\n" + + proxy-response: + status: 200 + content: + encoding: plain + data: "third host response\n" + verify: { as: equal } diff --git a/tests/gold_tests/cripts/cripts_bundle.replay.yaml b/tests/gold_tests/cripts/cripts_bundle.replay.yaml new file mode 100644 index 00000000000..dc49550b2b4 --- /dev/null +++ b/tests/gold_tests/cripts/cripts_bundle.replay.yaml @@ -0,0 +1,325 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +# Configuration for autest integration. This replay file is the harness for +# a bundle of cripts tests — bundle_headers.cript exercising cripts::Bundle::Headers +# and direct_headers.cript exercising the equivalent direct Cripts API, with room +# to add more cripts and more transactions under the same driver. Multiple parallel +# client sessions widen the +# exposure surface; under a TSAN build this is the regression target for +# any future bug that puts shared mutable state back into a Cripts plugin +# instance. +autest: + description: 'cripts: bundle of feature tests (bundle_headers.cript: every Bundle::Headers dynamic substitution across all four hooks; direct_headers.cript: the equivalent direct Cripts API path)' + + dns: + name: 'dns' + + server: + name: 'server' + + client: + name: 'client' + + ats: + name: 'ts' + + process_config: + enable_cripts: true + enable_cache: false + + copy_to_config_dir: + - 'files/bundle_headers.cript' + - 'files/direct_headers.cript' + - 'files/server_bundle_headers.cript' + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'cript|plugin' + proxy.config.plugin.dynamic_reload_mode: 1 + + remap_config: + - from: "http://www.example.com/" + to: "http://backend.ex:{SERVER_HTTP_PORT}/" + plugins: + - name: "bundle_headers.cript" + args: [] + - from: "http://direct.example.com/" + to: "http://backend.ex:{SERVER_HTTP_PORT}/" + plugins: + - name: "direct_headers.cript" + args: [] + - from: "http://server.example.com/" + to: "http://backend.ex:{SERVER_HTTP_PORT}/" + plugins: + - name: "server_bundle_headers.cript" + args: [] + + log_validation: + diags_log: + excludes: + - expression: "FATAL" + description: "No fatal cript or plugin errors" + - expression: "Failed to load plugin" + description: "all cripts under files/ should compile and load" + +# Sessions are independent connections; Proxy Verifier dispatches them in +# parallel by default, which is what we want for exposing any latent +# Instance-shared mutable state in the bundle. +sessions: + +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /alpha?q=1 + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, sess-1 ] + # rm_headers("Client::Request") should drop this before it reaches origin. + - [ X-Cript-Strip-CReq, "should-be-removed" ] + + proxy-request: + headers: + fields: + - [ X-Cript-Req-UUID, { as: present } ] + - [ X-Cript-Req-TxnID, { as: present } ] + - [ X-Cript-Req-ProcID, { as: present } ] + - [ X-Cript-Req-CIP, { as: present } ] + - [ X-Cript-Req-IIP, { as: present } ] + - [ X-Cript-Req-CIDR, { as: present } ] + - [ X-Cript-Req-Host, { as: present } ] + - [ X-Cript-Req-Path, { value: "alpha", as: equal } ] + - [ X-Cript-Req-Port, { as: present } ] + - [ X-Cript-Req-Query, { value: "q=1", as: equal } ] + - [ X-Cript-Req-Scheme, { value: "http", as: equal } ] + - [ X-Cript-SReq-UUID, { as: present } ] + - [ X-Cript-SReq-CIP, { as: present } ] + - [ X-Cript-SReq-Path, { value: "alpha", as: equal } ] + - [ X-Cript-Strip-CReq, { as: absent } ] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [ Content-Length, 0 ] + # rm_headers("Client::Response") should drop this from the response + # the client ultimately sees. + - [ X-Cript-Strip-CResp, "should-be-removed" ] + + proxy-response: + headers: + fields: + - [ X-Cript-Resp-UUID, { as: present } ] + - [ X-Cript-Resp-ProcID, { as: present } ] + - [ X-Cript-Resp-CIP, { as: present } ] + - [ X-Cript-Resp-Host, { as: present } ] + - [ X-Cript-Resp-Path, { value: "alpha", as: equal } ] + - [ X-Cript-Strip-CResp, { as: absent } ] + +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /beta/gamma?x=y&a=b + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, sess-2 ] + + proxy-request: + headers: + fields: + - [ X-Cript-Req-UUID, { as: present } ] + - [ X-Cript-Req-Path, { value: "beta/gamma", as: equal } ] + - [ X-Cript-Req-Query, { value: "x=y&a=b", as: equal } ] + - [ X-Cript-SReq-Path, { value: "beta/gamma", as: equal } ] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [ Content-Length, 0 ] + + proxy-response: + headers: + fields: + - [ X-Cript-Resp-UUID, { as: present } ] + - [ X-Cript-Resp-Path, { value: "beta/gamma", as: equal } ] + +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /delta + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, sess-3 ] + + proxy-request: + headers: + fields: + - [ X-Cript-Req-UUID, { as: present } ] + - [ X-Cript-Req-Path, { value: "delta", as: equal } ] + - [ X-Cript-Req-CIP, { as: present } ] + - [ X-Cript-Req-CIDR, { as: present } ] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [ Content-Length, 0 ] + + proxy-response: + headers: + fields: + - [ X-Cript-Resp-UUID, { as: present } ] + - [ X-Cript-Resp-Path, { value: "delta", as: equal } ] + +# Sessions for direct_headers.cript — same data sources, no HRWBridge. + +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /epsilon?k=v + headers: + fields: + - [ Host, direct.example.com ] + - [ uuid, sess-direct-1 ] + + proxy-request: + headers: + fields: + - [ X-Direct-Req-UUID, { as: present } ] + - [ X-Direct-Req-Host, { as: present } ] + - [ X-Direct-Req-Path, { value: "epsilon", as: equal } ] + - [ X-Direct-Req-Query, { value: "k=v", as: equal } ] + - [ X-Direct-Req-Scheme, { value: "http", as: equal } ] + - [ X-Direct-Req-Port, { as: present } ] + - [ X-Direct-Req-CIP, { as: present } ] + - [ X-Direct-SReq-UUID, { as: present } ] + - [ X-Direct-SReq-Path, { value: "epsilon", as: equal } ] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [ Content-Length, 0 ] + + proxy-response: + headers: + fields: + - [ X-Direct-Resp-UUID, { as: present } ] + - [ X-Direct-Resp-ProcID, { as: present } ] + - [ X-Direct-Resp-Host, { as: present } ] + - [ X-Direct-Resp-Path, { value: "epsilon", as: equal } ] + - [ X-Direct-Resp-CIP, { as: present } ] + +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /zeta/eta + headers: + fields: + - [ Host, direct.example.com ] + - [ uuid, sess-direct-2 ] + + proxy-request: + headers: + fields: + - [ X-Direct-Req-UUID, { as: present } ] + - [ X-Direct-Req-Path, { value: "zeta/eta", as: equal } ] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [ Content-Length, 0 ] + + proxy-response: + headers: + fields: + - [ X-Direct-Resp-UUID, { as: present } ] + - [ X-Direct-Resp-Path, { value: "zeta/eta", as: equal } ] + +# server_bundle_headers.cript: Bundle::Headers on the server hooks. rm_headers +# drops the listed headers (server strips must not be no-ops), +# set_headers adds X-Cript-Set-* and overwrites X-Cript-Over-*, and X-Cript-Keep-* +# must pass through untouched. + +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /theta + headers: + fields: + - [ Host, server.example.com ] + - [ uuid, sess-server-1 ] + # rm_headers("Server::Request") must drop these before they reach origin. + - [ Authorization, "Bearer should-be-removed" ] + - [ Cookie, "sid=should-be-removed" ] + - [ X-Cript-Strip-SReq, "should-be-removed" ] + # set_headers("Server::Request") must overwrite this value. + - [ X-Cript-Over-SReq, "original-sreq" ] + # Not touched by the cript — must reach origin unchanged. + - [ X-Cript-Keep-SReq, "should-survive" ] + + proxy-request: + headers: + fields: + - [ Authorization, { as: absent } ] + - [ Cookie, { as: absent } ] + - [ X-Cript-Strip-SReq, { as: absent } ] + - [ X-Cript-Set-SReq, { value: "set-sreq", as: equal } ] + - [ X-Cript-Over-SReq, { value: "over-sreq", as: equal } ] + - [ X-Cript-Keep-SReq, { value: "should-survive", as: equal } ] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [ Content-Length, 0 ] + # rm_headers("Server::Response") must drop these before the client sees them. + - [ Set-Cookie, "sid=should-be-removed" ] + - [ X-Cript-Strip-SResp, "should-be-removed" ] + # set_headers("Server::Response") must overwrite this value. + - [ X-Cript-Over-SResp, "original-sresp" ] + # Not touched by the cript — must reach the client unchanged. + - [ X-Cript-Keep-SResp, "should-survive" ] + + proxy-response: + headers: + fields: + - [ Set-Cookie, { as: absent } ] + - [ X-Cript-Strip-SResp, { as: absent } ] + - [ X-Cript-Set-SResp, { value: "set-sresp", as: equal } ] + - [ X-Cript-Over-SResp, { value: "over-sresp", as: equal } ] + - [ X-Cript-Keep-SResp, { value: "should-survive", as: equal } ] + diff --git a/tests/gold_tests/cripts/cripts_bundle.test.py b/tests/gold_tests/cripts/cripts_bundle.test.py new file mode 100644 index 00000000000..8b5630b162d --- /dev/null +++ b/tests/gold_tests/cripts/cripts_bundle.test.py @@ -0,0 +1,49 @@ +''' +Bundle of cripts feature tests, driven by one replay-file harness. Each +.cript under files/ targets a distinct cripts feature; the replay file +configures one remap rule per cript and a transaction set per feature. + +Currently bundled: + - bundle_headers.cript : exercises cripts::Bundle::Headers across all + four hooks with every dynamic substitution + source (ID/IP/CIDR/CLIENT-URL components), + via the HRWBridge / %{...} syntax. + - direct_headers.cript : same data sources but via the direct cripts + API (UUID/IP/Url accessors), bypassing + Bundle::Headers entirely. + - server_bundle_headers.cript : Bundle::Headers on the two server hooks + — rm_headers (incl. Authorization/Cookie/ + Set-Cookie) plus set_headers with literal + values (add + overwrite). Server-only so it + also pins the regression where rm_headers() + misrouted both server targets into the + client-response list and silently dropped the + strips. + +Concurrent client sessions widen the exposure surface; under a TSAN +build this is the regression target for any bug that puts shared mutable +state back into a Cripts plugin instance. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +cripts: bundle of feature tests (bundle_headers.cript + direct_headers.cript) +''' + +Test.SkipUnless(Condition.HasATSFeature('TS_HAS_CRIPTS')) + +Test.ATSReplayTest(replay_file="cripts_bundle.replay.yaml") diff --git a/tests/gold_tests/cripts/files/bundle_headers.cript b/tests/gold_tests/cripts/files/bundle_headers.cript new file mode 100644 index 00000000000..fdb20b4a512 --- /dev/null +++ b/tests/gold_tests/cripts/files/bundle_headers.cript @@ -0,0 +1,60 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Exercises cripts::Bundle::Headers across all four header hooks with every +// dynamic substitution source the bundle's HRWBridge supports. Used as a +// regression target for the bundle's dynamic-value plumbing — any bridge +// subclass that depends on shared mutable state will surface here under +// concurrent traffic (especially when run against a TSAN build). + +#include +#include + +do_create_instance() +{ + cripts::Bundle::Headers::Activate() + .rm_headers("Client::Request", + { + "X-Cript-Strip-CReq" + }) + .rm_headers("Server::Request", {"X-Cript-Strip-SReq"}) + .rm_headers("Server::Response", {"X-Cript-Strip-SResp"}) + .rm_headers("Client::Response", {"X-Cript-Strip-CResp"}) + .set_headers("Client::Request", {{"X-Cript-Req-UUID", "%{ID:UNIQUE}"}, + {"X-Cript-Req-TxnID", "%{ID:REQUEST}"}, + {"X-Cript-Req-ProcID", "%{ID:PROCESS}"}, + {"X-Cript-Req-CIP", "%{IP:CLIENT}"}, + {"X-Cript-Req-IIP", "%{IP:INBOUND}"}, + {"X-Cript-Req-CIDR", "%{CIDR:24,64}"}, + {"X-Cript-Req-Host", "%{CLIENT-URL:HOST}"}, + {"X-Cript-Req-Path", "%{CLIENT-URL:PATH}"}, + {"X-Cript-Req-Port", "%{CLIENT-URL:PORT}"}, + {"X-Cript-Req-Query", "%{CLIENT-URL:QUERY}"}, + {"X-Cript-Req-Scheme", "%{CLIENT-URL:SCHEME}"}}) + .set_headers( + "Server::Request", + {{"X-Cript-SReq-UUID", "%{ID:UNIQUE}"}, {"X-Cript-SReq-CIP", "%{IP:CLIENT}"}, {"X-Cript-SReq-Path", "%{CLIENT-URL:PATH}"}}) + .set_headers("Server::Response", {{"X-Cript-SResp-UUID", "%{ID:UNIQUE}"}, {"X-Cript-SResp-CIP", "%{IP:CLIENT}"}}) + .set_headers("Client::Response", {{"X-Cript-Resp-UUID", "%{ID:UNIQUE}"}, + {"X-Cript-Resp-ProcID", "%{ID:PROCESS}"}, + {"X-Cript-Resp-CIP", "%{IP:CLIENT}"}, + {"X-Cript-Resp-Host", "%{CLIENT-URL:HOST}"}, + {"X-Cript-Resp-Path", "%{CLIENT-URL:PATH}"}}); +} + +#include diff --git a/tests/gold_tests/cripts/files/direct_headers.cript b/tests/gold_tests/cripts/files/direct_headers.cript new file mode 100644 index 00000000000..bebc6ba7d26 --- /dev/null +++ b/tests/gold_tests/cripts/files/direct_headers.cript @@ -0,0 +1,65 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Sets the same kinds of dynamic header values as bundle_headers.cript but +// via the direct cripts API (UUID/IP/Url accessors), without going through +// cripts::Bundle::Headers / HRWBridge. Pairs with the bundle test to give +// coverage of both header-substitution code paths. + +#include + +do_remap() +{ + borrow req = cripts::Client::Request::Get(); + borrow url = cripts::Client::URL::Get(); + borrow conn = cripts::Client::Connection::Get(); + auto ip = conn.IP(); + + req["X-Direct-Req-UUID"] = cripts::UUID::Unique::_get(context); + req["X-Direct-Req-Host"] = url.host; + req["X-Direct-Req-Path"] = url.path; + req["X-Direct-Req-Query"] = url.query; + req["X-Direct-Req-Scheme"] = url.scheme; + req["X-Direct-Req-Port"] = url.port; // integer overload + req["X-Direct-Req-CIP"] = ip.string(); +} + +do_send_request() +{ + borrow sreq = cripts::Server::Request::Get(); + borrow url = cripts::Client::URL::Get(); + + sreq["X-Direct-SReq-UUID"] = cripts::UUID::Unique::_get(context); + sreq["X-Direct-SReq-Path"] = url.path; +} + +do_send_response() +{ + borrow resp = cripts::Client::Response::Get(); + borrow url = cripts::Client::URL::Get(); + borrow conn = cripts::Client::Connection::Get(); + auto ip = conn.IP(); + + resp["X-Direct-Resp-UUID"] = cripts::UUID::Unique::_get(context); + resp["X-Direct-Resp-ProcID"] = cripts::UUID::Process::_get(context); + resp["X-Direct-Resp-Host"] = url.host; + resp["X-Direct-Resp-Path"] = url.path; + resp["X-Direct-Resp-CIP"] = ip.string(); +} + +#include diff --git a/tests/gold_tests/cripts/files/server_bundle_headers.cript b/tests/gold_tests/cripts/files/server_bundle_headers.cript new file mode 100644 index 00000000000..6a3fad840a3 --- /dev/null +++ b/tests/gold_tests/cripts/files/server_bundle_headers.cript @@ -0,0 +1,45 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Bundle::Headers coverage on the two server hooks: rm_headers (including the +// Authorization/Cookie/Set-Cookie an operator means to scrub) plus set_headers +// with literal values — both adding a new header and overwriting an existing +// one. Complements bundle_headers.cript, which sets dynamic %{...} values +// across all four hooks. +// +// Also the regression for the server rm_headers misrouting to the +// client-response list. Deliberately no Client::Response op here: either a +// strip or a set on that target registers DO_SEND_RESPONSE, which would flush +// the misrouted entries and mask a Server::Response bug. Keep this server-only. + +#include +#include + +do_create_instance() +{ + cripts::Bundle::Headers::Activate() + .rm_headers("Server::Request", + { + "Authorization", "Cookie", "X-Cript-Strip-SReq" + }) + .set_headers("Server::Request", {{"X-Cript-Set-SReq", "set-sreq"}, {"X-Cript-Over-SReq", "over-sreq"}}) + .rm_headers("Server::Response", {"Set-Cookie", "X-Cript-Strip-SResp"}) + .set_headers("Server::Response", {{"X-Cript-Set-SResp", "set-sresp"}, {"X-Cript-Over-SResp", "over-sresp"}}); +} + +#include diff --git a/tests/gold_tests/h2/clients/h2_max_settings_per_minute.py b/tests/gold_tests/h2/clients/h2_max_settings_per_minute.py new file mode 100644 index 00000000000..ef906e7daee --- /dev/null +++ b/tests/gold_tests/h2/clients/h2_max_settings_per_minute.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Exercise the HTTP/2 max_settings_per_minute guard. + +This client sends one SETTINGS frame containing two parameters. The matching +AuTest config disables the per-frame settings limit and sets +max_settings_per_minute to one, so ATS should reject the connection with +ENHANCE_YOUR_CALM. +""" + +import argparse +import socket +import ssl +import sys + +H2_PREFACE = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + +TYPE_SETTINGS = 0x04 +TYPE_GOAWAY = 0x07 + +FLAG_ACK = 0x01 + +SETTINGS_HEADER_TABLE_SIZE = 0x01 +SETTINGS_MAX_CONCURRENT_STREAMS = 0x03 + +ENHANCE_YOUR_CALM = 0x0B + + +def make_frame(frame_type: int, flags: int, stream_id: int, payload: bytes = b"") -> bytes: + return len(payload).to_bytes(3, "big") + bytes([frame_type, flags]) + (stream_id & 0x7FFFFFFF).to_bytes(4, "big") + payload + + +def make_setting(setting_id: int, value: int) -> bytes: + return setting_id.to_bytes(2, "big") + value.to_bytes(4, "big") + + +def recv_exact(sock: socket.socket, size: int) -> bytes: + data = bytearray() + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + break + data.extend(chunk) + return bytes(data) + + +def read_frame(sock: socket.socket): + header = recv_exact(sock, 9) + if len(header) == 0: + return None + if len(header) != 9: + raise RuntimeError(f"incomplete frame header: got {len(header)} bytes") + + length = int.from_bytes(header[0:3], "big") + payload = recv_exact(sock, length) + if len(payload) != length: + raise RuntimeError(f"incomplete frame payload: expected {length}, got {len(payload)}") + + return { + "length": length, + "type": header[3], + "flags": header[4], + "stream_id": int.from_bytes(header[5:9], "big") & 0x7FFFFFFF, + "payload": payload, + } + + +def connect_socket(port: int) -> socket.socket: + socket.setdefaulttimeout(5) + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.set_alpn_protocols(["h2"]) + + tls_socket = socket.create_connection(("127.0.0.1", port)) + tls_socket = ctx.wrap_socket(tls_socket, server_hostname="localhost") + if tls_socket.selected_alpn_protocol() != "h2": + raise RuntimeError(f"failed to negotiate h2, got {tls_socket.selected_alpn_protocol()!r}") + return tls_socket + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("port", type=int, help="TLS port to connect to") + args = parser.parse_args() + + settings_payload = make_setting(SETTINGS_HEADER_TABLE_SIZE, 4096) + make_setting(SETTINGS_MAX_CONCURRENT_STREAMS, 100) + + tls_socket = connect_socket(args.port) + try: + tls_socket.sendall(H2_PREFACE) + tls_socket.sendall(make_frame(TYPE_SETTINGS, 0, 0, settings_payload)) + + while True: + frame = read_frame(tls_socket) + if frame is None: + print("Connection closed before receiving GOAWAY", file=sys.stderr) + return 1 + + frame_type = frame["type"] + if frame_type == TYPE_SETTINGS and not (frame["flags"] & FLAG_ACK): + tls_socket.sendall(make_frame(TYPE_SETTINGS, FLAG_ACK, 0)) + continue + + if frame_type == TYPE_GOAWAY: + error_code = int.from_bytes(frame["payload"][4:8], "big") + print(f"Received GOAWAY with error code {error_code}") + return 0 if error_code == ENHANCE_YOUR_CALM else 1 + except socket.timeout: + print("Timed out waiting for max_settings_per_minute GOAWAY", file=sys.stderr) + return 1 + finally: + tls_socket.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/gold_tests/h2/clients/h2_settings_ack_stall.py b/tests/gold_tests/h2/clients/h2_settings_ack_stall.py new file mode 100644 index 00000000000..1a340c0c41c --- /dev/null +++ b/tests/gold_tests/h2/clients/h2_settings_ack_stall.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +''' +HTTP/2 client that withholds SETTINGS ACKs while opening streams. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import socket +import ssl +import time +from typing import Optional, Tuple + +CONNECTION_PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' + +FRAME_TYPE_DATA = 0 +FRAME_TYPE_HEADERS = 1 +FRAME_TYPE_RST_STREAM = 3 +FRAME_TYPE_SETTINGS = 4 +FRAME_TYPE_GOAWAY = 7 + +FLAG_END_STREAM = 0x01 +FLAG_END_HEADERS = 0x04 + +ERROR_SETTINGS_TIMEOUT = 4 + + +def make_socket(port: int) -> ssl.SSLSocket: + """Create a TLS-wrapped HTTP/2 socket.""" + + socket.setdefaulttimeout(15) + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.set_alpn_protocols(['h2']) + + raw_socket = socket.create_connection(('localhost', port)) + tls_socket = ctx.wrap_socket(raw_socket, server_hostname='localhost') + negotiated = tls_socket.selected_alpn_protocol() + if negotiated != 'h2': + raise RuntimeError(f'Expected h2 ALPN, negotiated {negotiated!r}') + return tls_socket + + +def make_frame(frame_type: int, flags: int = 0, stream_id: int = 0, payload: bytes = b'') -> bytes: + """Serialize a minimal HTTP/2 frame.""" + + return (len(payload).to_bytes(3, 'big') + bytes([frame_type, flags]) + (stream_id & 0x7fffffff).to_bytes(4, 'big') + payload) + + +def read_exact(sock: ssl.SSLSocket, size: int) -> bytes: + """Read exactly @a size bytes or raise EOFError.""" + + chunks = [] + remaining = size + while remaining > 0: + chunk = sock.recv(remaining) + if not chunk: + raise EOFError('socket closed') + chunks.append(chunk) + remaining -= len(chunk) + return b''.join(chunks) + + +def read_frame(sock: ssl.SSLSocket) -> Tuple[int, int, int, bytes]: + """Read and parse one HTTP/2 frame.""" + + header = read_exact(sock, 9) + length = int.from_bytes(header[0:3], 'big') + frame_type = header[3] + flags = header[4] + stream_id = int.from_bytes(header[5:9], 'big') & 0x7fffffff + payload = read_exact(sock, length) + return frame_type, flags, stream_id, payload + + +def hpack_string(value: str) -> bytes: + """Encode a short, non-Huffman HPACK string literal.""" + + encoded = value.encode('utf-8') + if len(encoded) >= 128: + raise ValueError('test header value is too long for this helper') + return bytes([len(encoded)]) + encoded + + +def hpack_literal_header(name: str, value: str) -> bytes: + """Encode a non-indexed HPACK literal header with a new name.""" + + return b'\x00' + hpack_string(name) + hpack_string(value) + + +def request_header_block(path: str, stream_id: int) -> bytes: + """Build a tiny HPACK request block without using automatic SETTINGS ACKs.""" + + block = bytearray() + block.append(0x82) # :method: GET + block.append(0x87) # :scheme: https + block.append(0x01) # :authority literal without indexing, indexed name 1 + block.extend(hpack_string('localhost')) + if path == '/': + block.append(0x84) # :path: / + else: + block.append(0x04) # :path literal without indexing, indexed name 4 + block.extend(hpack_string(path)) + block.extend(hpack_literal_header('uuid', f'settings-ack-stall-{stream_id}')) + return bytes(block) + + +def send_request(sock: ssl.SSLSocket, stream_id: int) -> None: + """Send one GET request with END_STREAM set.""" + + flags = FLAG_END_HEADERS | FLAG_END_STREAM + path = f'/stream/{stream_id}' + sock.sendall(make_frame(FRAME_TYPE_HEADERS, flags, stream_id, request_header_block(path, stream_id))) + + +def read_until_streams_end(sock: ssl.SSLSocket, stream_ids: set[int]) -> Optional[int]: + """Read frames until @a stream_ids end, returning a GOAWAY error if seen.""" + + ended: set[int] = set() + while ended != stream_ids: + frame_type, flags, stream_id, payload = read_frame(sock) + if frame_type == FRAME_TYPE_GOAWAY: + error_code = int.from_bytes(payload[4:8], 'big') + print(f'GOAWAY error_code={error_code}') + return error_code + if stream_id in stream_ids: + if frame_type in (FRAME_TYPE_DATA, FRAME_TYPE_HEADERS) and flags & FLAG_END_STREAM: + ended.add(stream_id) + elif frame_type == FRAME_TYPE_RST_STREAM: + ended.add(stream_id) + elif frame_type == FRAME_TYPE_SETTINGS: + # Deliberately do nothing. This test is about withholding SETTINGS + # ACKs while continuing to read the connection. + pass + return None + + +def run(port: int) -> int: + """Open enough stream waves to exhaust ATS's outstanding SETTINGS cap.""" + + with make_socket(port) as sock: + sock.sendall(CONNECTION_PREFACE) + sock.sendall(make_frame(FRAME_TYPE_SETTINGS)) + + next_stream_id = 1 + for _ in range(5): + stream_ids = {next_stream_id, next_stream_id + 2} + for stream_id in sorted(stream_ids): + send_request(sock, stream_id) + next_stream_id += 4 + + error_code = read_until_streams_end(sock, stream_ids) + if error_code is not None: + return 0 if error_code == ERROR_SETTINGS_TIMEOUT else 1 + # Give ATS a moment to retire the completed streams before opening the next wave. + time.sleep(0.05) + + print('Expected SETTINGS_TIMEOUT GOAWAY, but the connection closed first') + return 1 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('port', type=int, help='ATS TLS port') + args = parser.parse_args() + return run(args.port) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/gold_tests/h2/h1_trailer_client.py b/tests/gold_tests/h2/h1_trailer_client.py new file mode 100644 index 00000000000..4539bf9c7f5 --- /dev/null +++ b/tests/gold_tests/h2/h1_trailer_client.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Issue a raw HTTP/1 request and reject trailers after the terminal chunk.""" + +from __future__ import annotations + +import socket +import sys + +TERMINAL_CHUNK = b"0\r\n\r\n" + + +def send_request_and_read_response(host: str, port: int) -> bytes: + """Return the raw bytes from a single HTTP/1 response.""" + request = ( + b"GET /trailers HTTP/1.1\r\n" + b"Host: example.data.com\r\n" + b"uuid: h2-origin-trailers-h1\r\n" + b"Connection: keep-alive\r\n" + b"\r\n") + response = bytearray() + saw_terminal_chunk = False + + with socket.create_connection((host, port), timeout=5) as conn: + conn.settimeout(5) + conn.sendall(request) + while True: + try: + chunk = conn.recv(4096) + except socket.timeout: + if saw_terminal_chunk: + break + raise + if not chunk: + break + response.extend(chunk) + if TERMINAL_CHUNK in response: + saw_terminal_chunk = True + conn.settimeout(0.5) + + return bytes(response) + + +def main() -> int: + """Verify ATS does not append H2 origin trailers to an HTTP/1 response.""" + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + + response = send_request_and_read_response(sys.argv[1], int(sys.argv[2])) + terminal_chunk_index = response.find(TERMINAL_CHUNK) + trailer_index = response.lower().find(b"x-ats-h2-trailer") + + print(response.decode("utf-8", errors="replace")) + + if b"hello from h2 origin" not in response: + print("Did not receive the expected response body.", file=sys.stderr) + return 1 + if terminal_chunk_index == -1: + print("Did not receive an HTTP/1 chunked terminal marker.", file=sys.stderr) + return 1 + if trailer_index != -1: + print("H2 origin trailer was forwarded to the HTTP/1 client.", file=sys.stderr) + return 1 + + print("No H2 origin trailers were forwarded to the HTTP/1 client.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/gold_tests/h2/h2_origin_cert_reverify.test.py b/tests/gold_tests/h2/h2_origin_cert_reverify.test.py new file mode 100644 index 00000000000..d36e365472c --- /dev/null +++ b/tests/gold_tests/h2/h2_origin_cert_reverify.test.py @@ -0,0 +1,72 @@ +''' +Verify HTTP/2 origin session reuse re-checks the origin certificate name. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify HTTP/2 origin session reuse re-checks the origin certificate name. +''' + +ts = Test.MakeATSProcess("ts", enable_tls=True) +server = Test.MakeVerifierServerProcess( + "h2-origin", "replay_h2_origin_cert_reverify.yaml", ssl_cert="../tls/ssl/signed-foo.pem", ca_cert="../tls/ssl/signer.pem") + +ts.addDefaultSSLFiles() +ts.addSSLfile("../tls/ssl/signer.pem") + +ts.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + 'proxy.config.ssl.client.CA.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.client.CA.cert.filename': 'signer.pem', + 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', + 'proxy.config.ssl.client.verify.server.policy': 'ENFORCED', + 'proxy.config.ssl.client.verify.server.properties': 'ALL', + 'proxy.config.url_remap.pristine_host_hdr': 1, + 'proxy.config.http.server_session_sharing.pool': 'thread', + 'proxy.config.http.server_session_sharing.match': 'ip', + 'proxy.config.exec_thread.autoconfig.enabled': 0, + 'proxy.config.exec_thread.limit': 1, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|ssl_verify', + }) + +ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + +ts.Disk.remap_config.AddLine(f"map / https://127.0.0.1:{server.Variables.https_port}/") + +tr = Test.AddTestRun("Prime an H2 origin connection for foo.com") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.MakeCurlCommand(f"-v -H 'Host: foo.com' -H 'uuid: foo' http://127.0.0.1:{ts.Variables.port}/foo", ts=ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("foo-response", "foo.com should receive the origin response") +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +tr = Test.AddTestRun("Reject reuse for bar.com") +tr.MakeCurlCommand(f"-v -H 'Host: bar.com' http://127.0.0.1:{ts.Variables.port}/bar", ts=ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "Could Not Connect", "bar.com should fail certificate name verification") +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +ts.Disk.diags_log.Content = Testers.ContainsExpression( + r"WARNING: Origin hostname \(bar.com\) not in certificate. Action=Terminate", + "The pooled H2 origin session should be rejected for bar.com.") diff --git a/tests/gold_tests/h2/h2_origin_trailers_h1.replay.yaml b/tests/gold_tests/h2/h2_origin_trailers_h1.replay.yaml new file mode 100644 index 00000000000..43d92baa012 --- /dev/null +++ b/tests/gold_tests/h2/h2_origin_trailers_h1.replay.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: '1.0' + +sessions: +- protocol: + stack: http2 + tls: + sni: test_sni + transactions: + - client-request: + headers: + fields: + - [:method, GET] + - [:scheme, https] + - [:authority, example.data.com] + - [:path, /trailers] + - [uuid, h2-origin-trailers-h1] + + proxy-request: + headers: + fields: + - [uuid, {value: h2-origin-trailers-h1, as: equal}] + + server-response: + status: 200 + headers: + fields: + - [Content-Type, text/plain] + - [Cache-Control, no-store] + content: + encoding: plain + data: hello from h2 origin + size: 20 + trailers: + fields: + - [x-ats-h2-trailer, smuggled] + - [x-trailer-check, after-body] diff --git a/tests/gold_tests/h2/h2_origin_trailers_h1.test.py b/tests/gold_tests/h2/h2_origin_trailers_h1.test.py new file mode 100644 index 00000000000..7ea653a979a --- /dev/null +++ b/tests/gold_tests/h2/h2_origin_trailers_h1.test.py @@ -0,0 +1,88 @@ +''' +Verify HTTP/2 origin trailers are not forwarded to HTTP/1 clients. +''' + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +Test.Summary = __doc__ + +Test.SkipUnless(Condition.HasProxyVerifierVersion('2.8.0')) + + +class TestOriginTrailers: + """Verify HTTP/2 origin trailers are handled safely by client protocol.""" + + _replay_file = 'h2_origin_trailers_h1.replay.yaml' + + def __init__(self): + self._h1_server = self._configure_server('h2-origin-h1') + self._h1_ts = self._configure_ats('ts-h1', self._h1_server) + self._configure_h1_client() + + self._h2_server = self._configure_server('h2-origin-h2') + self._h2_ts = self._configure_ats('ts-h2', self._h2_server) + self._configure_h2_client() + + def _configure_server(self, name): + return Test.MakeVerifierServerProcess(name, self._replay_file) + + def _configure_ats(self, name, server): + ts = Test.MakeATSProcess(name, enable_tls=True, enable_cache=False) + ts.addDefaultSSLFiles() + + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|http2', + 'proxy.config.exec_thread.autoconfig.enabled': 0, + 'proxy.config.exec_thread.limit': 1, + 'proxy.config.http.server_session_sharing.pool': 'thread', + 'proxy.config.http.server_session_sharing.match': 'ip,sni,cert', + 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + }) + + ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + + ts.Disk.remap_config.AddLine(f'map / https://127.0.0.1:{server.Variables.https_port}') + return ts + + def _configure_h1_client(self): + Test.Setup.CopyAs('h1_trailer_client.py', Test.RunDirectory) + + tr = Test.AddTestRun('HTTP/2 origin trailers are dropped for HTTP/1 clients') + tr.Processes.Default.StartBefore(self._h1_server) + tr.Processes.Default.StartBefore(self._h1_ts) + tr.Processes.Default.Command = f'{sys.executable} h1_trailer_client.py 127.0.0.1 {self._h1_ts.Variables.port}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'No H2 origin trailers were forwarded to the HTTP/1 client.', 'The HTTP/1 response must end at its terminal chunk.') + + def _configure_h2_client(self): + tr = Test.AddTestRun('HTTP/2 origin trailers are forwarded to HTTP/2 clients') + client = tr.AddVerifierClientProcess('h2-client', self._replay_file, https_ports=[self._h2_ts.Variables.ssl_port]) + client.StartBefore(self._h2_server) + client.StartBefore(self._h2_ts) + client.Streams.All += Testers.ContainsExpression( + 'x-ats-h2-trailer: smuggled', 'The HTTP/2 client must receive the origin trailer.') + + +TestOriginTrailers() diff --git a/tests/gold_tests/h2/http2.test.py b/tests/gold_tests/h2/http2.test.py index 1635615ddaa..bcccc3c1630 100644 --- a/tests/gold_tests/h2/http2.test.py +++ b/tests/gold_tests/h2/http2.test.py @@ -153,6 +153,22 @@ ts.Setup.CopyAs('h2client.py', Test.RunDirectory) ts.Setup.CopyAs('h2active_timeout.py', Test.RunDirectory) +settings_limit_ts = Test.MakeATSProcess("ts_settings_limit", enable_tls=True, enable_cache=False) +settings_limit_ts.addDefaultSSLFiles() +settings_limit_ts.Setup.CopyAs('clients/h2_max_settings_per_minute.py', Test.RunDirectory) +settings_limit_ts.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{settings_limit_ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{settings_limit_ts.Variables.SSLDir}', + 'proxy.config.http2.max_settings_per_frame': -1, + 'proxy.config.http2.max_settings_per_minute': 1, + 'proxy.config.http2.max_settings_frames_per_minute': 100, + }) +settings_limit_ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') +settings_limit_ts.Disk.diags_log.Content = Testers.ContainsExpression( + "ERROR: HTTP/2 connection error.*recv settings too frequent setting changes", + "ATS should log the SETTINGS limit connection error.") + # ---- # Test Cases # ---- @@ -239,3 +255,11 @@ # Different versions of curl will have different cases for HTTP/2 field names. tr.Processes.Default.Streams.stderr = Testers.GoldFile("gold/http2_9_stderr.gold", case_insensitive=True) tr.StillRunningAfter = server + +# Test Case 10: max_settings_per_minute with max_settings_per_frame disabled +tr = Test.AddTestRun("max_settings_per_minute with max_settings_per_frame disabled") +tr.Processes.Default.Command = f'{sys.executable} h2_max_settings_per_minute.py {settings_limit_ts.Variables.ssl_port}' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(settings_limit_ts) +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "Received GOAWAY with error code 11", "Received ENHANCE_YOUR_CALM GOAWAY.") diff --git a/tests/gold_tests/h2/http2_crlf_header_validation.test.py b/tests/gold_tests/h2/http2_crlf_header_validation.test.py new file mode 100644 index 00000000000..0c1eea38801 --- /dev/null +++ b/tests/gold_tests/h2/http2_crlf_header_validation.test.py @@ -0,0 +1,91 @@ +''' +Verify HTTP/2 requests with NUL, CR, or LF in header values are rejected. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +Test.Summary = 'HTTP/2 requests with NUL, CR, or LF in header values are rejected' +Test.ContinueOnFail = True + +MALFORMED_CLIENT = 'malformed_h2_request_client.py' + +CONTROL_CHARACTER_CASES = ( + { + 'scenario': 'crlf-in-header-value', + 'description': 'HTTP/2 request with CRLF in header value', + }, + { + 'scenario': 'cr-in-header-value', + 'description': 'HTTP/2 request with bare CR in header value', + }, + { + 'scenario': 'lf-in-header-value', + 'description': 'HTTP/2 request with bare LF in header value', + }, + { + 'scenario': 'nul-in-header-value', + 'description': 'HTTP/2 request with NUL in header value', + }, +) + +server = Test.MakeOriginServer('server') +server.Streams.All = Testers.ExcludesExpression( + 'x-injected', + 'Malformed control-character requests must not reach the origin server.', +) +server.Streams.All += Testers.ExcludesExpression( + 'malformed-nul-value', + 'Malformed NUL request must not reach the origin server.', +) +server.addResponse( + 'sessionlog.json', { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '', + }, { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '', + }) + +ts = Test.MakeATSProcess('ts', enable_tls=True, enable_cache=False) +ts.addDefaultSSLFiles() +ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') +ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + }) +ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server.Variables.Port}/') + +Test.Setup.CopyAs('../connect/' + MALFORMED_CLIENT, Test.RunDirectory) + +for i, case in enumerate(CONTROL_CHARACTER_CASES): + tr = Test.AddTestRun(case['description']) + tr.Processes.Default.Command = (f'{sys.executable} {MALFORMED_CLIENT} {ts.Variables.ssl_port} {case["scenario"]}') + tr.Processes.Default.ReturnCode = 0 + if i == 0: + tr.Processes.Default.StartBefore(server) + tr.Processes.Default.StartBefore(ts) + tr.StillRunningAfter = ts + tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + r'Received (RST_STREAM|GOAWAY|HTTP/2 response with status 4\d\d)', + f'ATS should reject the request: {case["description"]}', + ) diff --git a/tests/gold_tests/h2/http2_flow_control.test.py b/tests/gold_tests/h2/http2_flow_control.test.py index f09429f7d48..d34365cdcb3 100644 --- a/tests/gold_tests/h2/http2_flow_control.test.py +++ b/tests/gold_tests/h2/http2_flow_control.test.py @@ -17,6 +17,7 @@ # limitations under the License. import re +import sys from enum import Enum from typing import List, Optional @@ -332,6 +333,44 @@ def run(self) -> None: self._configure_outbound_test_run() +class Http2DynamicWindowSettingsCapTest: + """Verify dynamic stream windows cap unacknowledged SETTINGS frames.""" + + _replay_file: str = 'replay/http2_settings_ack_stall.replay.yaml' + + def run(self) -> None: + """Configure the test run.""" + tr = Test.AddTestRun('Dynamic stream windows cap unacknowledged SETTINGS') + server = tr.AddVerifierServerProcess('server-settings-cap', self._replay_file) + ts = tr.MakeATSProcess('ts-settings-cap', enable_tls=True, enable_cache=False) + + ts.addDefaultSSLFiles() + ts.Setup.CopyAs('clients/h2_settings_ack_stall.py', Test.RunDirectory) + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http2', + 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', + 'proxy.config.http.insert_response_via_str': 2, + 'proxy.config.http2.active_timeout_in': 5, + 'proxy.config.http2.flow_control.policy_in': 2, + 'proxy.config.http2.max_concurrent_streams_in': 2, + }) + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server.Variables.http_port}') + ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + + tr.Processes.Default.StartBefore(server) + tr.Processes.Default.StartBefore(ts) + tr.Processes.Default.Command = f'{sys.executable} h2_settings_ack_stall.py {ts.Variables.ssl_port}' + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + 'GOAWAY error_code=4', 'ATS should close the connection with SETTINGS_TIMEOUT.') + ts.Disk.diags_log.Content = Testers.ContainsExpression( + 'ERROR: HTTP/2 connection error code=0x04.*send settings too many outstanding SETTINGS frames', + 'ATS should log the expected SETTINGS_TIMEOUT connection error.') + + # # Default configuration. # @@ -373,3 +412,6 @@ def run(self) -> None: initial_window_size=10, flow_control_policy=2) test.run() + +test = Http2DynamicWindowSettingsCapTest() +test.run() diff --git a/tests/gold_tests/h2/http2_txn_start_read_gate.test.py b/tests/gold_tests/h2/http2_txn_start_read_gate.test.py new file mode 100644 index 00000000000..db710858a61 --- /dev/null +++ b/tests/gold_tests/h2/http2_txn_start_read_gate.test.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Verify HTTP/2 read events do not bypass async TXN_START hooks. +''' + +Test.SkipUnless(Condition.HasProxyVerifierVersion('2.8.0')) + +replay_file = "replay/http2_txn_start_read_gate.replay.yaml" +server = Test.MakeVerifierServerProcess("server", replay_file) + +ts = Test.MakeATSProcess("ts", enable_tls=True, enable_cache=False) +ts.addDefaultSSLFiles() +ts.Disk.remap_config.AddLine(f"map / http://127.0.0.1:{server.Variables.http_port}") +ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') +ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'delay_txn_start', + 'proxy.config.ssl.server.cert.path': f"{ts.Variables.SSLDir}", + 'proxy.config.ssl.server.private_key.path': f"{ts.Variables.SSLDir}", + }) + +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'delay_txn_start.so'), ts, '500') + +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.AddVerifierClientProcess("client", replay_file, http_ports=[ts.Variables.port], https_ports=[ts.Variables.ssl_port]) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All += Testers.ExcludesExpression(r'\[ERROR\]', 'Proxy Verifier should not report errors.') +tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'Equals Success: Key: "1", Content Data: "body", Value: "response-body"', 'Client should receive the response body.') + +server.Streams.All += Testers.ContainsExpression( + 'Equals Success: Key: "1", Content Data: "body", Value: "request-body"', 'Origin should receive the request body.') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + "delayed TXN_START reenable", "The test plugin should delay and then resume TXN_START.") +ts.Disk.traffic_out.Content += Testers.ExcludesExpression( + "READ_REQUEST_HDR before delayed TXN_START reenable", "READ_REQUEST_HDR must wait for TXN_START reenable.") diff --git a/tests/gold_tests/h2/replay/http2_settings_ack_stall.replay.yaml b/tests/gold_tests/h2/replay/http2_settings_ack_stall.replay.yaml new file mode 100644 index 00000000000..5e7ccb7e762 --- /dev/null +++ b/tests/gold_tests/h2/replay/http2_settings_ack_stall.replay.yaml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +sessions: +- transactions: + + - client-request: + method: GET + version: '1.1' + url: /stream/1 + headers: + fields: + - [ uuid, settings-ack-stall-1 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, "0" ] + + - client-request: + method: GET + version: '1.1' + url: /stream/3 + headers: + fields: + - [ uuid, settings-ack-stall-3 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, "0" ] + + - client-request: + method: GET + version: '1.1' + url: /stream/5 + headers: + fields: + - [ uuid, settings-ack-stall-5 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, "0" ] diff --git a/tests/gold_tests/h2/replay/http2_txn_start_read_gate.replay.yaml b/tests/gold_tests/h2/replay/http2_txn_start_read_gate.replay.yaml new file mode 100644 index 00000000000..6f9bddd24b0 --- /dev/null +++ b/tests/gold_tests/h2/replay/http2_txn_start_read_gate.replay.yaml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: '1.0' + +sessions: +- protocol: + stack: http2 + tls: + sni: delay-txn-start.test + transactions: + - client-request: + frames: + - HEADERS: + headers: + fields: + - [:method, POST] + - [:scheme, https] + - [:authority, delay-txn-start.test] + - [:path, /read-gate] + - [Content-Length, '12'] + - [uuid, '1'] + - DATA: + content: + encoding: plain + data: request-body + size: 12 + + proxy-request: + content: + encoding: plain + data: request-body + verify: {as: equal} + + server-response: + status: 200 + reason: OK + content: + encoding: plain + data: response-body + size: 13 + + proxy-response: + status: 200 + content: + encoding: plain + data: response-body + verify: {as: equal} diff --git a/tests/gold_tests/h2/replay_h2_origin_cert_reverify.yaml b/tests/gold_tests/h2/replay_h2_origin_cert_reverify.yaml new file mode 100644 index 00000000000..bae9480f8df --- /dev/null +++ b/tests/gold_tests/h2/replay_h2_origin_cert_reverify.yaml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: '1.0' + +sessions: + - protocol: + stack: http2 + tls: + version: TLSv1.3 + sni: foo.com + proxy-verify-mode: 1 + proxy-provided-cert: false + transactions: + - all: { headers: { fields: [[ uuid, foo ]]}} + + proxy-request: + protocol: + stack: http2 + tls: + sni: foo.com + proxy-verify-mode: 1 + proxy-provided-cert: false + version: '2' + scheme: https + method: GET + url: /foo + headers: + encoding: esc_json + fields: + - [ Host, foo.com ] + content: + encoding: plain + size: 0 + + server-response: + version: '2' + status: 200 + headers: + encoding: esc_json + fields: + - [ Content-Length, 12 ] + content: + encoding: plain + data: foo-response + size: 12 diff --git a/tests/gold_tests/headers/clients/oversized_field_h2_client.py b/tests/gold_tests/headers/clients/oversized_field_h2_client.py new file mode 100644 index 00000000000..83d9b3feec0 --- /dev/null +++ b/tests/gold_tests/headers/clients/oversized_field_h2_client.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Minimal raw HTTP/2 client for the oversized-header autest. + +Why this exists: curl and nghttp both refuse to *send* a header set larger than +~60000 bytes (an nghttp2 client-side guard), so they cannot drive the oversized +HTTP/2 header (name or value length >= 65535) path. This client builds the HPACK +block and frames it itself, emitting CONTINUATION frames when the block exceeds +the 16 KB max frame size, so there is no client-side cap. It connects over TLS +with ALPN h2. + +Usage: + oversized_field_h2_client.py PATH NAME_SIZE VALUE_SIZE [HOST] [PORT] [AUTHORITY] + NAME_SIZE > 0 -> add a header whose NAME is that many bytes ("x"*N) + VALUE_SIZE > 0 -> add a header "x-big" whose VALUE is that many bytes + both 0 -> plain GET (sanity check) +Defaults: HOST=127.0.0.1 PORT=8543 AUTHORITY=example.com + +Prints one line: + status=<:status or None> rst_error= goaway_error= sent_block_bytes= frames= +- status None + goaway_error=9 => HPACK connection error (GOAWAY COMPRESSION_ERROR) +- status None + rst_error set => stream reset +- status 200 + origin received it => request was forwarded (bug) + +Requires: python3 hpack module. +""" +import socket +import ssl +import struct +import sys +import time + +from hpack import Decoder, Encoder + +path = sys.argv[1] if len(sys.argv) > 1 else "/raw" +name_size = int(sys.argv[2]) if len(sys.argv) > 2 else 0 +value_size = int(sys.argv[3]) if len(sys.argv) > 3 else 0 +HOST = sys.argv[4] if len(sys.argv) > 4 else "127.0.0.1" +PORT = int(sys.argv[5]) if len(sys.argv) > 5 else 8543 +authority = sys.argv[6] if len(sys.argv) > 6 else "example.com" + + +def frame(ftype, flags, sid, payload): + return struct.pack(">I", len(payload))[1:] + bytes([ftype, flags]) + struct.pack(">I", sid) + payload + + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE +ctx.set_alpn_protocols(["h2"]) +raw = socket.create_connection((HOST, PORT), timeout=15) +s = ctx.wrap_socket(raw, server_hostname=authority) +assert s.selected_alpn_protocol() == "h2", f"ALPN negotiation failed: {s.selected_alpn_protocol()}" + +s.sendall(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n") +s.sendall(frame(0x4, 0, 0, b"")) # empty SETTINGS +# Do not ACK here: a SETTINGS ACK acknowledges the peer's SETTINGS, which we have +# not received yet. The read loop below sends the ACK when the server's SETTINGS +# frame arrives. + +# Send a uuid header so the Proxy Verifier server can key the transaction on it +# (the replay matches each request by uuid). The path maps directly to the key, +# e.g. /h2-normal -> "h2-normal". +hdrs = [(":method", "GET"), (":scheme", "https"), (":authority", authority), (":path", path), ("uuid", path.lstrip("/"))] +if value_size > 0: + hdrs.append(("x-big", "A" * value_size)) +if name_size > 0: + hdrs.append(("x" * name_size, "small")) +block = Encoder().encode(hdrs) + +MAXF = 16384 +chunks = [block[i:i + MAXF] for i in range(0, len(block), MAXF)] or [b""] +flags = 0x1 | (0x4 if len(chunks) == 1 else 0) # END_STREAM; END_HEADERS only if single frame +s.sendall(frame(0x1, flags, 1, chunks[0])) +for i, c in enumerate(chunks[1:], start=1): + last = (i == len(chunks) - 1) + s.sendall(frame(0x9, 0x4 if last else 0, 1, c)) # CONTINUATION, END_HEADERS on last + +dec = Decoder() +status = rst = goaway = None +# Generous read deadline: on a loaded CI host (parallel autest shards under ASAN) +# ATS can take several seconds to receive the oversized CONTINUATION-framed +# HEADERS, reject it, and send GOAWAY. A tight timeout turns that latency into a +# spurious "wrong code" failure, so allow ample time and report a distinct +# timed_out marker if no terminal frame (HEADERS/RST/GOAWAY) ever arrives. +READ_DEADLINE_S = 30 +timed_out = False +buf = b"" +end = time.time() + READ_DEADLINE_S +try: + while time.time() < end: + s.settimeout(max(0.1, end - time.time())) + data = s.recv(65536) + if not data: + break + buf += data + while len(buf) >= 9: + ln = struct.unpack(">I", b"\x00" + buf[:3])[0] + ftype = buf[3] + fl = buf[4] + if len(buf) < 9 + ln: + break + payload = buf[9:9 + ln] + buf = buf[9 + ln:] + if ftype == 0x1: # HEADERS + pl = payload + pad = 0 + if fl & 0x08: # PADDED: leading pad-length byte, trailing padding + pad = pl[0] + pl = pl[1:] + if fl & 0x20: # PRIORITY: 5-byte stream-dependency + weight prefix + pl = pl[5:] + if pad: + pl = pl[:-pad] + try: + for k, v in dec.decode(pl): + if k == ":status": + status = v + except Exception as e: + status = f"decode_err:{e}" + elif ftype == 0x3: # RST_STREAM + rst = struct.unpack(">I", payload[:4])[0] + elif ftype == 0x7: # GOAWAY + goaway = struct.unpack(">I", payload[4:8])[0] + elif ftype == 0x4 and not (fl & 0x1): + s.sendall(frame(0x4, 0x1, 0, b"")) # ack server SETTINGS + if status is not None or rst is not None or goaway is not None: + break + else: + timed_out = True # deadline reached without a terminal frame +except socket.timeout: + timed_out = True +print( + f"status={status} rst_error={rst} goaway_error={goaway} timed_out={timed_out} sent_block_bytes={len(block)} frames={len(chunks)}" +) +s.close() diff --git a/tests/gold_tests/headers/hopbyhop_connection.test.py b/tests/gold_tests/headers/hopbyhop_connection.test.py new file mode 100644 index 00000000000..1492b7c7f58 --- /dev/null +++ b/tests/gold_tests/headers/hopbyhop_connection.test.py @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Test.Summary = "autest for dynamic Connection-token hop-by-hop stripping (forward-path)" + +# Minimal ATSReplay invocation pointing at the replay YAML + +Test.ATSReplayTest(replay_file="replays/hopbyhop_connection.replay.yaml") diff --git a/tests/gold_tests/headers/oversized_field.test.py b/tests/gold_tests/headers/oversized_field.test.py new file mode 100644 index 00000000000..3dcc7205ea6 --- /dev/null +++ b/tests/gold_tests/headers/oversized_field.test.py @@ -0,0 +1,27 @@ +''' +Verify that a header field exceeding the uint16_t field-length limit is rejected +with a 400 and never forwarded to the origin. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify that a single header field whose length exceeds the uint16_t field-length +limit is rejected by the parser with a 400, rather than stored with a truncated +length. +''' + +Test.ATSReplayTest(replay_file='replay/oversized_field.replay.yaml') diff --git a/tests/gold_tests/headers/oversized_field_h2.test.py b/tests/gold_tests/headers/oversized_field_h2.test.py new file mode 100644 index 00000000000..7f312382a57 --- /dev/null +++ b/tests/gold_tests/headers/oversized_field_h2.test.py @@ -0,0 +1,136 @@ +''' +Verify that an HTTP/2 header whose name or value length exceeds the uint16_t +field-length limit (>= 65535 bytes) is treated as an HPACK connection error +(GOAWAY with COMPRESSION_ERROR, code 0x9) and is never forwarded to the origin. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +Test.Summary = ''' +Verify that an HTTP/2 header field whose name or value length exceeds the +uint16_t field-length limit (>= 65535 bytes) is rejected as an HPACK connection +error (GOAWAY COMPRESSION_ERROR, code 0x9) rather than stored with a truncated +length, and is never forwarded to the origin. +''' + +Test.SkipUnless(Condition.HasOpenSSLVersion('1.1.1'), Condition.HasProxyVerifierVersion('2.8.0')) + + +class OversizedFieldH2Test: + '''Drive an oversized HTTP/2 header with a raw HPACK client.''' + + replayFile = "replay/oversized_field_h2.replay.yaml" + clientScript = "oversized_field_h2_client.py" + + # Header sizes large enough to exceed the uint16_t (65535) field-length limit. + oversizedSize = 70000 + + def __init__(self): + self.__setupOriginServer() + self.__setupTS() + self.__setupClient() + + def __setupOriginServer(self): + self._server = Test.MakeVerifierServerProcess("verifier-server", self.replayFile) + # The origin must never receive the oversized requests. If ATS forwarded + # them (the pre-fix behavior), the verifier server would log a request + # for these paths / serve the marker bodies. + self._server.Streams.All += Testers.ExcludesExpression( + 'h2-oversized-value', 'Origin must not receive the oversized-value request.') + self._server.Streams.All += Testers.ExcludesExpression( + 'h2-oversized-name', 'Origin must not receive the oversized-name request.') + # Regression guard: the normal, under-limit request MUST reach the origin. + self._server.Streams.All += Testers.ContainsExpression('h2-normal', 'Origin must receive the normal under-limit request.') + + def __setupTS(self): + self._ts = Test.MakeATSProcess("ts", enable_tls=True, enable_cache=False) + self._ts.addDefaultSSLFiles() + self._ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http2|hpack', + 'proxy.config.ssl.server.cert.path': f"{self._ts.Variables.SSLDir}", + 'proxy.config.ssl.server.private_key.path': f"{self._ts.Variables.SSLDir}", + # max_header_list_size is raised well above the oversized field so the + # request is not rejected at the HTTP/2 header-list-size level first. + # header_field_max_size is set to its maximum (65535, the uint16_t + # ceiling enforced by the records range check); a field larger than that + # is rejected by the configured field-size limit with a COMPRESSION_ERROR. + # The uint16_t storage limit in the MIME setters is exercised directly by + # the HpackIndexingTable unit test: header_field_max_size can no longer be + # configured above 65535, so an oversized field can no longer reach the + # storage path end to end through config. + 'proxy.config.http.header_field_max_size': 65535, + 'proxy.config.http2.max_header_list_size': 8 * 1024 * 1024, + }) + # Rejecting the oversized field is an HPACK connection error, so ATS + # intentionally logs "ERROR: HTTP/2 connection error code=0x09 ... + # compression error". Whitelist exactly that line; the default check + # treats any "ERROR:" in diags.log as a failure, so assert this + # expected line is present instead. + self._ts.Disk.diags_log.Content = Testers.ContainsExpression( + r"ERROR: HTTP/2 connection error code=0x09 .* compression error", + "ATS must log the expected HTTP/2 COMPRESSION_ERROR for the oversized field.") + self._ts.Disk.remap_config.AddLine(f"map / http://127.0.0.1:{self._server.Variables.http_port}") + self._ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + + def __setupClient(self): + self._ts.Setup.CopyAs(f"clients/{self.clientScript}", Test.RunDirectory) + + def run(self): + # Case 1: oversized header VALUE. + tr = Test.AddTestRun("oversized H2 header value") + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(self._ts) + port = self._ts.Variables.ssl_port + tr.Processes.Default.Command = ( + f"{sys.executable} {self.clientScript} /h2-oversized-value 0 {self.oversizedSize} 127.0.0.1 {port} example.com") + tr.Processes.Default.ReturnCode = 0 + # No :status (connection error, not a response) and GOAWAY COMPRESSION_ERROR (0x9). + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'status=None', 'Client must not get an HTTP response for the oversized header.') + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'goaway_error=9', 'ATS must send GOAWAY with COMPRESSION_ERROR (0x9).') + tr.StillRunningAfter = self._ts + tr.StillRunningAfter = self._server + + # Case 2: oversized header NAME. + tr = Test.AddTestRun("oversized H2 header name") + tr.Processes.Default.Command = ( + f"{sys.executable} {self.clientScript} /h2-oversized-name {self.oversizedSize} 0 127.0.0.1 {port} example.com") + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'status=None', 'Client must not get an HTTP response for the oversized header.') + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'goaway_error=9', 'ATS must send GOAWAY with COMPRESSION_ERROR (0x9).') + tr.StillRunningAfter = self._ts + tr.StillRunningAfter = self._server + + # Case 3: NORMAL, under-limit request (regression guard against + # over-rejection). Sanity mode "0 0" sends a plain GET; the client must + # get a 200 and the origin must receive it. + tr = Test.AddTestRun("normal under-limit H2 request") + tr.Processes.Default.Command = (f"{sys.executable} {self.clientScript} /h2-normal 0 0 127.0.0.1 {port} example.com") + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'status=200', 'Client must get a 200 for the normal under-limit request.') + tr.StillRunningAfter = self._ts + tr.StillRunningAfter = self._server + + +OversizedFieldH2Test().run() diff --git a/tests/gold_tests/headers/replay/oversized_field.replay.yaml b/tests/gold_tests/headers/replay/oversized_field.replay.yaml new file mode 100644 index 00000000000..ac6690fdf9a --- /dev/null +++ b/tests/gold_tests/headers/replay/oversized_field.replay.yaml @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: 'Verify ATS rejects a header field longer than the uint16_t field-length limit with a 400.' + + dns: + name: 'dns' + + server: + name: 'server' + + client: + name: 'client' + + ats: + name: 'ts' + process_config: + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http' + + remap_config: + - from: "http://example.com/" + to: "http://backend.example.com:{SERVER_HTTP_PORT}/" + +sessions: +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /oversized-field + headers: + fields: + - [ Host, example.com ] + - [ uuid, oversized-field ] + - [ X-Big, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ] + + # ATS must reject during parse and never forward the request to the origin. + proxy-request: + expect: absent + + proxy-response: + status: 400 + + # Normal, under-limit request. Regression guard: a small header must NOT be + # over-rejected. ATS must forward it to the origin and return 200. + - client-request: + method: "GET" + version: "1.1" + url: /normal-field + headers: + fields: + - [ Host, example.com ] + - [ uuid, normal-field ] + - [ X-Small, small-value ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, "4" ] + - [ uuid, normal-field ] + + proxy-request: + headers: + fields: + - [ uuid, { value: normal-field, as: equal } ] + + proxy-response: + status: 200 diff --git a/tests/gold_tests/headers/replay/oversized_field_h2.replay.yaml b/tests/gold_tests/headers/replay/oversized_field_h2.replay.yaml new file mode 100644 index 00000000000..a88f5ba88f8 --- /dev/null +++ b/tests/gold_tests/headers/replay/oversized_field_h2.replay.yaml @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This replay file only defines the origin (verifier server) side. The raw HPACK +# client (clients/oversized_field_h2_client.py) drives the actual HTTP/2 traffic, +# because Proxy Verifier (like curl/nghttp) refuses to send a header set larger +# than ~60000 bytes and so cannot exercise the oversized-header path. +# +# These transactions exist so that, if ATS were to (incorrectly) forward an +# oversized-header request to the origin, the verifier server would match a +# request for /h2-oversized-* and log the unique marker in its response body. +# The test asserts the server log EXCLUDES those markers, proving the origin +# received no request. +meta: + version: "1.0" + +sessions: +- transactions: + + - client-request: + method: "GET" + url: /h2-oversized-value + version: "1.1" + headers: + fields: + - [Host, example.com] + - [uuid, h2-oversized-value] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "26"] + content: + encoding: plain + data: "ORIGIN_SAW_OVERSIZED_VALUE" + + - client-request: + method: "GET" + url: /h2-oversized-name + version: "1.1" + headers: + fields: + - [Host, example.com] + - [uuid, h2-oversized-name] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "25"] + content: + encoding: plain + data: "ORIGIN_SAW_OVERSIZED_NAME" + + # Normal, under-limit request. Regression guard: a small header must NOT be + # over-rejected. The origin must receive it and return 200. + - client-request: + method: "GET" + url: /h2-normal + version: "1.1" + headers: + fields: + - [Host, example.com] + - [uuid, h2-normal] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "17"] + content: + encoding: plain + data: "ORIGIN_SAW_NORMAL" diff --git a/tests/gold_tests/headers/replays/hopbyhop_connection.replay.yaml b/tests/gold_tests/headers/replays/hopbyhop_connection.replay.yaml new file mode 100644 index 00000000000..2b084f747ab --- /dev/null +++ b/tests/gold_tests/headers/replays/hopbyhop_connection.replay.yaml @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: "Verify dynamic Connection-token hop-by-hop header stripping (forward-path)" + + server: + name: 'hopbyhop-server' + + client: + name: 'hopbyhop-client' + + ats: + name: 'ts-hopbyhop' + remap_config: + - from: "http://www.example.com/" + to: "http://127.0.0.1:{SERVER_HTTP_PORT}/" + +sessions: + - transactions: + - client-request: + method: GET + version: "1.1" + url: /t1 + headers: + fields: + - [Host, www.example.com] + - [Connection, X-Internal-Auth] + - [X-Internal-Auth, admin] + - [uuid, hopbyhop-1] + + proxy-request: + headers: + fields: + - [X-Internal-Auth, { as: absent }] + - [Connection, { as: absent }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 0] + + proxy-response: + status: 200 + + - client-request: + method: GET + version: "1.1" + url: /t2 + headers: + fields: + - [Host, www.example.com] + - [Connection, "X-Foo, X-Bar"] + - [X-Foo, a] + - [X-Bar, b] + - [uuid, hopbyhop-2] + + proxy-request: + headers: + fields: + - [X-Foo, { as: absent }] + - [X-Bar, { as: absent }] + - [Connection, { as: absent }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 0] + + proxy-response: + status: 200 + + - client-request: + method: GET + version: "1.1" + url: /t3 + headers: + fields: + - [Host, www.example.com] + - [Connection, TE] + - [TE, trailers] + - [uuid, hopbyhop-3] + + proxy-request: + headers: + fields: + - [TE, { as: present }] + - [Connection, { as: absent }] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 0] + + proxy-response: + status: 200 diff --git a/tests/gold_tests/ip_allow/connect_destination_acl.test.py b/tests/gold_tests/ip_allow/connect_destination_acl.test.py new file mode 100644 index 00000000000..2218af3f503 --- /dev/null +++ b/tests/gold_tests/ip_allow/connect_destination_acl.test.py @@ -0,0 +1,168 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' +Verify outbound ip_allow filtering for CONNECT destinations. +''' + +Test.Summary = ''' +Verify outbound ip_allow filtering for CONNECT destinations. +''' + +Test.SkipIf(Condition.CurlUsingUnixDomainSocket()) +Test.ContinueOnFail = True + +server = Test.MakeOriginServer("server", ssl=True) + +request = { + "headers": f"GET / HTTP/1.1\r\nHost: 127.0.0.1:{server.Variables.SSL_Port}\r\n\r\n", + "timestamp": "1469733493.993", + "body": "", +} +response = { + "headers": "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "", +} +server.addResponse("sessionlog.json", request, response) + + +def configure_ts(name): + ts = Test.MakeATSProcess(name, enable_cache=False) + ts.Disk.records_config.update( + { + "proxy.config.diags.debug.enabled": 1, + "proxy.config.diags.debug.tags": "http|ip_allow", + "proxy.config.http.connect_ports": f"{server.Variables.SSL_Port}", + "proxy.config.url_remap.remap_required": 0, + }) + return ts + + +ts_default = configure_ts("ts-default") +ts_allowed = configure_ts("ts-allowed") +ts_allowed.Disk.ip_allow_yaml.AddLines( + [ + "ip_allow:", + " - apply: in", + " ip_addrs: 127.0.0.1", + " action: allow", + " methods: ALL", + " - apply: out", + " ip_addrs: 127.0.0.1", + " action: allow", + " methods: CONNECT", + " - apply: out", + " ip_addrs:", + " - 0.0.0.0/8", + " - 127.0.0.0/8", + " - \"::\"", + " - ::1", + " - 10.0.0.0/8", + " - 172.16.0.0/12", + " - 192.168.0.0/16", + " - 169.254.0.0/16", + " - ::/96", + " - fc00::/7", + " - fe80::/10", + " - ::ffff:0:0/96", + " action: deny", + " methods: CONNECT", + ]) + +ts_sni_default = Test.MakeATSProcess("ts-sni-default", enable_cache=False, enable_tls=True) +ts_sni_default.addDefaultSSLFiles() +ts_sni_default.Disk.records_config.update( + { + "proxy.config.diags.debug.enabled": 1, + "proxy.config.diags.debug.tags": "http|ip_allow|ssl|sni", + "proxy.config.http.connect_ports": f"{server.Variables.SSL_Port}", + "proxy.config.ssl.server.cert.path": ts_sni_default.Variables.SSLDir, + "proxy.config.ssl.server.private_key.path": ts_sni_default.Variables.SSLDir, + }) +ts_sni_default.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') +ts_sni_default.Disk.sni_yaml.AddLines( + [ + "sni:", + "- fqdn: sni-denied.example.com", + f" tunnel_route: 127.0.0.1:{server.Variables.SSL_Port}", + ]) +ts_sni_default.Disk.diags_log.Content += Testers.ContainsExpression( + r"server '127\.0\.0\.1.*' prohibited by ip-allow policy", "SNI tunnel_route should be denied by outbound ip_allow.") + +loopback_url = f"https://127.0.0.1:{server.Variables.SSL_Port}/" +unspecified_url = f"https://0.0.0.0:{server.Variables.SSL_Port}/" +reserved_ipv4_url = f"https://0.1.2.3:{server.Variables.SSL_Port}/" +unspecified_v6_url = f"https://[::]:{server.Variables.SSL_Port}/" +compatible_loopback_url = f"https://[::7f00:1]:{server.Variables.SSL_Port}/" +mapped_loopback_url = f"https://[::ffff:127.0.0.1]:{server.Variables.SSL_Port}/" +mapped_loopback_hex_url = f"https://[::ffff:7f00:1]:{server.Variables.SSL_Port}/" + + +def add_denied_connect_run(name, url, http_connect="403"): + tr = Test.AddTestRun(name) + tr.MakeCurlCommand( + f'-sk --noproxy does-not-match --proxy http://127.0.0.1:{ts_default.Variables.port} ' + f'-o /dev/null -w "http_code=%{{http_code}} http_connect=%{{http_connect}}\\n" {url}', + ts=ts_default) + tr.Processes.Default.ReturnCode = Any(7, 56) + tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + f"http_code=000 http_connect={http_connect}", "CONNECT should be rejected before tunneling.") + tr.StillRunningAfter = server + tr.StillRunningAfter = ts_default + return tr + + +tr = Test.AddTestRun("Default policy denies CONNECT to loopback") +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.SSL_Port)) +tr.Processes.Default.StartBefore(ts_default) +tr.MakeCurlCommand( + f'-sk --noproxy does-not-match --proxy http://127.0.0.1:{ts_default.Variables.port} ' + f'-o /dev/null -w "http_code=%{{http_code}} http_connect=%{{http_connect}}\\n" {loopback_url}', + ts=ts_default) +tr.Processes.Default.ReturnCode = Any(7, 56) +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "http_code=000 http_connect=403", "CONNECT to loopback should be rejected by the default outbound policy.") +tr.StillRunningAfter = server +tr.StillRunningAfter = ts_default + +add_denied_connect_run("ATS rejects CONNECT to unspecified IPv4 before outbound policy", unspecified_url, http_connect="400") +add_denied_connect_run("Default policy denies CONNECT to 0.0.0.0/8", reserved_ipv4_url) +add_denied_connect_run("ATS rejects CONNECT to unspecified IPv6 before outbound policy", unspecified_v6_url, http_connect="400") +add_denied_connect_run("Default policy denies CONNECT to IPv4-compatible loopback", compatible_loopback_url) +add_denied_connect_run("Default policy denies CONNECT to IPv4-mapped loopback", mapped_loopback_url) +add_denied_connect_run("Default policy denies CONNECT to hex IPv4-mapped loopback", mapped_loopback_hex_url) + +tr = Test.AddTestRun("Default policy denies SNI tunnel_route to loopback") +tr.Processes.Default.StartBefore(ts_sni_default) +tr.MakeCurlCommand( + f"-skv --resolve sni-denied.example.com:{ts_sni_default.Variables.ssl_port}:127.0.0.1 " + f"https://sni-denied.example.com:{ts_sni_default.Variables.ssl_port}/", + ts=ts_sni_default) +tr.Processes.Default.ReturnCode = Any(35, 52, 56) +tr.StillRunningAfter = server +tr.StillRunningAfter = ts_sni_default + +tr = Test.AddTestRun("Explicit outbound allow permits CONNECT to loopback") +tr.Processes.Default.StartBefore(ts_allowed) +tr.MakeCurlCommand( + f'-sk --noproxy does-not-match --proxy http://127.0.0.1:{ts_allowed.Variables.port} ' + f'-o /dev/null -w "http_code=%{{http_code}} http_connect=%{{http_connect}}\\n" {loopback_url}', + ts=ts_allowed) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "http_code=200 http_connect=200", "Explicit outbound allow should permit the CONNECT tunnel.") +tr.StillRunningAfter = server +tr.StillRunningAfter = ts_allowed diff --git a/tests/gold_tests/next_hop/strategies_ch5/strategies_ch5.test.py b/tests/gold_tests/next_hop/strategies_ch5/strategies_ch5.test.py new file mode 100644 index 00000000000..fa4ad52948a --- /dev/null +++ b/tests/gold_tests/next_hop/strategies_ch5/strategies_ch5.test.py @@ -0,0 +1,153 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test next hop consistent hashing with 5 rings (MAX_GROUP_RINGS). +Validates mapWrapped and chashIter work at full ring capacity. +''' + +# Define and populate MicroServer. +# +server = Test.MakeOriginServer("server") +response_header = { + "headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Cache-control: max-age=85000\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "This is the body.\n" +} +num_objects = 32 +for i in range(num_objects): + request_header = { + "headers": f"GET /obj{i} HTTP/1.1\r\n" + "Host: does.not.matter\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "" + } + server.addResponse("sessionlog.json", request_header, response_header) + +dns = Test.MakeDNServer("dns") + +# 5 rings, 2 hosts each = 10 next hops. +# only the last ring will be started +# +num_rings = 5 +hosts_per_ring = 2 +num_hosts = num_rings * hosts_per_ring +ts_nh = [] +for ii in range(num_hosts): + ts = Test.MakeATSProcess(f"ts_nh{ii}") + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|dns', + 'proxy.config.dns.nameservers': f"127.0.0.1:{dns.Variables.Port}", + 'proxy.config.dns.resolv_conf': "NULL", + }) + line = f"map / http://127.0.0.1:{server.Variables.Port}" + if ii < (num_hosts - hosts_per_ring): + line += " @plugin=header_rewrite.so @pparam=hdr_rw.conf" + ts.Disk.remap_config.AddLine(line) + ts.Disk.MakeConfigFile("hdr_rw.conf").AddLine("set-status 502") + ts_nh.append(ts) + +ts = Test.MakeATSProcess("ts", use_traffic_out=False, command="traffic_server 2> trace.log") +ts.ReturnCode = Any(0, -2) + +ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|dns|parent|next_hop|host_statuses|hostdb', + 'proxy.config.dns.nameservers': f"127.0.0.1:{dns.Variables.Port}", + 'proxy.config.dns.resolv_conf': "NULL", + 'proxy.config.http.cache.http': 0, + 'proxy.config.http.parent_proxy.per_parent_connect_attempts': 1, + 'proxy.config.http.uncacheable_requests_bypass_parent': 0, + 'proxy.config.http.no_dns_just_forward_to_parent': 1, + 'proxy.config.http.parent_proxy.mark_down_hostdb': 0, + 'proxy.config.http.down_server.cache_time': 1, + 'proxy.config.http.parent_proxy.self_detect': 0, + }) + +ts.Disk.File(ts.Variables.CONFIGDIR + "/strategies.yaml", id="strategies", typename="ats:config") +s = ts.Disk.strategies +s.AddLine("groups:") + +# Build 5 groups. +idx = 0 +for ring in range(num_rings): + s.AddLine(f" - &g{ring}") + for h in range(hosts_per_ring): + dns.addRecords(records={f"next_hop_{idx}": ["127.0.0.1"]}) + s.AddLine(f" - host: next_hop_{idx}") + s.AddLine(f" protocol:") + s.AddLine(f" - scheme: http") + s.AddLine(f" port: {ts_nh[idx].Variables.port}") + s.AddLine(f" weight: 1.0") + idx = idx + 1 + +strategy_lines = [ + "strategies:", + " - strategy: the-strategy", + " policy: consistent_hash", + " hash_key: path", + " go_direct: false", + " parent_is_proxy: true", + " ignore_self_detect: true", + " scheme: http", + " failover:", + " ring_mode: alternate_ring", + " max_simple_retries: 5", + " response_codes: [404]", + " max_unavailable_retries: 5", + " markdown_codes: [502]", + " groups:", +] +for ring in range(num_rings): + strategy_lines.append(f" - *g{ring}") +s.AddLines(strategy_lines) + +ts.Disk.remap_config.AddLine("map http://dummy.com http://not_used @strategy=the-strategy") + +# Only start ring 5 (last ring, indices 8-9). Forces fallover through all 5 rings. +tr = Test.AddTestRun() +ps = tr.Processes.Default +ps.StartBefore(server) +ps.StartBefore(dns) +#for idx in range((num_rings - 1) * hosts_per_ring, len(ts_nh)): +for idx in range(len(ts_nh)): + ps.StartBefore(ts_nh[idx]) +ps.StartBefore(Test.Processes.ts) +ps.Command = 'echo start TS, origin, DNS, and only last-ring next hops' +ps.ReturnCode = 0 + +# Send requests - must fall through rings 1-4 (down) to ring 5. +for i in range(num_objects): + tr = Test.AddTestRun() + tr.MakeCurlCommand(f'--verbose --proxy 127.0.0.1:{ts.Variables.port} http://dummy.com/obj{i}', ts=ts) + ps = tr.Processes.Default + ps.Streams.stdout.Content = Testers.ContainsExpression("This is the body.", "expected body") + ps.ReturnCode = 0 + +tr = Test.AddTestRun() +ps = tr.Processes.Default +ps.Command = ("grep -F ParentResultType::SPECIFIED trace.log | sed 's/^.*(next_hop) [^ ]* //' | sed 's/[.][0-9]*$$//'") +ps.Streams.stdout = "trace.gold" +ps.ReturnCode = 0 diff --git a/tests/gold_tests/next_hop/strategies_ch5/trace.gold b/tests/gold_tests/next_hop/strategies_ch5/trace.gold new file mode 100644 index 00000000000..832f8652262 --- /dev/null +++ b/tests/gold_tests/next_hop/strategies_ch5/trace.gold @@ -0,0 +1,192 @@ +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_1 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_2 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_5 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_6 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_1 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_3 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_5 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_6 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_0 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_3 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_5 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_7 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_1 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_3 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_4 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_6 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_0 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_2 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_5 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_7 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_1 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_3 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_5 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_6 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_1 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_2 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_5 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_6 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_1 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_2 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_5 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_7 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_0 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_2 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_5 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_7 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_0 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_3 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_5 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_7 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_1 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_3 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_5 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_6 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_1 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_2 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_7 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_1 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_2 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_4 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_6 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_0 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_3 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_4 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_7 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_1 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_2 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_4 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_6 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_2 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_6 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_2 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_6 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_4 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_3 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_4 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_4 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_3 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_7 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_0 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_4 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_7 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_0 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_3 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_4 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_7 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_0 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_4 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_0 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_0 +not firstcall, line_number: 0, result: ParentResultType::SPECIFIED +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_9 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 +result->result: ParentResultType::SPECIFIED Chosen parent: next_hop_8 diff --git a/tests/gold_tests/origin_connection/per_server_connection_max.test.py b/tests/gold_tests/origin_connection/per_server_connection_max.test.py index 57e80f769e6..e7bad788ab3 100644 --- a/tests/gold_tests/origin_connection/per_server_connection_max.test.py +++ b/tests/gold_tests/origin_connection/per_server_connection_max.test.py @@ -126,6 +126,7 @@ def _configure_trafficserver(self, max_conn) -> None: self._ts.Disk.remap_config.AddLines([ f"map http://foo.com/ http://www.this.origin.com:{self._server.Variables.Port}/", ]) + self._ts.addPrivateConnectAllowYaml() def _configure_client_with_slow_response(self, tr) -> 'Test.Process': """Configure a client to perform a CONNECT request with a slow response from the server.""" diff --git a/tests/gold_tests/parent_proxy/parent_retry_non_retryable_post.test.py b/tests/gold_tests/parent_proxy/parent_retry_non_retryable_post.test.py new file mode 100644 index 00000000000..7c9a0e68dca --- /dev/null +++ b/tests/gold_tests/parent_proxy/parent_retry_non_retryable_post.test.py @@ -0,0 +1,25 @@ +""" +Verify parent_retry does not mark the parent down for non-retryable requests. +""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +A POST that elicits a configured unavailable_server_retry response code (502) +from the parent must not cause the parent to be marked down. +''' + +Test.ATSReplayTest(replay_file='replays/parent_retry_non_retryable_post.replay.yaml') diff --git a/tests/gold_tests/parent_proxy/replays/parent_retry_non_retryable_post.replay.yaml b/tests/gold_tests/parent_proxy/replays/parent_retry_non_retryable_post.replay.yaml new file mode 100644 index 00000000000..68c0f55d699 --- /dev/null +++ b/tests/gold_tests/parent_proxy/replays/parent_retry_non_retryable_post.replay.yaml @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# A non-idempotent POST elicits a configured-retryable 502 from the parent. +# The parent must not be marked down because POST is not retryable. +# + +meta: + version: "1.0" + +autest: + description: 'Parent must not be marked down for a non-retryable POST' + + server: + name: 'parent-502' + + client: + name: 'client' + # The non-retryable POST may not produce a clean response cycle (ATS may + # still attempt a body-less retry that proxy-verifier reports as a body + # underrun). The security-relevant assertion is on the ATS diags log. + return_code: [0, 1] + + ats: + name: 'ts' + process_config: + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http|parent' + proxy.config.http.no_dns_just_forward_to_parent: 1 + proxy.config.http.parent_proxy.fail_threshold: 1 + proxy.config.http.parent_proxy.self_detect: 0 + proxy.config.url_remap.remap_required: 0 + + parent_config: + - 'dest_domain=. parent="127.0.0.1:{SERVER_HTTP_PORT}|1" go_direct=false parent_is_proxy=true parent_retry=unavailable_server_retry unavailable_server_retry_responses="502,503"' + + log_validation: + diags_log: + excludes: + # Both messages are Note()s emitted by + # ParentSelectionStrategy::markParentDown; either appearing in + # diags.log indicates the markdown path ran. + - expression: 'Parent .* marked as down 127\.0\.0\.1:\d+' + description: 'markParentDown must not run for a non-retryable request' + - expression: 'http parent proxy 127\.0\.0\.1:\d+ marked down' + description: 'Parent must not cross the failure threshold' + +sessions: +- transactions: + - client-request: + method: POST + version: "1.1" + url: http://example.com/unretryable + headers: + fields: + - [Host, example.com] + - [Content-Length, "5"] + - [Proxy-Connection, close] + - [uuid, post-502] + content: + encoding: plain + data: "hello" + + server-response: + status: 502 + reason: Bad Gateway + headers: + fields: + - [Content-Length, "0"] + - [Connection, close] + + proxy-response: + status: 502 diff --git a/tests/gold_tests/pipeline/delete_maxforwards_body_desync.test.py b/tests/gold_tests/pipeline/delete_maxforwards_body_desync.test.py new file mode 100644 index 00000000000..a7f0b4d221e --- /dev/null +++ b/tests/gold_tests/pipeline/delete_maxforwards_body_desync.test.py @@ -0,0 +1,89 @@ +'''Verify a DELETE self-answered from cache drains its request body.''' + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ports import get_port +import sys + +Test.Summary = ''' +Verify that a DELETE (Max-Forwards: 0) answered from the cache drains its request +body instead of leaving it to be parsed as the next request on the keep-alive +connection. +''' + +Test.ContinueOnFail = False + + +class TestDeleteBodyDrain: + """A DELETE with Max-Forwards: 0 is answered directly from cache. If the + request body is not drained, its bytes are parsed as the next request on the + keep-alive connection (a CL.0 desync). This drives that path and asserts the + smuggled request never reaches the origin nor produces a second response.""" + + _server_script: str = 'desync_server.py' + _client_script: str = 'desync_client.py' + _hostname: str = 'www.example.com' + + def __init__(self) -> None: + tr = Test.AddTestRun('DELETE self-response must drain the request body.') + tr.TimeOut = 60 + self._configure_server(tr) + self._configure_traffic_server(tr) + self._configure_client(tr) + + def _configure_server(self, tr: 'TestRun') -> 'Process': + server = tr.Processes.Process('server') + tr.Setup.Copy(self._server_script) + port = get_port(server, 'http_port') + server.Command = f'{sys.executable} {self._server_script} 127.0.0.1 {port}' + server.Ready = When.PortOpenv4(port) + # The origin must serve the warmed path, and must never see the smuggled one. + server.Streams.All += Testers.ContainsExpression(r'ORIGIN_RECV path=\[/\]', 'origin should serve the warmed GET /') + server.Streams.All += Testers.ExcludesExpression('poisoned', 'the smuggled request must never reach the origin') + self._server = server + return server + + def _configure_traffic_server(self, tr: 'TestRun') -> 'Process': + ts = tr.MakeATSProcess('ts', enable_cache=True) + self._ts = ts + ts.Disk.remap_config.AddLine(f'map http://{self._hostname}/ http://127.0.0.1:{self._server.Variables.http_port}/') + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.http.cache.http': 1, + 'proxy.config.http.insert_age_in_response': 1, + }) + return ts + + def _configure_client(self, tr: 'TestRun') -> 'Process': + client = tr.Processes.Default + tr.Setup.Copy(self._client_script) + client.Command = f'{sys.executable} {self._client_script} 127.0.0.1 {self._ts.Variables.port} {self._hostname}' + client.ReturnCode = 0 + # The DELETE must self-answer 200 from a warm cache, or the hit path was + # never exercised and the test would pass vacuously. + client.Streams.All += Testers.ContainsExpression('DELETE_STATUS=200', 'the DELETE must self-answer 200 from a warm cache') + # The desync signatures: a second (smuggled) response, or smuggled bytes. + client.Streams.All += Testers.ExcludesExpression( + 'SECOND_RESPONSE_RECEIVED=True', 'the client must not receive a smuggled second response') + client.Streams.All += Testers.ExcludesExpression('poisoned', 'no smuggled response bytes should reach the client') + client.StartBefore(self._server) + client.StartBefore(self._ts) + + +TestDeleteBodyDrain() diff --git a/tests/gold_tests/pipeline/delete_maxfwd_noop_drain.test.py b/tests/gold_tests/pipeline/delete_maxfwd_noop_drain.test.py new file mode 100644 index 00000000000..0c19b00aac5 --- /dev/null +++ b/tests/gold_tests/pipeline/delete_maxfwd_noop_drain.test.py @@ -0,0 +1,89 @@ +'''Verify the cache-miss (INTERNAL_CACHE_NOOP) DELETE self-response drains its body.''' + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ports import get_port +import sys + +Test.Summary = 'A DELETE to an uncached path self-answers 404 (NOOP) and must drain its body.' +Test.ContinueOnFail = False + + +class TestDeleteMaxfwdNoopDrain: + """A DELETE to an uncached path is self-answered 404 via INTERNAL_CACHE_NOOP. + + Verify the accompanying request body is drained so its bytes are not framed as + the next request on the connection. + """ + + _server_script: str = 'desync_server.py' + _client_script: str = 'desync_client_miss.py' + _hostname: str = 'www.example.com' + + def __init__(self) -> None: + tr = Test.AddTestRun('A NOOP-path DELETE self-response must drain its request body.') + tr.TimeOut = 40 + self._configure_server(tr) + self._configure_traffic_server(tr) + self._configure_client(tr) + + def _configure_server(self, tr: 'TestRun') -> 'Process': + """Configure the origin server. + + :param tr: The test run to associate the origin server with. + :return: The origin server process. + """ + server = tr.Processes.Process('server') + tr.Setup.Copy(self._server_script) + port = get_port(server, 'http_port') + server.Command = f'{sys.executable} {self._server_script} 127.0.0.1 {port}' + server.Ready = When.PortOpenv4(port) + server.Streams.All += Testers.ExcludesExpression('misspoison', 'the smuggled request must not reach the origin') + self._server = server + return server + + def _configure_traffic_server(self, tr: 'TestRun') -> 'Process': + """Configure ATS. + + :param tr: The test run to associate the ATS process with. + :return: The ATS process. + """ + ts = tr.MakeATSProcess('ts', enable_cache=True) + self._ts = ts + ts.Disk.remap_config.AddLine(f'map http://{self._hostname}/ http://127.0.0.1:{self._server.Variables.http_port}/') + ts.Disk.records_config.update({'proxy.config.http.cache.http': 1}) + return ts + + def _configure_client(self, tr: 'TestRun') -> 'Process': + """Configure the client. + + :param tr: The test run to associate the client process with. + :return: The client process. + """ + client = tr.Processes.Default + tr.Setup.Copy(self._client_script) + client.Command = f'{sys.executable} {self._client_script} 127.0.0.1 {self._ts.Variables.port} {self._hostname}' + client.ReturnCode = 0 + client.Streams.All += Testers.ContainsExpression( + 'DELETE_STATUS=404', 'the DELETE must hit the NOOP (miss) path, not a cache hit') + client.Streams.All += Testers.ExcludesExpression('SECOND_RESPONSE_RECEIVED=True', 'no smuggled second response') + client.Streams.All += Testers.ExcludesExpression('misspoison', 'no smuggled bytes should reach the client') + client.StartBefore(self._server) + client.StartBefore(self._ts) + + +TestDeleteMaxfwdNoopDrain() diff --git a/tests/gold_tests/pipeline/desync_client.py b/tests/gold_tests/pipeline/desync_client.py new file mode 100644 index 00000000000..3282fc485c0 --- /dev/null +++ b/tests/gold_tests/pipeline/desync_client.py @@ -0,0 +1,112 @@ +"""Reproduce the DELETE/Max-Forwards self-response body-leak desync. + +The proxy answers a DELETE with Max-Forwards: 0 directly from cache. If it does +not drain the accompanying request body, the leftover bytes are parsed as the +next request on the keep-alive connection (a CL.0 desync). This client warms a +path into the cache, then sends a DELETE whose body is a complete smuggled +request, and reports whether a second (smuggled) response came back. + +To reproduce reliably the DELETE must land on a cache hit: a miss takes the +INTERNAL_CACHE_NOOP path, which drains the body regardless. The proxy self-answers +a hit with 200 and a miss with 404, so we key on that status to know we exercised +the hit path, retrying the warm if the object is not cached yet. +""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import socket +import sys +import time + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("proxy_address", help="Address of the proxy to connect to.") + parser.add_argument("proxy_port", type=int, help="The port of the proxy to connect to.") + parser.add_argument("hostname", help="The Host header field value to use.") + return parser.parse_args() + + +def warm(address: str, port: int, hostname: str) -> None: + """Fetch '/' so the proxy caches it.""" + req = (f"GET / HTTP/1.1\r\nHost: {hostname}\r\n" + f"Connection: close\r\n\r\n").encode() + with socket.create_connection((address, port), timeout=8) as s: + s.sendall(req) + s.settimeout(8) + while True: + try: + if not s.recv(65536): + break + except socket.timeout: + break + + +def attack(address: str, port: int, hostname: str) -> bytes: + """Send DELETE / (Max-Forwards: 0) whose body is a smuggled request.""" + smuggled = (f"GET /poisoned HTTP/1.1\r\nHost: {hostname}\r\n\r\n").encode() + req = (f"DELETE / HTTP/1.1\r\nHost: {hostname}\r\n" + f"Max-Forwards: 0\r\n" + f"Content-Length: {len(smuggled)}\r\n\r\n").encode() + smuggled + with socket.create_connection((address, port), timeout=8) as s: + s.sendall(req) + s.settimeout(3) + data = b"" + try: + while True: + chunk = s.recv(65536) + if not chunk: + break + data += chunk + except socket.timeout: + pass + return data + + +def status_of(response: bytes) -> str: + first_line = response.split(b"\r\n", 1)[0].decode("latin1", "replace") + parts = first_line.split(" ") + return parts[1] if len(parts) > 1 else "?" + + +def main() -> int: + args = parse_args() + response = b"" + delete_status = "?" + for attempt in range(20): + warm(args.proxy_address, args.proxy_port, args.hostname) + time.sleep(0.3) + response = attack(args.proxy_address, args.proxy_port, args.hostname) + delete_status = status_of(response) + print(f"attempt={attempt} DELETE_STATUS={delete_status}", flush=True) + if delete_status == "200": + # We exercised the cache-hit self-response path. Measure here. + break + + num_responses = response.count(b"HTTP/1.1 ") + print(f"DELETE_STATUS={delete_status}") + print(f"RESPONSE_COUNT={num_responses}") + print(f"SECOND_RESPONSE_RECEIVED={num_responses >= 2}") + print("----- raw bytes the proxy returned for the DELETE -----") + sys.stdout.write(response.decode("latin1", "replace")) + print("\n----- end -----") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/gold_tests/pipeline/desync_client_miss.py b/tests/gold_tests/pipeline/desync_client_miss.py new file mode 100644 index 00000000000..9673f4fb8a9 --- /dev/null +++ b/tests/gold_tests/pipeline/desync_client_miss.py @@ -0,0 +1,45 @@ +"""NOOP-path check: DELETE an uncached path (Max-Forwards:0) with a smuggled body. +A cache miss self-answers via INTERNAL_CACHE_NOOP (404); the body must be drained.""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import socket +import sys + +addr, port, host = sys.argv[1], int(sys.argv[2]), sys.argv[3] +smug = f"GET /misspoison HTTP/1.1\r\nHost: {host}\r\n\r\n".encode() +req = (f"DELETE /coldpath HTTP/1.1\r\nHost: {host}\r\nMax-Forwards: 0\r\n" + f"Content-Length: {len(smug)}\r\n\r\n").encode() + smug +s = socket.create_connection((addr, port), timeout=8) +s.sendall(req) +s.settimeout(3) +data = b"" +try: + while True: + c = s.recv(65536) + if not c: + break + data += c +except socket.timeout: + pass +s.close() +st = data.split(b"\r\n", 1)[0].split(b" ") +print(f"DELETE_STATUS={st[1].decode() if len(st)>1 else '?'}") +n = data.count(b"HTTP/1.1 ") +print(f"RESPONSE_COUNT={n}") +print(f"SECOND_RESPONSE_RECEIVED={n>=2}") +sys.stdout.write(data.decode('latin1', 'replace')) +print("\n---end---") diff --git a/tests/gold_tests/pipeline/desync_server.py b/tests/gold_tests/pipeline/desync_server.py new file mode 100644 index 00000000000..fcf10b5a9f0 --- /dev/null +++ b/tests/gold_tests/pipeline/desync_server.py @@ -0,0 +1,97 @@ +"""A cacheable origin that logs every request path it receives. + +Used by delete_maxforwards_body_desync.test.py. It serves a cacheable 200 for +any GET so the proxy can turn a warmed path into a cache hit, and it prints a +distinctive marker for every request it receives so the test can assert that a +smuggled request never reaches the origin. +""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import socket +import sys +import threading + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("address", help="Address to listen on.") + parser.add_argument("port", type=int, help="The port to listen on.") + return parser.parse_args() + + +def handle(conn: socket.socket) -> None: + conn.settimeout(30) + buf = b"" + try: + while True: + while b"\r\n\r\n" not in buf: + chunk = conn.recv(65536) + if not chunk: + return + buf += chunk + head, buf = buf.split(b"\r\n\r\n", 1) + lines = head.split(b"\r\n") + request_line = lines[0].decode("latin1", "replace") + parts = request_line.split(" ") + path = parts[1] if len(parts) > 1 else "/" + # Distinctive, greppable marker for the test's assertions. + print(f"ORIGIN_RECV path=[{path}]", flush=True) + + # Drain a declared request body so keep-alive framing stays intact. + content_length = 0 + for h in lines[1:]: + if h.lower().startswith(b"content-length:"): + try: + content_length = int(h.split(b":", 1)[1].strip()) + except ValueError: + content_length = 0 + while len(buf) < content_length: + chunk = conn.recv(65536) + if not chunk: + break + buf += chunk + buf = buf[content_length:] + + body = f"origin-response path={path}\n".encode() + resp = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Cache-Control: public, max-age=300\r\n" + b"Content-Length: %d\r\n\r\n" % len(body)) + body + conn.sendall(resp) + except OSError: + pass + finally: + conn.close() + + +def main() -> int: + args = parse_args() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((args.address, args.port)) + sock.listen(16) + print(f"Listening on {args.address}:{args.port}", flush=True) + while True: + conn, _ = sock.accept() + threading.Thread(target=handle, args=(conn,), daemon=True).start() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/gold_tests/pipeline/noop_keepalive.test.py b/tests/gold_tests/pipeline/noop_keepalive.test.py new file mode 100644 index 00000000000..6d6ea2604de --- /dev/null +++ b/tests/gold_tests/pipeline/noop_keepalive.test.py @@ -0,0 +1,92 @@ +'''Non-idempotency audit: a NOOP self-response drains its body exactly once and keeps the connection alive.''' + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ports import get_port +import sys + +Test.Summary = 'A cache-miss/NOOP DELETE self-response drains its body once and preserves keep-alive for the next request.' +Test.ContinueOnFail = False + + +class TestNoopKeepAlive: + """Audit that the NOOP self-response drain runs exactly once. + + On one keep-alive connection, a NOOP self-response (a DELETE to an uncached + path) with a benign body must drain that body exactly once; a following GET + must then still be served. A double-drain would consume into the next request + and close the connection. + """ + + _server_script: str = 'desync_server.py' + _client_script: str = 'noop_keepalive_client.py' + _hostname: str = 'www.example.com' + + def __init__(self) -> None: + tr = Test.AddTestRun('A NOOP self-response must drain the body exactly once (keep-alive preserved).') + tr.TimeOut = 40 + self._configure_server(tr) + self._configure_traffic_server(tr) + self._configure_client(tr) + + def _configure_server(self, tr: 'TestRun') -> 'Process': + """Configure the origin server. + + :param tr: The test run to associate the origin server with. + :return: The origin server process. + """ + server = tr.Processes.Process('server') + tr.Setup.Copy(self._server_script) + port = get_port(server, 'http_port') + server.Command = f'{sys.executable} {self._server_script} 127.0.0.1 {port}' + server.Ready = When.PortOpenv4(port) + self._server = server + return server + + def _configure_traffic_server(self, tr: 'TestRun') -> 'Process': + """Configure ATS. + + :param tr: The test run to associate the ATS process with. + :return: The ATS process. + """ + ts = tr.MakeATSProcess('ts', enable_cache=True) + self._ts = ts + ts.Disk.remap_config.AddLine(f'map http://{self._hostname}/ http://127.0.0.1:{self._server.Variables.http_port}/') + ts.Disk.records_config.update({'proxy.config.http.cache.http': 1}) + return ts + + def _configure_client(self, tr: 'TestRun') -> 'Process': + """Configure the client. + + :param tr: The test run to associate the client process with. + :return: The client process. + """ + client = tr.Processes.Default + tr.Setup.Copy(self._client_script) + client.Command = f'{sys.executable} {self._client_script} 127.0.0.1 {self._ts.Variables.port} {self._hostname}' + client.ReturnCode = 0 + client.Streams.All += Testers.ContainsExpression( + 'DELETE_STATUS=404', 'the DELETE to an uncached path must self-answer 404 (NOOP path)') + client.Streams.All += Testers.ContainsExpression( + 'SECOND_REQUEST_STATUS=200', 'the following GET / must be served (keep-alive preserved, drained exactly once)') + client.Streams.All += Testers.ContainsExpression( + 'KEEPALIVE_PRESERVED=yes', 'the connection must not be closed by a spurious double-drain') + client.StartBefore(self._server) + client.StartBefore(self._ts) + + +TestNoopKeepAlive() diff --git a/tests/gold_tests/pipeline/noop_keepalive_client.py b/tests/gold_tests/pipeline/noop_keepalive_client.py new file mode 100644 index 00000000000..e28c01754aa --- /dev/null +++ b/tests/gold_tests/pipeline/noop_keepalive_client.py @@ -0,0 +1,57 @@ +"""Double-drain / non-idempotency audit for the NOOP self-response path. +On one keep-alive connection: (1) a DELETE to an uncached path (Max-Forwards:0) with a +benign fully-buffered body self-answers 404 via INTERNAL_CACHE_NOOP and must drain the +body EXACTLY once; (2) a following GET / must still be served. If the NOOP path drained +twice, do_drain_request_body (not idempotent) would stamp Connection: close and the GET +would get nothing.""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import socket +import sys + +addr, port, host = sys.argv[1], int(sys.argv[2]), sys.argv[3] + + +def recv_one(s: socket.socket) -> bytes: + s.settimeout(4) + buf = b"" + try: + while b"\r\n\r\n" not in buf: + c = s.recv(65536) + if not c: + break + buf += c + except socket.timeout: + pass + return buf + + +body = b"xxxxx" +r1 = (f"DELETE /coldpath HTTP/1.1\r\nHost: {host}\r\nMax-Forwards: 0\r\n" + f"Content-Length: {len(body)}\r\n\r\n").encode() + body +r2 = (f"GET / HTTP/1.1\r\nHost: {host}\r\n\r\n").encode() +s = socket.create_connection((addr, port), timeout=8) +s.sendall(r1) +resp1 = recv_one(s) +st1 = (resp1.split(b" ") + [b"?"])[1].decode('latin1', 'replace') if resp1 else "NONE" +s.sendall(r2) +resp2 = recv_one(s) +st2 = (resp2.split(b" ") + [b"?"])[1].decode('latin1', 'replace') if resp2 else "NONE" +print(f"DELETE_STATUS={st1}") +print(f"SECOND_REQUEST_STATUS={st2}") +print(f"KEEPALIVE_PRESERVED={'yes' if resp2 else 'no'}") +s.close() diff --git a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py index bf2a88c57bc..5bf59fb4c82 100644 --- a/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py +++ b/tests/gold_tests/pluginTest/cache_range_requests/cache_range_requests.test.py @@ -159,6 +159,26 @@ server.addResponse("sessionlog.json", req_psd, res_pselect) +# long cache key test: URL path long enough to push cache key over 16384 bytes +long_path = 'A' * 16400 +req_long_key = { + "headers": + "GET /{} HTTP/1.1\r\n".format(long_path) + "Host: www.longkey.com\r\n" + "Accept: */*\r\n" + "Range: bytes=0-17\r\n" + + "uuid: long_key\r\n" + "\r\n", + "timestamp": "1469733493.993", + "body": "" +} + +res_long_key = { + "headers": + "HTTP/1.1 206 Partial Content\r\n" + "Accept-Ranges: bytes\r\n" + "Cache-Control: max-age=500\r\n" + + "Content-Range: bytes 0-17/{}\r\n".format(len(body)) + "Connection: close\r\n" + 'Etag: "longkey"\r\n' + "\r\n", + "timestamp": "1469733493.993", + "body": body +} + +server.addResponse("sessionlog.json", req_long_key, res_long_key) + # cache range requests plugin remap ts.Setup.CopyAs('reason.conf', Test.RunDirectory) ts.Disk.remap_config.AddLines( @@ -166,6 +186,9 @@ 'map http://www.example.com http://127.0.0.1:{}'.format(server.Variables.Port) + ' @plugin=header_rewrite.so @pparam={}/reason.conf @plugin=cache_range_requests.so'.format(Test.RunDirectory), + # long cache key: URL alone exceeds the 16384-byte stack buffer + 'map http://www.longkey.com http://127.0.0.1:{}'.format(server.Variables.Port) + ' @plugin=cache_range_requests.so', + # parent select cache key option 'map http://parentselect http://127.0.0.1:{}'.format(server.Variables.Port) + ' @plugin=cache_range_requests.so @pparam=--ps-cachekey', @@ -330,3 +353,16 @@ ) tr.StillRunningAfter = ts tr.StillRunningAfter = server + +# 13 Test - range request where URL+range_value exceeds 16384 bytes (spill path) +tr = Test.AddTestRun("long cache key spill") +ps = tr.Processes.Default +tr.MakeCurlCommand(curl_and_args + ' http://www.longkey.com/{} -r 0-17 -H "uuid: long_key"'.format(long_path), ts=ts) +ps.ReturnCode = 0 +ps.Streams.stdout.Content = Testers.ContainsExpression("206", "expected 206 for long-key range request") +ts.Disk.diags_log.Content = Testers.ExcludesExpression( + "disabling cache for this transaction", + "spill path must not disable cache", +) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server diff --git a/tests/gold_tests/pluginTest/certifier/certifier.test.py b/tests/gold_tests/pluginTest/certifier/certifier.test.py index cb397fd1fd6..56a5360cf52 100644 --- a/tests/gold_tests/pluginTest/certifier/certifier.test.py +++ b/tests/gold_tests/pluginTest/certifier/certifier.test.py @@ -151,3 +151,110 @@ def run(self): ReuseExistingCertTest().run() + + +class UnsafeSniTest: + httpsReplayFile = "replays/https-path-traversal.replay.yaml" + certPathSrc = os.path.join(Test.TestDirectory, "certs") + escapedCert = "certifier-escaped.crt" + certPathDest = "" + + def __init__(self): + self.setupOriginServer() + self.setupTS() + + def setupOriginServer(self): + self.server = Test.MakeVerifierServerProcess("verifier-server3", self.httpsReplayFile) + + def setupTS(self): + self.ts = Test.MakeATSProcess("ts3", enable_tls=True) + self.ts.addDefaultSSLFiles() + self.certPathDest = os.path.join(self.ts.Variables.CONFIGDIR, "certifier-certs") + Setup.Copy(self.certPathSrc, self.certPathDest) + Setup.MakeDir(os.path.join(self.certPathDest, 'store')) + self.ts.Disk.records_config.update( + { + "proxy.config.diags.debug.enabled": 1, + "proxy.config.diags.debug.tags": "http|certifier|ssl", + "proxy.config.ssl.server.cert.path": f'{self.ts.Variables.SSLDir}', + "proxy.config.ssl.server.private_key.path": f'{self.ts.Variables.SSLDir}', + }) + self.ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + self.ts.Disk.remap_config.AddLine(f"map / http://127.0.0.1:{self.server.Variables.http_port}/",) + self.ts.Disk.plugin_config.AddLine( + f'certifier.so -s {os.path.join(self.certPathDest, "store")} -m 1000 -c {os.path.join(self.certPathDest, "ca.cert")} -k {os.path.join(self.certPathDest, "ca.key")} -r {os.path.join(self.certPathDest, "ca-serial.txt")}' + ) + self.ts.Disk.traffic_out.Content += Testers.ContainsExpression( + "rejecting unsafe SNI for certificate storage", "Should reject SNI values that cannot be used safely as file names.") + + def runHTTPSTraffic(self): + tr = Test.AddTestRun("Test unsafe SNI is rejected") + tr.AddVerifierClientProcess( + "client3", self.httpsReplayFile, http_ports=[self.ts.Variables.port], https_ports=[self.ts.Variables.ssl_port]) + tr.Processes.Default.StartBefore(self.server) + tr.Processes.Default.StartBefore(self.ts) + tr.StillRunningAfter = self.server + tr.StillRunningAfter = self.ts + + def verifyCertNotExist(self, certPath): + tr = Test.AddTestRun("Verify unsafe SNI did not escape the store") + tr.Processes.Default.Command = "echo verify" + tr.Disk.File(certPath, exists=False) + + def run(self): + escapedCertPath = os.path.join(self.certPathDest, self.escapedCert) + self.verifyCertNotExist(escapedCertPath) + self.runHTTPSTraffic() + self.verifyCertNotExist(escapedCertPath) + + +UnsafeSniTest().run() + + +class NoSniTest: + httpsReplayFile = "replays/https-no-sni.replay.yaml" + certPathSrc = os.path.join(Test.TestDirectory, "certs") + certPathDest = "" + + def __init__(self): + self.setupOriginServer() + self.setupTS() + + def setupOriginServer(self): + self.server = Test.MakeVerifierServerProcess("verifier-server4", self.httpsReplayFile) + + def setupTS(self): + self.ts = Test.MakeATSProcess("ts4", enable_tls=True) + self.ts.addDefaultSSLFiles() + self.certPathDest = os.path.join(self.ts.Variables.CONFIGDIR, "certifier-certs") + Setup.Copy(self.certPathSrc, self.certPathDest) + Setup.MakeDir(os.path.join(self.certPathDest, 'store')) + self.ts.Disk.records_config.update( + { + "proxy.config.diags.debug.enabled": 1, + "proxy.config.diags.debug.tags": "http|certifier|ssl", + "proxy.config.ssl.server.cert.path": f'{self.ts.Variables.SSLDir}', + "proxy.config.ssl.server.private_key.path": f'{self.ts.Variables.SSLDir}', + }) + self.ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + self.ts.Disk.remap_config.AddLine(f"map / http://127.0.0.1:{self.server.Variables.http_port}/",) + self.ts.Disk.plugin_config.AddLine( + f'certifier.so -s {os.path.join(self.certPathDest, "store")} -m 1000 -c {os.path.join(self.certPathDest, "ca.cert")} -k {os.path.join(self.certPathDest, "ca.key")} -r {os.path.join(self.certPathDest, "ca-serial.txt")}' + ) + self.ts.Disk.traffic_out.Content += Testers.ContainsExpression( + "no SNI available; using default certificate", "Should continue the handshake when no SNI is available.") + + def runHTTPSTraffic(self): + tr = Test.AddTestRun("Test missing SNI falls back to default cert") + tr.AddVerifierClientProcess( + "client4", self.httpsReplayFile, http_ports=[self.ts.Variables.port], https_ports=[self.ts.Variables.ssl_port]) + tr.Processes.Default.StartBefore(self.server) + tr.Processes.Default.StartBefore(self.ts) + tr.StillRunningAfter = self.server + tr.StillRunningAfter = self.ts + + def run(self): + self.runHTTPSTraffic() + + +NoSniTest().run() diff --git a/tests/gold_tests/pluginTest/certifier/replays/https-no-sni.replay.yaml b/tests/gold_tests/pluginTest/certifier/replays/https-no-sni.replay.yaml new file mode 100644 index 00000000000..6dc3e77d6b5 --- /dev/null +++ b/tests/gold_tests/pluginTest/certifier/replays/https-no-sni.replay.yaml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +sessions: + - protocol: + stack: https + + transactions: + - client-request: + method: "GET" + version: "1.1" + url: "/path/4" + headers: + fields: + - [Host, www.tls.com] + - [Content-Length, 10] + - [uuid, 4] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 16] + + proxy-response: + status: 200 diff --git a/tests/gold_tests/pluginTest/certifier/replays/https-path-traversal.replay.yaml b/tests/gold_tests/pluginTest/certifier/replays/https-path-traversal.replay.yaml new file mode 100644 index 00000000000..33b5c6b9ee9 --- /dev/null +++ b/tests/gold_tests/pluginTest/certifier/replays/https-path-traversal.replay.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +sessions: + - protocol: + stack: https + tls: + sni: "../../certifier-escaped" + + transactions: + - client-request: + method: "GET" + version: "1.1" + url: "/path/4" + headers: + fields: + - [Host, www.tls.com] + - [Content-Length, 10] + - [uuid, 3] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 16] + + proxy-response: + status: 200 diff --git a/tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py b/tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py index 711566a9b70..235a9c482bc 100644 --- a/tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py +++ b/tests/gold_tests/pluginTest/combo_handler/combo_handler.test.py @@ -62,7 +62,7 @@ def tcp_client(host, port, data): server = Test.MakeOriginServer("server") -def add_server_obj(content_type, path): +def add_server_obj(content_type, path, cache_control="public, max-age=31536000"): request_header = { "headers": "GET " + path + " HTTP/1.1\r\n" + "Host: just.any.thing\r\n\r\n", "timestamp": "1469733493.993", @@ -70,9 +70,8 @@ def add_server_obj(content_type, path): } response_header = { "headers": - "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + 'Etag: "359670651"\r\n' + - "Cache-Control: public, max-age=31536000\r\n" + "Accept-Ranges: bytes\r\n" + "Content-Type: " + content_type + "\r\n" + - "\r\n", + "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + 'Etag: "359670651"\r\n' + "Cache-Control: " + cache_control + "\r\n" + + "Accept-Ranges: bytes\r\n" + "Content-Type: " + content_type + "\r\n" + "\r\n", "timestamp": "1469733493.993", "body": "Content for " + path + "\n" } @@ -84,6 +83,18 @@ def add_server_obj(content_type, path): add_server_obj("text/argh", "/obj3") add_server_obj("application/javascript", "/obj4") add_server_obj("application/javascript", "/s/assets/module:variant_v1.js") +# Empty Content-Type header: the field is present but carries no value. +# With an allowlist configured this must be rejected; otherwise it would +# silently bypass the type check. +add_server_obj("", "/obj_empty_ct") +# Exercises CacheControlHeader::update via the refactored +# parse_cache_control_value(): private must propagate to the combo +# response and the smaller max-age must win across objects. +add_server_obj("text/javascript", "/obj_priv_short", cache_control="private, max-age=60") +# max-age=0 is a valid directive ("must revalidate"). The combo must +# propagate it as the minimum, not silently fall back to the 10-year +# default. +add_server_obj("text/javascript", "/obj_revalidate", cache_control="public, max-age=0") ts = Test.MakeATSProcess("ts") @@ -134,4 +145,38 @@ def add_server_obj(content_type, path): f.Content += Testers.ContainsExpression( "Content for /s/assets/module:variant_v1.js", "Should fetch the object path after the first colon") +# An object whose Content-Type header is present but empty must not slip +# past the allowlist. Pairing it with the allowed obj1 ensures the request +# would have succeeded if the empty value were not enforced -- so a 403 +# here is a regression check for the bypass fix. +tr = Test.AddTestRun() +tr.Processes.Default.Command = tcp_client( + "127.0.0.1", ts.Variables.port, + "GET /admin/v1/combo?obj1&obj_empty_ct HTTP/1.1\n" + "Host: xyz\n" + "Connection: close\n" + "\n") +tr.Processes.Default.ReturnCode = 0 +f = tr.Disk.File("_output/4-tr-Default/stream.all.txt") +f.Content = "combo_handler_files/tr3.gold" + +# Combining a long-TTL public object with a short-TTL private object +# must yield a Cache-Control of "max-age=60, private". This exercises +# both parse paths in the refactored parse_cache_control_value(). +tr = Test.AddTestRun() +tr.Processes.Default.Command = tcp_client( + "127.0.0.1", ts.Variables.port, + "GET /admin/v1/combo?obj1&obj_priv_short HTTP/1.1\n" + "Host: xyz\n" + "Connection: close\n" + "\n") +tr.Processes.Default.ReturnCode = 0 +f = tr.Disk.File("_output/5-tr-Default/stream.all.txt") +f.Content = "combo_handler_files/cache_control_aggregation.gold" + +# An object with max-age=0 must drive the combined response down to +# max-age=0 instead of being filtered out as if it were absent. Pairs +# with the long-TTL obj1 to confirm the min-merge respects zero. +tr = Test.AddTestRun() +tr.Processes.Default.Command = tcp_client( + "127.0.0.1", ts.Variables.port, + "GET /admin/v1/combo?obj1&obj_revalidate HTTP/1.1\n" + "Host: xyz\n" + "Connection: close\n" + "\n") +tr.Processes.Default.ReturnCode = 0 +f = tr.Disk.File("_output/6-tr-Default/stream.all.txt") +f.Content = "combo_handler_files/max_age_zero.gold" + ts.Disk.diags_log.Content = Testers.ContainsExpression("ERROR", "Some tests are failure tests") diff --git a/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/cache_control_aggregation.gold b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/cache_control_aggregation.gold new file mode 100644 index 00000000000..a88fecb84ca --- /dev/null +++ b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/cache_control_aggregation.gold @@ -0,0 +1,17 @@ +HTTP/1.1 200 OK +Vary: Accept-Encoding +Last-Modified: `` +Content-Type: text/css ; charset=utf-8 +Cache-Control: max-age=60, private +Date: `` +Age: `` +Transfer-Encoding: chunked +Connection: close +Server: ATS/`` + +2e +Content for /obj1 +Content for /obj_priv_short + +0 + diff --git a/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/max_age_zero.gold b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/max_age_zero.gold new file mode 100644 index 00000000000..9d88f533d95 --- /dev/null +++ b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/max_age_zero.gold @@ -0,0 +1,17 @@ +HTTP/1.1 200 OK +Vary: Accept-Encoding +Last-Modified: `` +Content-Type: text/css ; charset=utf-8 +Cache-Control: max-age=0, Public +Date: `` +Age: `` +Transfer-Encoding: chunked +Connection: close +Server: ATS/`` + +2e +Content for /obj1 +Content for /obj_revalidate + +0 + diff --git a/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr3.gold b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr3.gold new file mode 100644 index 00000000000..133179b94b8 --- /dev/null +++ b/tests/gold_tests/pluginTest/combo_handler/combo_handler_files/tr3.gold @@ -0,0 +1,9 @@ +HTTP/1.1 403 Forbidden +Date: `` +Age: `` +Transfer-Encoding: chunked +Connection: close +Server: ATS/`` + +0 + diff --git a/tests/gold_tests/pluginTest/compress/replay/compress-content-type-params.replay.yaml b/tests/gold_tests/pluginTest/compress/replay/compress-content-type-params.replay.yaml index 4953bc5c6f6..633aa94accc 100644 --- a/tests/gold_tests/pluginTest/compress/replay/compress-content-type-params.replay.yaml +++ b/tests/gold_tests/pluginTest/compress/replay/compress-content-type-params.replay.yaml @@ -71,3 +71,31 @@ sessions: fields: - [ Content-Encoding, { value: gzip, as: equal } ] - [ Content-Length, { value: 223, as: equal } ] + + # C: Verify the compress plugin handles a character with a high bit set correctly. + - client-request: + method: "GET" + version: "1.1" + url: /ignore-params-true/ + headers: + fields: + - [ uuid, C-1] + - [ Host, example.com ] + - [ Accept-Encoding, gzip ] + + server-response: + status: 200 + headers: + fields: + - [ Cache-Control, public;max-age=3600 ] + - [ Content-Type, "application/json\xfe; charset=utf-8" ] + - [ Content-Length, 1024 ] + content: + encoding: plain + size: 1024 + + proxy-response: + status: 200 + headers: + fields: + - [ Content-Type, { value: "application/json\xfe; charset=utf-8", as: equal } ] diff --git a/tests/gold_tests/pluginTest/esi/esi_nested_html_comment.replay.yaml b/tests/gold_tests/pluginTest/esi/esi_nested_html_comment.replay.yaml new file mode 100644 index 00000000000..020f5494a9a --- /dev/null +++ b/tests/gold_tests/pluginTest/esi/esi_nested_html_comment.replay.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sessions: + +- transactions: + + # Origin returns an HTML comment wrapper () around an + # tag. The ESI processor must strip the wrapper, expand the + # inner ESI markup, and produce the expanded body. + - client-request: + method: GET + url: /esi-html-comment.php + version: '1.1' + headers: + fields: + - [ Host, www.example.com ] + - [ Accept, "*/*" ] + + server-response: + status: 200 + headers: + fields: + - [ X-Esi, "1" ] + - [ Cache-Control, "private" ] + - [ Content-Type, "text/html" ] + - [ Connection, "close" ] + - [ Transfer-Encoding, "chunked" ] + content: + encoding: plain + data: "start--end" diff --git a/tests/gold_tests/pluginTest/esi/esi_nested_html_comment.test.py b/tests/gold_tests/pluginTest/esi/esi_nested_html_comment.test.py new file mode 100644 index 00000000000..02462d1b0f3 --- /dev/null +++ b/tests/gold_tests/pluginTest/esi/esi_nested_html_comment.test.py @@ -0,0 +1,156 @@ +''' +Regression test for the ESI comment wrapper handling. + +Covers the sec-035 fix that rejects nested wrappers inside +another . Two scenarios: + + 1. Legitimate single-level wrapper: must still be expanded normally and + the nested-wrapper guard must NOT fire. + + 2. Nested wrapper hidden inside / child_nodes: + must be rejected by the guard and the diags log must record the + "Nested inside " error. This is the + deeper case the prior top-level-only check missed. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify the ESI plugin processes a legitimate single-level wrapper +and rejects a nested hidden inside /. +''' + +Test.SkipUnless(Condition.PluginExists('esi.so'),) + + +class EsiHtmlCommentTest(): + """ + Drive a single request through ATS whose origin response contains an + `` wrapper, and verify the plugin + expands it without tripping the nested-wrapper guard. + """ + + _replay_file: str = "esi_nested_html_comment.replay.yaml" + + def __init__(self, plugin_config: str) -> None: + tr = Test.AddTestRun("ESI single-level wrapper is processed") + self._create_server(tr) + self._create_ats(tr, plugin_config) + self._create_client(tr) + + def _create_server(self, tr: 'TestRun') -> 'Process': + server = tr.AddVerifierServerProcess("server", self._replay_file, other_args='--format "{url}"') + self._server = server + + server.Streams.All += Testers.ContainsExpression( + 'GET /esi-html-comment.php', 'Verify the server received the ESI document request.') + return server + + def _create_ats(self, tr: 'TestRun', plugin_config: str) -> 'Process': + ts = tr.MakeATSProcess("ts") + self._ts = ts + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|plugin_esi|plugin_esi_procesor', + }) + server_port = self._server.Variables.http_port + ts.Disk.remap_config.AddLine(f'map http://www.example.com/ http://127.0.0.1:{server_port}') + ts.Disk.plugin_config.AddLine(plugin_config) + + # The nested-wrapper guard added for sec-035 must NOT fire on a + # legitimate single-level wrapper. + ts.Disk.diags_log.Content = Testers.ExcludesExpression( + r'Nested inside ', 'The nested-wrapper guard must not fire on legitimate input.') + return ts + + def _create_client(self, tr: 'TestRun') -> None: + p = tr.AddVerifierClientProcess( + "client", + self._replay_file, + http_ports=[self._ts.Variables.port], + other_args='--format "{url}" --keys /esi-html-comment.php') + p.ReturnCode = 0 + p.StartBefore(self._server) + p.StartBefore(self._ts) + + # The body must contain the expanded variable from inside the + # wrapper. + p.Streams.stdout += Testers.ContainsExpression('www.example.com', 'Verify the client received the expanded ESI body.') + + +class EsiNestedHtmlCommentRejectTest(): + """ + Drive a request whose origin response contains a nested + hidden inside / child_nodes, and verify the + processor rejects it via the raw-substring guard. The prior fix only + scanned the top level of inner_nodes and would have let this through. + """ + + _replay_file: str = "esi_nested_html_comment_reject.replay.yaml" + + def __init__(self, plugin_config: str) -> None: + tr = Test.AddTestRun("ESI nested wrapper is rejected") + self._create_server(tr) + self._create_ats(tr, plugin_config) + self._create_client(tr) + + def _create_server(self, tr: 'TestRun') -> 'Process': + server = tr.AddVerifierServerProcess("server-reject", self._replay_file, other_args='--format "{url}"') + self._server = server + + server.Streams.All += Testers.ContainsExpression( + 'GET /esi-nested-reject.php', 'Verify the server received the nested-wrapper request.') + return server + + def _create_ats(self, tr: 'TestRun', plugin_config: str) -> 'Process': + ts = tr.MakeATSProcess("ts-reject") + self._ts = ts + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|plugin_esi|plugin_esi_procesor', + }) + server_port = self._server.Variables.http_port + ts.Disk.remap_config.AddLine(f'map http://www.example.com/ http://127.0.0.1:{server_port}') + ts.Disk.plugin_config.AddLine(plugin_config) + + # The guard must fire on a nested wrapper, even when the nested + # is hidden inside / rather + # than appearing at the top level of the outer wrapper's content. + ts.Disk.diags_log.Content = Testers.ContainsExpression( + r'Nested inside is not allowed', + 'The nested-wrapper guard must fire on a nested ESI comment hidden inside /.') + return ts + + def _create_client(self, tr: 'TestRun') -> None: + p = tr.AddVerifierClientProcess( + "client-reject", + self._replay_file, + http_ports=[self._ts.Variables.port], + other_args='--format "{url}" --keys /esi-nested-reject.php') + p.ReturnCode = 0 + p.StartBefore(self._server) + p.StartBefore(self._ts) + # The strong signal that the guard fired is the diags_log + # ContainsExpression in _create_ats. We deliberately do not + # assert on the client body here, since the failure path + # (pass-through vs. error response) is orthogonal to the + # guard and may evolve independently. + + +EsiHtmlCommentTest(plugin_config='esi.so') +EsiNestedHtmlCommentRejectTest(plugin_config='esi.so') diff --git a/tests/gold_tests/pluginTest/esi/esi_nested_html_comment_reject.replay.yaml b/tests/gold_tests/pluginTest/esi/esi_nested_html_comment_reject.replay.yaml new file mode 100644 index 00000000000..30df382b03a --- /dev/null +++ b/tests/gold_tests/pluginTest/esi/esi_nested_html_comment_reject.replay.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sessions: + +- transactions: + + # Origin returns an outer whose content includes a nested + # hidden inside /. The top-level scan + # in the previous fix missed this case because the nested wrapper lives in + # attempt_node->child_nodes; this test locks in the deeper-nesting reject. + - client-request: + method: GET + url: /esi-nested-reject.php + version: '1.1' + headers: + fields: + - [ Host, www.example.com ] + - [ Accept, "*/*" ] + + server-response: + status: 200 + headers: + fields: + - [ X-Esi, "1" ] + - [ Cache-Control, "private" ] + - [ Content-Type, "text/html" ] + - [ Connection, "close" ] + - [ Transfer-Encoding, "chunked" ] + content: + encoding: plain + data: "start-fallback -->-end" diff --git a/tests/gold_tests/pluginTest/esi/esi_request_size_cap.test.py b/tests/gold_tests/pluginTest/esi/esi_request_size_cap.test.py new file mode 100644 index 00000000000..cf796a39039 --- /dev/null +++ b/tests/gold_tests/pluginTest/esi/esi_request_size_cap.test.py @@ -0,0 +1,91 @@ +''' +Test the ESI plugin's HTTP fetch request size cap. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify HttpDataFetcherImpl rejects ESI include fetches whose total HTTP +request size (request line + URL + forwarded headers) would exceed +MAX_REQ_LEN (32 KB), logging an error rather than allocating a buffer +sized by an overflowed size_t. +''' + +Test.SkipUnless(Condition.PluginExists('esi.so'),) + +# Matches MAX_REQ_LEN in plugins/esi/fetcher/HttpDataFetcherImpl.cc. +MAX_REQ_LEN = 32 * 1024 + +# total_len in addFetchRequest is: +# sizeof("GET ") - 1 -> 4 +# + url.length() +# + sizeof(" HTTP/1.0\r\n") - 1 -> 11 +# + _headers_str.length() +# + sizeof("\r\n") - 1 -> 2 +# A path one byte longer than MAX_REQ_LEN guarantees the cap is tripped +# regardless of how many headers end up being forwarded. +oversized_path = 'A' * (MAX_REQ_LEN + 1) +esi_body = ('\n\n' + f'Hello, \n' + '\n\n') + +server = Test.MakeOriginServer("server") +server.addResponse( + "sessionfile.log", { + "headers": ("GET /oversized.php HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Content-Length: 0\r\n\r\n"), + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": + ( + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/html\r\n" + "X-Esi: 1\r\n" + "Connection: close\r\n" + f"Content-Length: {len(esi_body)}\r\n" + "Cache-Control: max-age=300\r\n\r\n"), + "timestamp": "1469733493.993", + "body": esi_body + }) + +ts = Test.MakeATSProcess("ts") +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|plugin_esi', +}) +ts.Disk.remap_config.AddLine(f'map http://www.example.com/ http://127.0.0.1:{server.Variables.Port}') +ts.Disk.plugin_config.AddLine('esi.so') + +tr = Test.AddTestRun("Start the server and ATS.") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Command = "echo starting" +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts + +tr = Test.AddTestRun("Issue a request whose ESI include URL exceeds the 32 KB fetch cap.") +tr.MakeCurlCommand( + f'http://127.0.0.1:{ts.Variables.port}/oversized.php ' + '-H"Host: www.example.com" -H"Accept: */*" --output /dev/null --silent', + ts=ts) +tr.Processes.Default.ReturnCode = 0 +ts.Disk.diags_log.Content = Testers.ContainsExpression( + r"HTTP request size exceeds maximum 32768", "ESI fetcher must log the MAX_REQ_LEN cap error for oversize requests") +tr.StillRunningAfter = server +tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/esi/esi_ssrf_validate.replay.yaml b/tests/gold_tests/pluginTest/esi/esi_ssrf_validate.replay.yaml new file mode 100644 index 00000000000..9a393b38067 --- /dev/null +++ b/tests/gold_tests/pluginTest/esi/esi_ssrf_validate.replay.yaml @@ -0,0 +1,164 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Exercises the SSRF validator added to the ESI plugin. Each top-level +# document below contains an esi:include whose src= is either rejected by +# the validator (cloud-metadata IP literal, loopback IP literal, bad +# scheme, attacker-controlled variable expansion) or permitted (the +# upstream host configured in remap.config). On rejection, the include +# tag is left as-is in the served body; on acceptance, ATS fetches the +# snippet and inlines it. +# + +sessions: + +- transactions: + + # 1. Reject: cloud-metadata IPv4 literal (link-local 169.254/16). + - client-request: + method: GET + url: /metadata.php + version: '1.1' + headers: + fields: + - [ Host, www.example.com ] + + server-response: + status: 200 + headers: + fields: + - [ X-Esi, "1" ] + - [ Content-Type, "text/html" ] + - [ Connection, "close" ] + - [ Transfer-Encoding, "chunked" ] + content: + encoding: plain + data: "\n" + + # 2. Reject: loopback IPv4 literal. + - client-request: + method: GET + url: /loopback.php + version: '1.1' + headers: + fields: + - [ Host, www.example.com ] + + server-response: + status: 200 + headers: + fields: + - [ X-Esi, "1" ] + - [ Content-Type, "text/html" ] + - [ Connection, "close" ] + - [ Transfer-Encoding, "chunked" ] + content: + encoding: plain + data: "\n" + + # 3. Reject: non-http(s) scheme. + - client-request: + method: GET + url: /badscheme.php + version: '1.1' + headers: + fields: + - [ Host, www.example.com ] + + server-response: + status: 200 + headers: + fields: + - [ X-Esi, "1" ] + - [ Content-Type, "text/html" ] + - [ Connection, "close" ] + - [ Transfer-Encoding, "chunked" ] + content: + encoding: plain + data: "\n" + + # 4. Reject: attacker-influenced variable expansion. The upstream + # template uses $(HTTP_HEADER{x-esi-target}) to compose the host, + # and the client supplies a private-IP value in that header. + # Validation runs post-expansion so the resulting URL is rejected. + # + # The header name is lowercase on purpose: ESI's HTTP_HEADER{...} + # lookup is case-sensitive (see plugins/esi/lib/Variables.cc), and + # proxy-verifier / ATS may normalize wire header names. Using a + # name that is already canonical removes the ambiguity. + - client-request: + method: GET + url: /varinject.php + version: '1.1' + headers: + fields: + - [ Host, www.example.com ] + - [ x-esi-target, "10.0.0.5" ] + + server-response: + status: 200 + headers: + fields: + - [ X-Esi, "1" ] + - [ Content-Type, "text/html" ] + - [ Connection, "close" ] + - [ Transfer-Encoding, "chunked" ] + content: + encoding: plain + data: "\n" + + # 5. Accept: ordinary include to the upstream host configured via + # remap.config. The snippet must be fetched and inlined. + - client-request: + method: GET + url: /allowed.php + version: '1.1' + headers: + fields: + - [ Host, www.example.com ] + + server-response: + status: 200 + headers: + fields: + - [ X-Esi, "1" ] + - [ Content-Type, "text/html" ] + - [ Connection, "close" ] + - [ Transfer-Encoding, "chunked" ] + content: + encoding: plain + data: "before::after\n" + + # 6. The fetched snippet for case 5. + - client-request: + method: GET + url: /snippet.html + version: '1.1' + headers: + fields: + - [ Host, www.example.com ] + + server-response: + status: 200 + headers: + fields: + - [ Content-Type, "text/html" ] + - [ Connection, "close" ] + - [ Transfer-Encoding, "chunked" ] + content: + encoding: plain + data: "SNIPPET-OK" diff --git a/tests/gold_tests/pluginTest/esi/esi_ssrf_validate.test.py b/tests/gold_tests/pluginTest/esi/esi_ssrf_validate.test.py new file mode 100644 index 00000000000..ef922920605 --- /dev/null +++ b/tests/gold_tests/pluginTest/esi/esi_ssrf_validate.test.py @@ -0,0 +1,111 @@ +''' +Test SSRF validation for ESI include src= URLs. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify the ESI plugin's SSRF guard rejects include URLs with private-IP +hosts, non-http(s) schemes, and attacker-controlled variable expansion, +while still allowing ordinary includes to the configured upstream. +''' + +Test.SkipUnless(Condition.PluginExists('esi.so'),) + + +class EsiSsrfTest: + """Drives the replay file through ATS with the SSRF guard enabled and + confirms each rejection path logs the expected reason while the + allowed include is still fetched and inlined.""" + + _replay_file: str = "esi_ssrf_validate.replay.yaml" + + def __init__(self) -> None: + tr = Test.AddTestRun("ESI include URLs are validated for SSRF") + self._create_server(tr) + self._create_ats(tr) + self._create_client(tr) + + def _create_server(self, tr: 'TestRun') -> None: + server = tr.AddVerifierServerProcess("server", self._replay_file, other_args='--format "{url}"') + self._server = server + + # The snippet for the allowed-include case must reach the origin. + # The rejected cases must never reach any backend other than the + # five top-level documents themselves; we don't assert their + # absence on the server stream because the rejected hosts are + # not mapped to this verifier. + server.Streams.All += Testers.ContainsExpression('GET /allowed.php', 'Verify the allowed top-level request reached origin.') + server.Streams.All += Testers.ContainsExpression( + 'GET /snippet.html', 'Verify the snippet for the allowed include was fetched.') + + def _create_ats(self, tr: 'TestRun') -> None: + ts = tr.MakeATSProcess("ts") + self._ts = ts + ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|plugin_esi', + }) + + server_port = self._server.Variables.http_port + ts.Disk.remap_config.AddLine(f'map http://www.example.com/ http://127.0.0.1:{server_port}') + + # Default-deny private/loopback hosts; no allow-regex configured, + # so any non-private host is permitted (the upstream maps via + # www.example.com). + ts.Disk.plugin_config.AddLine('esi.so') + + # Each rejection must be logged with its reason. Reason strings + # come from IncludeUrlValidator::reasonString(). + # TSError emits one line per rejection that contains both the + # offending URL substring and the reason token. Match each pair + # loosely so the assertion survives bracket/paren formatting + # differences in the diags log. ``.`` is newline-bounded by + # default, so each regex stays single-line. + ts.Disk.diags_log.Content = Testers.ContainsExpression( + r'Rejecting include URL.*169\.254\.169\.254.*private-host', + 'Cloud-metadata IPv4 literal must be rejected as private-host.') + ts.Disk.diags_log.Content += Testers.ContainsExpression( + r'Rejecting include URL.*127\.0\.0\.1/admin.*private-host', 'Loopback IPv4 literal must be rejected as private-host.') + ts.Disk.diags_log.Content += Testers.ContainsExpression( + r'Rejecting include URL.*gopher://internal\.svc/x.*bad-scheme', 'Non-http(s) scheme must be rejected as bad-scheme.') + ts.Disk.diags_log.Content += Testers.ContainsExpression( + r'Rejecting include URL.*10\.0\.0\.5/secret.*private-host', + 'Attacker-influenced variable expansion must be rejected after \\$\\(...\\) is expanded.') + + # The allowed case must NOT show up as a rejection. + ts.Disk.diags_log.Content += Testers.ExcludesExpression( + r'Rejecting include URL.*www\.example\.com/snippet\.html', + 'The legitimate include to the mapped upstream must not be rejected.') + + def _create_client(self, tr: 'TestRun') -> None: + # Proxy Verifier's default --keys format in this codebase keys + # transactions by URL (see esi_nested_include.test.py), not by the + # `uuid` request header. Match on the top-level document URLs so + # the client drives the four rejection cases and the one allowed + # case; the /snippet.html transaction is omitted because ATS + # fetches it internally via the ESI include. + p = tr.AddVerifierClientProcess( + "client", + self._replay_file, + http_ports=[self._ts.Variables.port], + other_args='--format "{url}" --keys /metadata.php /loopback.php /badscheme.php /varinject.php /allowed.php') + p.ReturnCode = 0 + p.StartBefore(self._server) + p.StartBefore(self._ts) + + +EsiSsrfTest() diff --git a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_bundle.replay.yaml b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_bundle.replay.yaml index af07d37972e..c67ad080b6a 100644 --- a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_bundle.replay.yaml +++ b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_bundle.replay.yaml @@ -183,6 +183,20 @@ autest: args: - "rules/set_status_in_if.conf" + - from: "http://www.example.com/from_19/" + to: "http://backend.ex:{SERVER_HTTP_PORT}/to_19/" + plugins: + - name: "header_rewrite.so" + args: + - "rules/rule_cookie.conf" + + - from: "http://www.example.com/from_20/" + to: "http://backend.ex:{SERVER_HTTP_PORT}/to_20/" + plugins: + - name: "header_rewrite.so" + args: + - "rules/rule_cidr.conf" + metric_checks: - metric: "proxy.process.plugin.header_rewrite.operators" min: 1 @@ -1894,11 +1908,11 @@ sessions: - [ X-Server-Host-Header, { value: "backend.ex", as: contains } ] - [ X-Path-Match, { value: "Yes", as: equal } ] -# Test 64: SESSION-FLAG persists across keep-alive transactions. -# The first request sees the flag unset; set-session-flag marks it true. -# The second request on the same keep-alive session sees the flag set. +# Test 64: STATE (txn) and SESSION (ssn) state variables across a keep-alive +# session -- the read paths hardened by the issue #143 fix. SESSION-* persists: +# request 1 sees it unset, request 2 sees it set. - transactions: - # First transaction - flag is unset, header absent; flag is then set. + # Request 1: session vars unset; STATE set+read within the transaction. - client-request: method: "GET" version: "1.1" @@ -1922,9 +1936,15 @@ sessions: status: 200 headers: fields: + - [ X-State-Flag, { value: "set", as: equal } ] + - [ X-State-Int8, { value: "42", as: equal } ] + - [ X-State-Int16, { value: "4242", as: equal } ] - [ X-Session-Seen, { as: absent } ] + - [ X-Session-Int8, { value: "0", as: equal } ] + - [ X-Session-Int8-Seen, { as: absent } ] + - [ X-Session-Int16, { value: "0", as: equal } ] - # Second transaction - flag set by previous request; header must appear. + # Second transaction on the same keep-alive session - session vars persist. - client-request: method: "GET" version: "1.1" @@ -1948,7 +1968,13 @@ sessions: status: 200 headers: fields: + - [ X-State-Flag, { value: "set", as: equal } ] + - [ X-State-Int8, { value: "42", as: equal } ] + - [ X-State-Int16, { value: "4242", as: equal } ] - [ X-Session-Seen, { value: "yes", as: equal } ] + - [ X-Session-Int8, { value: "100", as: equal } ] + - [ X-Session-Int8-Seen, { value: "yes", as: equal } ] + - [ X-Session-Int16, { value: "4242", as: equal } ] # Test 67: set-status inside if/endif at REMAP_PSEUDO_HOOK time. # Before the fix, the set-status operator inside an if/endif block retained @@ -1973,3 +1999,347 @@ sessions: proxy-response: status: 403 + + # Test 78: set-cookie preserves the ';' separator when the rewritten cookie + # is followed by "; " in the Cookie value (was dropped before the fix). +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/sep + headers: + fields: + - [ Host, www.example.com ] + - [ Cookie, "X-Test=old; Other=keep" ] + - [ uuid, 78 ] + + proxy-request: + headers: + fields: + - [ Cookie, { value: "X-Test=newval; Other=keep", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + + # Test 79: a configured cookie key longer than the trailing token must not + # read past the non-NUL-terminated Cookie value; the key is not matched and + # set-cookie appends the pair at the end. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/oob + headers: + fields: + - [ Host, www.example.com ] + - [ Cookie, "sessio" ] + - [ uuid, 79 ] + + proxy-request: + headers: + fields: + - [ Cookie, { value: "sessio;session=newval", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + + # Test 80: rm-cookie removes a cookie from the middle of the list. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/rm_mid + headers: + fields: + - [ Host, www.example.com ] + - [ Cookie, "a=1; b=2; c=3" ] + - [ uuid, 80 ] + + proxy-request: + headers: + fields: + - [ Cookie, { value: "a=1; c=3", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + + # Test 81: rm-cookie removes the last cookie, dropping the preceding separator. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/rm_last + headers: + fields: + - [ Host, www.example.com ] + - [ Cookie, "a=1; b=2; c=3" ] + - [ uuid, 81 ] + + proxy-request: + headers: + fields: + - [ Cookie, { value: "a=1; b=2", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + + # Test 82: rm-cookie removes the only cookie, so the field is destroyed. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/rm_only + headers: + fields: + - [ Host, www.example.com ] + - [ Cookie, "a=1" ] + - [ uuid, 82 ] + + proxy-request: + headers: + fields: + - [ Cookie, { as: absent } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + + # Test 83: rm-cookie on an absent key is a no-op. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/rm_absent + headers: + fields: + - [ Host, www.example.com ] + - [ Cookie, "a=1; b=2" ] + - [ uuid, 83 ] + + proxy-request: + headers: + fields: + - [ Cookie, { value: "a=1; b=2", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + + # Test 84: add-cookie appends a key that is not already present. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/add_new + headers: + fields: + - [ Host, www.example.com ] + - [ Cookie, "a=1; b=2" ] + - [ uuid, 84 ] + + proxy-request: + headers: + fields: + - [ Cookie, { value: "a=1; b=2;newk=newv", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + + # Test 85: add-cookie is a no-op when the key already exists. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/add_exist + headers: + fields: + - [ Host, www.example.com ] + - [ Cookie, "a=1; b=2" ] + - [ uuid, 85 ] + + proxy-request: + headers: + fields: + - [ Cookie, { value: "a=1; b=2", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + + # Test 86: set-cookie creates the Cookie field when the request has none. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/set_nohdr + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, 86 ] + + proxy-request: + headers: + fields: + - [ Cookie, { value: "foo=bar", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + + # Test 87: %{COOKIE:name} extracts a present cookie's value. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/getfoo + headers: + fields: + - [ Host, www.example.com ] + - [ Cookie, "foo=bar; baz=qux" ] + - [ uuid, 87 ] + + proxy-request: + headers: + fields: + - [ X-Cookie-Foo, { value: "bar", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + + # Test 88: %{COOKIE:name} with a key longer than the trailing token must not + # over-read; extraction yields the empty string. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_19/shortkey + headers: + fields: + - [ Host, www.example.com ] + - [ Cookie, "sessio" ] + - [ uuid, 88 ] + + proxy-request: + headers: + fields: + - [ X-Cookie-Sess, { value: "empty", as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + + proxy-response: + status: 200 + +# Test 89: CIDR condition - IPv4 masking of the 127.0.0.1 client address. +# /32 keeps the full address, /24 masks the last octet, /0 collapses to +# 0.0.0.0 (also guards the former shift-by-32 UB). The two-mask form exercises +# the IPv6 field parsing; ',40' (empty IPv4 field) parses as a /0 IPv4 mask. +- transactions: + - client-request: + method: "GET" + version: "1.1" + url: /from_20/ + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, 89 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Server, microserver ] + - [ Content-Length, "0" ] + - [ Connection, close ] + + proxy-response: + status: 200 + headers: + fields: + - [ X-Cidr-32, { value: "127.0.0.1", as: equal } ] + - [ X-Cidr-24, { value: "127.0.0.0", as: equal } ] + - [ X-Cidr-0, { value: "0.0.0.0", as: equal } ] + - [ X-Cidr-Two-Mask, { value: "127.0.0.0", as: equal } ] + - [ X-Cidr-Empty-V4, { value: "0.0.0.0", as: equal } ] diff --git a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_set_body_from.test.py b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_set_body_from.test.py index f4c47e6edb6..bb03d7c3e60 100644 --- a/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_set_body_from.test.py +++ b/tests/gold_tests/pluginTest/header_rewrite/header_rewrite_set_body_from.test.py @@ -57,6 +57,19 @@ def setUpOriginServer(self): success_2_response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "body": "Custom body found\n"} self.server.addResponse("sessionfile.log", success_2_request_header, success_2_response_header) + # Request/response for original transaction that triggers binary set-body-from + remap_binary_request_header = {"headers": "GET /remap_binary HTTP/1.1\r\nHost: www.example.com\r\n\r\n"} + self.server.addResponse("sessionfile.log", remap_binary_request_header, response_header) + + # Response for the set-body-from fetch: body has an internal NUL. The strdup + # bug truncated everything after the NUL and emitted heap garbage instead. + binary_body_request_header = {"headers": "GET /binary_body HTTP/1.1\r\nHost: www.example.com\r\n\r\n"} + binary_body_response_header = { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", + "body": "BeforeNUL\x00AfterNUL_AAAAAAAAAAAAAAAAAAAAAAAA" + } + self.server.addResponse("sessionfile.log", binary_body_request_header, binary_body_response_header) + def setUpTS(self): self.ts = Test.MakeATSProcess("ts") @@ -69,6 +82,8 @@ def setUpTS(self): map http://www.example.com/remap_success http://127.0.0.1:{0}/remap_success @plugin=header_rewrite.so @pparam={1}/rule_set_body_from_remap.conf map http://www.example.com/200 http://127.0.0.1:{0}/200 @plugin=header_rewrite.so @pparam={1}/rule_set_body_from_remap.conf map http://www.example.com/remap_fail http://127.0.0.1:{0}/remap_fail @plugin=header_rewrite.so @pparam={1}/rule_set_body_from_remap.conf + map http://www.example.com/remap_binary http://127.0.0.1:{0}/remap_binary @plugin=header_rewrite.so @pparam={1}/rule_set_body_from_remap.conf + map http://www.example.com/binary_body http://127.0.0.1:{0}/binary_body map http://www.example.com/plugin_success http://127.0.0.1:{0}/plugin_success map http://www.example.com/plugin_fail http://127.0.0.1:{0}/plugin_fail map http://www.example.com/404.html http://127.0.0.1:{0}/404.html @@ -146,12 +161,41 @@ def test_setBodyFromSucceeds_200(self): tr.Processes.Default.Streams.stderr.Content = Testers.ContainsExpression("500 INKApi Error", "Expected 500 response") tr.StillRunningAfter = self.server + def test_setBodyFromBinary(self): + ''' + set-body-from must preserve binary bodies that contain internal NUL + bytes. The previous TSstrdup path truncated at the first NUL and + emitted heap bytes for the remainder; check the SHA256 of the body + the client actually receives. + ''' + body_path = f"{Test.RunDirectory}/binary_body.out" + # SHA256 of b"BeforeNUL\x00AfterNUL_AAAAAAAAAAAAAAAAAAAAAAAA" (43 bytes) + expected_sha = "a529bbd61061b739b611ca67a7b76fc433b3d3a9cc3bcc2e385b812f00b0fe63" + + tr = Test.AddTestRun() + tr.MakeCurlCommand( + f'-s --proxy 127.0.0.1:{self.ts.Variables.port} -o {body_path} "http://www.example.com/remap_binary"', ts=self.ts) + tr.Processes.Default.ReturnCode = 0 + tr.StillRunningAfter = self.server + + # Compute the hash via Python instead of sha256sum/shasum so the + # check does not depend on which CLI hash tool the host happens + # to ship. + tr = Test.AddTestRun() + tr.Processes.Default.Command = ( + f'python3 -c \'import hashlib; ' + f'print(hashlib.sha256(open("{body_path}", "rb").read()).hexdigest())\'') + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression( + expected_sha, "Client must receive the exact binary body bytes") + def runTraffic(self): self.test_setBodyFromFails_remap() self.test_setBodyFromSucceeds_remap() self.test_setBodyFromSucceeds_plugin() self.test_setBodyFromFails_plugin() self.test_setBodyFromSucceeds_200() + self.test_setBodyFromBinary() def run(self): self.runTraffic() diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/rule_cidr.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_cidr.conf new file mode 100644 index 00000000000..e4d3e1c5c28 --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_cidr.conf @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exercise %{CIDR:...}. The verifier client is 127.0.0.1, so only IPv4 masking +# is observable here; IPv6 masking is covered by the unit tests. +cond %{SEND_RESPONSE_HDR_HOOK} + set-header X-Cidr-32 %{CIDR:32} + set-header X-Cidr-24 %{CIDR:24} + set-header X-Cidr-0 %{CIDR:0} + set-header X-Cidr-Two-Mask %{CIDR:24,60} + set-header X-Cidr-Empty-V4 %{CIDR:,40} diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/rule_cookie.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_cookie.conf new file mode 100644 index 00000000000..ead29fe1b9e --- /dev/null +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_cookie.conf @@ -0,0 +1,68 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cookie operator and condition coverage. Each block is gated by the request +# path so exactly one operation runs per request. + +# set-cookie preserves the ';' separator when the rewritten cookie is followed +# by "; " in the Cookie value. +cond %{CLIENT-URL:PATH} /^from_19\/sep/ + set-cookie X-Test newval + +# A configured key longer than the trailing cookie token must not read past the +# (non NUL-terminated) Cookie value; the key is not matched and set-cookie falls +# through to appending the pair at the end. +cond %{CLIENT-URL:PATH} /^from_19\/oob/ + set-cookie session newval + +# rm-cookie removes a cookie from the middle of the list. +cond %{CLIENT-URL:PATH} /^from_19\/rm_mid/ + rm-cookie b + +# rm-cookie removes the last cookie, dropping the preceding separator. +cond %{CLIENT-URL:PATH} /^from_19\/rm_last/ + rm-cookie c + +# rm-cookie removes the only cookie, leaving the field empty so it is destroyed. +cond %{CLIENT-URL:PATH} /^from_19\/rm_only/ + rm-cookie a + +# rm-cookie on an absent key is a no-op. +cond %{CLIENT-URL:PATH} /^from_19\/rm_absent/ + rm-cookie zzz + +# add-cookie appends a key that is not already present. +cond %{CLIENT-URL:PATH} /^from_19\/add_new/ + add-cookie newk newv + +# add-cookie is a no-op when the key already exists. +cond %{CLIENT-URL:PATH} /^from_19\/add_exist/ + add-cookie a X + +# set-cookie creates the Cookie field when the request has none. +cond %{CLIENT-URL:PATH} /^from_19\/set_nohdr/ + set-cookie foo bar + +# %{COOKIE:name} extracts a present cookie's value. +cond %{CLIENT-URL:PATH} /^from_19\/getfoo/ + set-header X-Cookie-Foo "%{COOKIE:foo}" + +# %{COOKIE:name} with a key longer than the trailing token must not over-read; +# extraction yields the empty string. +cond %{CLIENT-URL:PATH} /^from_19\/shortkey/ [AND] +cond %{COOKIE:session} ="" + set-header X-Cookie-Sess "empty" diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/rule_session_vars.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_session_vars.conf index d4b13761951..76e51dcbf7b 100644 --- a/tests/gold_tests/pluginTest/header_rewrite/rules/rule_session_vars.conf +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_session_vars.conf @@ -15,12 +15,45 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Test SESSION-FLAG: on the first request the flag is unset so the header is -# absent; set-session-flag then marks it true. On the second keep-alive -# request the flag is still true, so X-Session-Seen is added. +# State-variable coverage for the issue #143 fix: txn (STATE-*) and session +# (SESSION-*) flag/int8/int16, via both eval and value-expansion reads. + +# Txn STATE: set at remap, read on the response of the same transaction. +set-state-flag 2 true +set-state-int8 1 42 +set-state-int16 0 4242 + +cond %{SEND_RESPONSE_HDR_HOOK} [AND] +cond %{STATE-FLAG:2} =TRUE + set-header X-State-Flag "set" + +cond %{SEND_RESPONSE_HDR_HOOK} + set-header X-State-Int8 "%{STATE-INT8:1}" + +cond %{SEND_RESPONSE_HDR_HOOK} + set-header X-State-Int16 "%{STATE-INT16}" + +# Session reads first (so request 1 of a keep-alive session sees unset)... cond %{SEND_RESPONSE_HDR_HOOK} [AND] cond %{SESSION-FLAG:0} =TRUE set-header X-Session-Seen "yes" +cond %{SEND_RESPONSE_HDR_HOOK} + set-header X-Session-Int8 "%{SESSION-INT8:2}" + +cond %{SEND_RESPONSE_HDR_HOOK} [AND] +cond %{SESSION-INT8:2} =100 + set-header X-Session-Int8-Seen "yes" + +cond %{SEND_RESPONSE_HDR_HOOK} + set-header X-Session-Int16 "%{SESSION-INT16}" + +# ...then the writes, which persist to later requests on the same session. cond %{SEND_RESPONSE_HDR_HOOK} set-session-flag 0 true + +cond %{SEND_RESPONSE_HDR_HOOK} + set-session-int8 2 100 + +cond %{SEND_RESPONSE_HDR_HOOK} + set-session-int16 0 4242 diff --git a/tests/gold_tests/pluginTest/header_rewrite/rules/rule_set_body_from_remap.conf b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_set_body_from_remap.conf index 351e17f7b8d..6c62362f9e1 100644 --- a/tests/gold_tests/pluginTest/header_rewrite/rules/rule_set_body_from_remap.conf +++ b/tests/gold_tests/pluginTest/header_rewrite/rules/rule_set_body_from_remap.conf @@ -22,3 +22,7 @@ set-body-from http://www.example.com/404.html cond %{READ_RESPONSE_HDR_HOOK} cond %{CLIENT-URL:PATH} = "remap_fail" set-body-from http://www.example.com/fail + +cond %{READ_RESPONSE_HDR_HOOK} +cond %{CLIENT-URL:PATH} = "remap_binary" +set-body-from http://www.example.com/binary_body diff --git a/tests/gold_tests/pluginTest/lua/fetch_ipv6_cliaddr.lua b/tests/gold_tests/pluginTest/lua/fetch_ipv6_cliaddr.lua new file mode 100644 index 00000000000..96643eae034 --- /dev/null +++ b/tests/gold_tests/pluginTest/lua/fetch_ipv6_cliaddr.lua @@ -0,0 +1,53 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +function send_response() + if ts.ctx['fetch_status'] ~= nil then + ts.client_response.header['Sub-Status'] = ts.ctx['fetch_status'] + end + if ts.ctx['fetch_body_len'] ~= nil then + ts.client_response.header['Sub-Body-Len'] = ts.ctx['fetch_body_len'] + end +end + +function post_remap() + local inner = ts.http.is_internal_request() + if inner ~= 0 then + return 0 + end + + local url = string.format('http://%s/inner.txt', ts.ctx['host']) + local res = ts.fetch(url, { + method = 'GET', + cliaddr = '[::1]:33333', + header = { ['Host'] = ts.ctx['host'] }, + }) + if res then + ts.ctx['fetch_status'] = res.status + ts.ctx['fetch_body_len'] = string.len(res.body or '') + end +end + +function do_remap() + local inner = ts.http.is_internal_request() + if inner ~= 0 then + return 0 + end + + ts.ctx['host'] = ts.client_request.header['Host'] + ts.hook(TS_LUA_HOOK_POST_REMAP, post_remap) + ts.hook(TS_LUA_HOOK_SEND_RESPONSE_HDR, send_response) +end diff --git a/tests/gold_tests/pluginTest/lua/lua_fetch_ipv6_cliaddr.test.py b/tests/gold_tests/pluginTest/lua/lua_fetch_ipv6_cliaddr.test.py new file mode 100644 index 00000000000..35c53755ed0 --- /dev/null +++ b/tests/gold_tests/pluginTest/lua/lua_fetch_ipv6_cliaddr.test.py @@ -0,0 +1,59 @@ +''' +Verify ts.fetch handles an IPv6 cliaddr option. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify ts.fetch parses an IPv6 cliaddr and the inner fetch completes. +''' + +Test.SkipUnless(Condition.PluginExists('tslua.so'),) + +Test.ContinueOnFail = True + +ts = Test.MakeATSProcess("ts") +server = Test.MakeOriginServer("server") + +# Inner sub-request fetched by the Lua post_remap hook. +inner_req = {"headers": "GET /inner.txt HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +inner_resp = { + "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 5\r\n\r\n", + "timestamp": "1469733493.993", + "body": "AAAAA" +} +server.addResponse("sessionfile.log", inner_req, inner_resp) + +# Outer request driven by curl below. +outer_req = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +outer_resp = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "outer"} +server.addResponse("sessionfile.log", outer_req, outer_resp) + +ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{}/'.format(server.Variables.Port) + ' @plugin=tslua.so @pparam=fetch_ipv6_cliaddr.lua') + +ts.Setup.Copy("fetch_ipv6_cliaddr.lua", ts.Variables.CONFIGDIR) + +ts.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'ts_lua'}) + +tr = Test.AddTestRun("ts.fetch with IPv6 cliaddr") +tr.MakeCurlCommand("-s -D - http://127.0.0.1:{0}/".format(ts.Variables.port), ts=ts) +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression( + "Sub-Body-Len: 5", "Inner fetch using IPv6 cliaddr should return the full origin body") +tr.StillRunningAfter = server diff --git a/tests/gold_tests/pluginTest/lua/lua_remap_after_hook.test.py b/tests/gold_tests/pluginTest/lua/lua_remap_after_hook.test.py new file mode 100644 index 00000000000..eeba08880a6 --- /dev/null +++ b/tests/gold_tests/pluginTest/lua/lua_remap_after_hook.test.py @@ -0,0 +1,55 @@ +''' +Verify ts.remap.* APIs respect their documented do_remap-only context. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +The ts.remap.* family is documented as context: do_remap. Verify that +calling one (ts.remap.get_from_url_host) from a transaction hook +registered during do_remap returns nil and does not crash, while the +same call inside do_remap returns the from-URL host. +''' + +Test.SkipUnless(Condition.PluginExists('tslua.so'),) + +Test.ContinueOnFail = True + +ts = Test.MakeATSProcess("ts") +server = Test.MakeOriginServer("server") + +req = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} +resp = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok"} +server.addResponse("sessionfile.log", req, resp) + +ts.Disk.remap_config.AddLine( + 'map http://www.example.com/ http://127.0.0.1:{}/'.format(server.Variables.Port) + + ' @plugin=tslua.so @pparam=remap_after_hook.lua') + +ts.Setup.Copy("remap_after_hook.lua", ts.Variables.CONFIGDIR) + +ts.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'ts_lua'}) + +tr = Test.AddTestRun("ts.remap.* in do_remap and from a txn hook") +tr.MakeCurlCommand("-s -D - -H 'Host: www.example.com' http://127.0.0.1:{0}/".format(ts.Variables.port), ts=ts) +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression( + "Remap-From-Host: www.example.com", "ts.remap.* should return the from-URL host inside do_remap") +tr.Processes.Default.Streams.stdout.Content += Testers.ContainsExpression( + "Hook-From-Host: ", "ts.remap.* should return nil when called outside do_remap") +tr.StillRunningAfter = server diff --git a/tests/gold_tests/pluginTest/lua/remap_after_hook.lua b/tests/gold_tests/pluginTest/lua/remap_after_hook.lua new file mode 100644 index 00000000000..0944a52e59d --- /dev/null +++ b/tests/gold_tests/pluginTest/lua/remap_after_hook.lua @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +local function host_or_nil(v) + if v == nil then return "" end + return v +end + +function send_response() + local hook_val = ts.remap.get_from_url_host() + ts.client_response.header['Remap-From-Host'] = ts.ctx['remap_from_host'] + ts.client_response.header['Hook-From-Host'] = host_or_nil(hook_val) +end + +function do_remap() + ts.ctx['remap_from_host'] = host_or_nil(ts.remap.get_from_url_host()) + ts.hook(TS_LUA_HOOK_SEND_RESPONSE_HDR, send_response) +end diff --git a/tests/gold_tests/pluginTest/multiplexer/multiplexer.test.py b/tests/gold_tests/pluginTest/multiplexer/multiplexer.test.py index c4ffe1d4797..c2238cd1876 100644 --- a/tests/gold_tests/pluginTest/multiplexer/multiplexer.test.py +++ b/tests/gold_tests/pluginTest/multiplexer/multiplexer.test.py @@ -227,5 +227,86 @@ def setupServers(self): 'uuid: PUT', "Verify the HTTPS server did not receive the PUT request.") +class MultiplexerInvalidChunkedResponseTest: + """ + Verify a copied upstream response with an oversized chunk-size does not + disrupt the original client transaction. + """ + + replay_file = os.path.join("replays", "multiplexer_invalid_chunk_original.replay.yaml") + multiplexed_host_replay_file = os.path.join("replays", "multiplexer_invalid_chunk_copy.replay.yaml") + + def __init__(self): + self.setupServers() + self.setupDns() + self.setupTS() + + def setupDns(self): + counter = MultiplexerTestBase.dns_counter + MultiplexerTestBase.dns_counter += 1 + self.dns = Test.MakeDNServer(f"dns_{counter}", default='127.0.0.1') + + def setupServers(self): + counter = MultiplexerTestBase.server_counter + MultiplexerTestBase.server_counter += 1 + self.server_origin = Test.MakeVerifierServerProcess(f"server_origin_{counter}", self.replay_file) + self.server_http = Test.MakeVerifierServerProcess(f"server_http_{counter}", self.multiplexed_host_replay_file) + self.server_https = Test.MakeVerifierServerProcess(f"server_https_{counter}", self.multiplexed_host_replay_file) + + self.server_origin.Streams.All += Testers.ContainsExpression( + 'uuid: INVALID_CHUNK', "Verify the original server received the invalid chunk test request.") + self.server_origin.Streams.All += Testers.ContainsExpression( + 'X-Multiplexer: original', 'Verify the original target received the "original" request.') + self.server_origin.Streams.All += Testers.ExcludesExpression(r'\[ERROR\]', 'Verify there were no errors in the replay.') + + for server in [self.server_http, self.server_https]: + server.Streams.All += Testers.ContainsExpression( + 'uuid: INVALID_CHUNK', "Verify the multiplexed server received the invalid chunk test request.") + server.Streams.All += Testers.ContainsExpression( + 'X-Multiplexer: copy', 'Verify the multiplexed server received a "copy" of the request.') + server.Streams.All += Testers.ExcludesExpression(r'\[ERROR\]', 'Verify there were no errors in the replay.') + + self.server_https.Streams.All += Testers.ContainsExpression( + 'Finished accept using TLSSession', "Verify the HTTPS was indeed used by the HTTPS server.") + + def setupTS(self): + counter = MultiplexerTestBase.ts_counter + MultiplexerTestBase.ts_counter += 1 + self.ts = Test.MakeATSProcess(f"ts_{counter}", enable_tls=True, enable_cache=False) + self.ts.addDefaultSSLFiles() + self.ts.Disk.records_config.update( + { + "proxy.config.ssl.server.cert.path": f'{self.ts.Variables.SSLDir}', + "proxy.config.ssl.server.private_key.path": f'{self.ts.Variables.SSLDir}', + "proxy.config.ssl.client.verify.server.policy": 'PERMISSIVE', + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http|multiplexer', + 'proxy.config.dns.nameservers': f'127.0.0.1:{self.dns.Variables.Port}', + 'proxy.config.dns.resolv_conf': 'NULL', + }) + self.ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + self.ts.Disk.remap_config.AddLines( + [ + f'map https://origin.server.com https://backend.origin.server.com:{self.server_origin.Variables.https_port} ' + f'@plugin=multiplexer.so @pparam=nontls.server.com @pparam=tls.server.com', + f'map http://nontls.server.com http://backend.nontls.server.com:{self.server_http.Variables.http_port}', + f'map http://tls.server.com https://backend.tls.server.com:{self.server_https.Variables.https_port}', + ]) + + def run(self): + tr = Test.AddTestRun("Multiplexed response with oversized chunk-size") + self.ts.StartBefore(self.dns) + tr.Processes.Default.StartBefore(self.server_origin) + tr.Processes.Default.StartBefore(self.server_http) + tr.Processes.Default.StartBefore(self.server_https) + tr.Processes.Default.StartBefore(self.ts) + + counter = MultiplexerTestBase.client_counter + MultiplexerTestBase.client_counter += 1 + client = tr.AddVerifierClientProcess(f"client_{counter}", self.replay_file, https_ports=[self.ts.Variables.ssl_port]) + client.Streams.All += Testers.ExcludesExpression(r'\[ERROR\]', 'Verify there were no errors in the replay.') + + MultiplexerTest().run() MultiplexerSkipPostTest().run() +MultiplexerInvalidChunkedResponseTest().run() diff --git a/tests/gold_tests/pluginTest/multiplexer/replays/multiplexer_invalid_chunk_copy.replay.yaml b/tests/gold_tests/pluginTest/multiplexer/replays/multiplexer_invalid_chunk_copy.replay.yaml new file mode 100644 index 00000000000..3e514a66717 --- /dev/null +++ b/tests/gold_tests/pluginTest/multiplexer/replays/multiplexer_invalid_chunk_copy.replay.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +sessions: +- protocol: + stack: https + transactions: + - client-request: + method: "GET" + version: "1.1" + url: /path/invalid_chunked_response + headers: + fields: + - [ Host, origin.server.com ] + - [ Content-Length, 0 ] + - [ X-Request, invalid_chunk ] + - [ uuid, INVALID_CHUNK ] + + proxy-request: + method: "GET" + headers: + fields: + - [ X-Request, { value: invalid_chunk, as: equal } ] + - [ X-Multiplexer, { value: copy, as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Transfer-Encoding, chunked ] + - [ X-Response, invalid_chunk_copy ] + content: + transfer: plain + encoding: uri + # 0x8000000000000000 is one larger than the largest int64_t value. + data: 8000000000000000%0D%0Aboom%0D%0A0%0D%0A%0D%0A diff --git a/tests/gold_tests/pluginTest/multiplexer/replays/multiplexer_invalid_chunk_original.replay.yaml b/tests/gold_tests/pluginTest/multiplexer/replays/multiplexer_invalid_chunk_original.replay.yaml new file mode 100644 index 00000000000..83eca4c519b --- /dev/null +++ b/tests/gold_tests/pluginTest/multiplexer/replays/multiplexer_invalid_chunk_original.replay.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +sessions: +- protocol: + stack: https + transactions: + - client-request: + method: "GET" + version: "1.1" + url: /path/invalid_chunked_response + headers: + fields: + - [ Host, origin.server.com ] + - [ Content-Length, 0 ] + - [ X-Request, invalid_chunk ] + - [ uuid, INVALID_CHUNK ] + + proxy-request: + method: "GET" + headers: + fields: + - [ X-Request, { value: invalid_chunk, as: equal } ] + - [ X-Multiplexer, { value: original, as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 32 ] + - [ X-Response, invalid_chunk ] + + proxy-response: + status: 200 + reason: OK + headers: + fields: + - [ X-Response, { value: invalid_chunk, as: equal } ] diff --git a/tests/gold_tests/pluginTest/prefetch/prefetch_query_path_traversal.replay.yaml b/tests/gold_tests/pluginTest/prefetch/prefetch_query_path_traversal.replay.yaml new file mode 100644 index 00000000000..6634ff64ac6 --- /dev/null +++ b/tests/gold_tests/pluginTest/prefetch/prefetch_query_path_traversal.replay.yaml @@ -0,0 +1,165 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: "Verify query prefetch blocks path traversal" + + server: + name: "server" + log_validation: + excludes: + - expression: "GET /texts/\\.\\./private/secret\\.txt" + description: "Traversal prefetch should not reach the origin" + - expression: "GET /private/secret\\.txt" + description: "Normalized traversal prefetch should not reach the origin" + + client: + name: "client" + + ats: + name: "ts" + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: "http|dns|prefetch" + remap_config: + - from: "http://domain.in/" + to: "http://127.0.0.1:{SERVER_HTTP_PORT}/" + plugins: + - name: "cachekey.so" + args: + - "--remove-all-params=true" + - name: "prefetch.so" + args: + - "--front=true" + - "--fetch-policy=simple" + - "--fetch-path-pattern=/(.*)/$1/" + - "--fetch-query=next" + - "--fetch-count=2" + - "--cmcd-nor=true" + log_validation: + traffic_out: + contains: + - expression: "schedule fetch: http://domain.in/texts/demo-2.txt" + description: "Safe query prefetch should be scheduled" + - expression: "skipping empty query prefetch path" + description: "Empty query prefetch values should be ignored" + - expression: "skipping unsafe query prefetch path: '../private/secret.txt'" + description: "Traversal query prefetch should be rejected" + - expression: "skipping unsafe cmcd nor path: '../private/secret.txt'" + description: "Percent-decoded CMCD nor traversal should be rejected" + excludes: + - expression: "schedule fetch: http://domain.in/texts/\\.\\./private/secret\\.txt" + description: "Traversal prefetch should not be scheduled" + - expression: "schedule fetch: http://domain.in/private/secret\\.txt" + description: "Normalized traversal prefetch should not be scheduled" + +sessions: + - transactions: + - client-request: + method: "GET" + version: "1.1" + url: "/texts/demo-1.txt?next=&next=demo-2.txt&next=../private/secret.txt" + headers: + fields: + - [Host, domain.in] + - [uuid, query-path-traversal] + + proxy-request: + method: "GET" + url: "/texts/demo-1.txt?next=&next=demo-2.txt&next=../private/secret.txt" + headers: + fields: + - [Host, {value: "127.0.0.1", as: prefix}] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [Connection, close] + - [Cache-Control, "max-age=85000"] + - [Content-Length, 40] + content: + data: "This is the body for /texts/demo-1.txt.\n" + + proxy-response: + status: 200 + + - client-request: + delay: 1s + method: "GET" + version: "1.1" + url: "/cmcd/segment-1.m4s" + headers: + fields: + - [Host, domain.in] + - [Cmcd-Request, 'nor="%2e%2e/private/secret.txt"'] + - [uuid, cmcd-path-traversal] + + proxy-request: + method: "GET" + url: "/cmcd/segment-1.m4s" + headers: + fields: + - [Host, {value: "127.0.0.1", as: prefix}] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [Connection, close] + - [Cache-Control, "max-age=85000"] + - [Content-Length, 42] + content: + data: "This is the body for /cmcd/segment-1.m4s.\n" + + proxy-response: + status: 200 + + - client-request: + delay: 1s + method: "GET" + version: "1.1" + url: "/texts/demo-2.txt?next=&next=demo-2.txt&next=../private/secret.txt" + headers: + fields: + - [Host, domain.in] + - [uuid, query-path-traversal] + + proxy-request: + method: "GET" + url: "/texts/demo-2.txt?next=&next=demo-2.txt&next=../private/secret.txt" + headers: + fields: + - [X-CDN-Prefetch, {value: "texts/demo-1.txt", as: equal}] + + server-response: + status: 200 + reason: "OK" + headers: + fields: + - [Connection, close] + - [Cache-Control, "max-age=85000"] + - [Content-Length, 40] + content: + data: "This is the body for /texts/demo-2.txt.\n" + + proxy-response: + status: 200 diff --git a/tests/gold_tests/pluginTest/prefetch/prefetch_query_path_traversal.test.py b/tests/gold_tests/pluginTest/prefetch/prefetch_query_path_traversal.test.py new file mode 100644 index 00000000000..908757bae4e --- /dev/null +++ b/tests/gold_tests/pluginTest/prefetch/prefetch_query_path_traversal.test.py @@ -0,0 +1,24 @@ +''' +Test prefetch.so query prefetch path traversal handling. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test prefetch.so query prefetch path traversal handling. +''' + +Test.ATSReplayTest(replay_file="prefetch_query_path_traversal.replay.yaml") diff --git a/tests/gold_tests/pluginTest/rate_limit/concurrent_reject.sh b/tests/gold_tests/pluginTest/rate_limit/concurrent_reject.sh new file mode 100755 index 00000000000..a4081c02b44 --- /dev/null +++ b/tests/gold_tests/pluginTest/rate_limit/concurrent_reject.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test: fire two requests concurrently at the rate limiter. +# First request hits a slow origin (holds the slot for 3s). +# Second request arrives 500ms later and should get 429. + +ATS_PORT=$1 +HOST="limit.example.com" + +curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${ATS_PORT}/slow" \ + -H "Host: ${HOST}" & +SLOW_PID=$! + +sleep 0.5 + +FAST_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${ATS_PORT}/fast" \ + -H "Host: ${HOST}") + +wait $SLOW_PID + +echo "fast=${FAST_CODE}" diff --git a/tests/gold_tests/pluginTest/rate_limit/independent_limiters.sh b/tests/gold_tests/pluginTest/rate_limit/independent_limiters.sh new file mode 100755 index 00000000000..8bdf5feb283 --- /dev/null +++ b/tests/gold_tests/pluginTest/rate_limit/independent_limiters.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test: independent limiters per remap rule. +# Two remap rules each with limit=1. Saturating one should not affect the other. + +ATS_PORT=$1 +HOST_A="limit-a.example.com" +HOST_B="limit-b.example.com" + +# Hold a slot on rule A (3s origin delay) +curl -s -o /dev/null \ + "http://127.0.0.1:${ATS_PORT}/slow" \ + -H "Host: ${HOST_A}" & + +sleep 0.5 + +# Request to rule B should still pass (independent limiter) +B_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${ATS_PORT}/fast" \ + -H "Host: ${HOST_B}") + +wait + +echo "independent=${B_CODE}" diff --git a/tests/gold_tests/pluginTest/rate_limit/queue_bypass_regression.sh b/tests/gold_tests/pluginTest/rate_limit/queue_bypass_regression.sh new file mode 100755 index 00000000000..41af560bd12 --- /dev/null +++ b/tests/gold_tests/pluginTest/rate_limit/queue_bypass_regression.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Regression test for the queue drain bug (Finding #106). +# +# With limit=1 and queue=5, fire 3 requests concurrently against a slow +# origin (3s delay). Correct behavior: +# - Request A gets the slot, holds it for 3s +# - Requests B and C are queued +# - After A completes (~3s), queue handler gives B the slot +# - After B completes (~6s total), queue handler gives C the slot +# - Total wall time: ~9s (3 sequential slow requests) +# +# With the old bug (reserve() != RESERVED / != FULL): +# - Request A gets the slot +# - Queue handler immediately resumes B and C WITHOUT a valid reservation +# - B and C run concurrently with A (all finish around ~3s) +# - Total wall time: ~3s +# +# We detect the bug by measuring wall time. If all 3 finish in under 5s, +# the limiter was bypassed. Correct behavior takes >= 6s (at least 2 +# sequential slow-origin round trips for the queued requests). + +ATS_PORT=$1 +HOST="queued.example.com" + +START=$(date +%s) + +# Fire 3 requests concurrently +curl -s -o /dev/null -w "a=%{http_code}\n" \ + "http://127.0.0.1:${ATS_PORT}/slow" \ + -H "Host: ${HOST}" & +PID_A=$! + +sleep 0.3 + +curl -s -o /dev/null -w "b=%{http_code}\n" \ + "http://127.0.0.1:${ATS_PORT}/slow" \ + -H "Host: ${HOST}" & +PID_B=$! + +sleep 0.3 + +curl -s -o /dev/null -w "c=%{http_code}\n" \ + "http://127.0.0.1:${ATS_PORT}/slow" \ + -H "Host: ${HOST}" & +PID_C=$! + +wait $PID_A +wait $PID_B +wait $PID_C + +END=$(date +%s) +ELAPSED=$((END - START)) + +# With correct limiting: >= 6s (2 queued requests each wait for a slot) +# With the bug: ~3s (all run concurrently, bypassing the limit) +if [ "$ELAPSED" -ge 6 ]; then + echo "timing=correct elapsed=${ELAPSED}s" +else + echo "timing=bypassed elapsed=${ELAPSED}s" +fi diff --git a/tests/gold_tests/pluginTest/rate_limit/queue_drain.sh b/tests/gold_tests/pluginTest/rate_limit/queue_drain.sh new file mode 100755 index 00000000000..bfe50b751e4 --- /dev/null +++ b/tests/gold_tests/pluginTest/rate_limit/queue_drain.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test: queue drain behavior (exercises the fixed reserve() loop). +# With limit=1 and queue=5, the second request gets queued (not rejected). +# After the first request completes (origin delay), the queue handler +# resumes the second request which then succeeds with 200. + +ATS_PORT=$1 +HOST="queued.example.com" + +# First request: holds the single slot for 3s (origin delay) +curl -s -o /dev/null -w "slow=%{http_code}\n" \ + "http://127.0.0.1:${ATS_PORT}/slow" \ + -H "Host: ${HOST}" & +SLOW_PID=$! + +# Wait for first request to reach the origin and hold the slot +sleep 0.5 + +# Second request: should be queued (not rejected), then resumed after +# the first completes and the queue handler runs (every 300ms). +FAST_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${ATS_PORT}/fast" \ + -H "Host: ${HOST}") + +wait $SLOW_PID + +echo "queued=${FAST_CODE}" diff --git a/tests/gold_tests/pluginTest/rate_limit/rate_limit.test.py b/tests/gold_tests/pluginTest/rate_limit/rate_limit.test.py new file mode 100644 index 00000000000..573de3e7730 --- /dev/null +++ b/tests/gold_tests/pluginTest/rate_limit/rate_limit.test.py @@ -0,0 +1,132 @@ +''' +Test rate_limit plugin: connection limit enforcement, queue drain, and 429 rejection. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test rate_limit plugin: concurrent limit enforcement, queue drain, and independent limiters. +''' + +Test.ContinueOnFail = True + +server = Test.MakeOriginServer("server", delay=3) +ts = Test.MakeATSProcess("ts") + +server.addResponse( + "sessionlog.json", { + "headers": "GET /slow HTTP/1.1\r\nHost: limit.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\n" + "Content-Length: 4\r\n" + "Connection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "SLOW" + }) + +server.addResponse( + "sessionlog.json", { + "headers": "GET /fast HTTP/1.1\r\nHost: limit.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\n" + "Content-Length: 4\r\n" + "Connection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "FAST" + }) + +ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'rate_limit', + 'proxy.config.http.insert_response_via_str': 0, + 'proxy.config.url_remap.remap_required': 1, + }) + +# Rule 1: limit=1, no queue — immediate rejection +ts.Disk.remap_config.AddLine( + f'map http://limit.example.com/ http://127.0.0.1:{server.Variables.Port}/' + f' @plugin=rate_limit.so @pparam=--limit @pparam=1 @pparam=--queue @pparam=0' + f' @pparam=--error @pparam=429 @pparam=--retry @pparam=1') + +# Rule 2: limit=1, queue=5 — queues excess, resumes when slot freed +ts.Disk.remap_config.AddLine( + f'map http://queued.example.com/ http://127.0.0.1:{server.Variables.Port}/' + f' @plugin=rate_limit.so @pparam=--limit @pparam=1 @pparam=--queue @pparam=5' + f' @pparam=--maxage @pparam=10000 @pparam=--error @pparam=429') + +# Rules 3 & 4: two independent limiters (limit=1 each) +ts.Disk.remap_config.AddLine( + f'map http://limit-a.example.com/ http://127.0.0.1:{server.Variables.Port}/' + f' @plugin=rate_limit.so @pparam=--limit @pparam=1 @pparam=--queue @pparam=0' + f' @pparam=--error @pparam=429') + +ts.Disk.remap_config.AddLine( + f'map http://limit-b.example.com/ http://127.0.0.1:{server.Variables.Port}/' + f' @plugin=rate_limit.so @pparam=--limit @pparam=1 @pparam=--queue @pparam=0' + f' @pparam=--error @pparam=429') + +# Test 1: Concurrent rejection — second request gets 429 +tr = Test.AddTestRun("Concurrent requests: second gets 429") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Command = f'sh {Test.TestDirectory}/concurrent_reject.sh {ts.Variables.port}' +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression( + "fast=429", "Second concurrent request should be rejected with 429") + +# Test 2: Sequential requests both pass +tr2 = Test.AddTestRun("Sequential requests: both get 200") +tr2.Processes.Default.Command = f'sh {Test.TestDirectory}/sequential_pass.sh {ts.Variables.port}' +tr2.Processes.Default.ReturnCode = 0 +tr2.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression("first=200", "First sequential request should pass") +tr2.Processes.Default.Streams.stdout.Content += Testers.ContainsExpression( + "second=200", "Second sequential request should also pass") + +# Test 3: Retry-After header on 429 rejection +tr3 = Test.AddTestRun("429 response includes Retry-After header") +tr3.Processes.Default.Command = f'sh {Test.TestDirectory}/retry_after.sh {ts.Variables.port}' +tr3.Processes.Default.ReturnCode = 0 +tr3.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression( + "Retry-After: 1", "429 response should include Retry-After header") + +# Test 4: Queue drain — exercises the fixed reserve() loop +tr4 = Test.AddTestRun("Queue drain: queued request resumes with 200") +tr4.Processes.Default.Command = f'sh {Test.TestDirectory}/queue_drain.sh {ts.Variables.port}' +tr4.Processes.Default.ReturnCode = 0 +tr4.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression( + "queued=200", "Queued request should eventually succeed after slot freed") + +# Test 5: Independent limiters — saturating one rule doesn't block the other +tr5 = Test.AddTestRun("Independent limiters: rule B passes while rule A is full") +tr5.Processes.Default.Command = f'sh {Test.TestDirectory}/independent_limiters.sh {ts.Variables.port}' +tr5.Processes.Default.ReturnCode = 0 +tr5.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression( + "independent=200", "Request to rule B should pass despite rule A being full") + +# Test 6: Regression for Finding #106 — queue bypass via incorrect reserve() check. +# With the bug, queued requests are resumed without a valid slot reservation, +# allowing all 3 requests to run concurrently (~3s). With the fix, they serialize +# through the single slot (~9s). We check wall time >= 6s as the pass criterion. +tr6 = Test.AddTestRun("Regression #106: queue does not bypass limit") +tr6.Processes.Default.Command = f'sh {Test.TestDirectory}/queue_bypass_regression.sh {ts.Variables.port}' +tr6.Processes.Default.ReturnCode = 0 +tr6.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression( + "timing=correct", "Queued requests must serialize through the limiter, not bypass it") diff --git a/tests/gold_tests/pluginTest/rate_limit/rate_limit_iprep.test.py b/tests/gold_tests/pluginTest/rate_limit/rate_limit_iprep.test.py new file mode 100644 index 00000000000..ba6003264d1 --- /dev/null +++ b/tests/gold_tests/pluginTest/rate_limit/rate_limit_iprep.test.py @@ -0,0 +1,114 @@ +''' +Test rate_limit plugin: IP reputation initialization (Finding #108). + +Validates that ip-rep buckets are properly initialized. The bug used +vector::reserve() instead of resize(), causing UB on indexed writes. +With the fix, ATS starts cleanly and processes TLS connections through +the ip-rep logic without crashing. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Test rate_limit ip-rep initialization: reserve() vs resize() regression (Finding #108). +''' + +Test.ContinueOnFail = True + +server = Test.MakeOriginServer("server") +ts = Test.MakeATSProcess("ts", enable_tls=True) + +server.addResponse( + "sessionlog.json", { + "headers": "GET /test HTTP/1.1\r\nHost: iprep.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\n" + "Content-Length: 2\r\n" + "Connection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "OK" + }) + +ts.addDefaultSSLFiles() + +# Write the rate_limit YAML config with ip-rep enabled +rate_limit_yaml = os.path.join(ts.Variables.CONFIGDIR, 'rate_limit.yaml') +ts.Disk.File( + rate_limit_yaml, typename="ats:config").AddLines( + [ + 'ip-rep:', + ' - name: test-iprep', + ' buckets: 5', + ' size: 10', + ' percentage: 90', + ' max_age: 300', + '', + 'selector:', + ' - sni: iprep.example.com', + ' limit: 100', + ' ip-rep: test-iprep', + '', + ]) + +ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'rate_limit', + 'proxy.config.http.insert_response_via_str': 0, + 'proxy.config.url_remap.remap_required': 0, + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + }) + +ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + +ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server.Variables.Port}/') + +ts.Disk.plugin_config.AddLine(f'rate_limit.so {rate_limit_yaml}') + +# Test 1: ATS starts with ip-rep config and handles a TLS request. +# With the reserve() bug, this would crash or produce UB on startup. +tr = Test.AddTestRun("IP reputation init: TLS request through ip-rep selector") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Command = ( + f"curl -sk -o /dev/null -w '%{{http_code}}' " + f"'https://iprep.example.com:{ts.Variables.ssl_port}/test' " + f"--resolve 'iprep.example.com:{ts.Variables.ssl_port}:127.0.0.1'") +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression( + "200", "TLS request through ip-rep selector should succeed") + +# Test 2: Make multiple requests to exercise the ip-rep increment path. +# Each TLS handshake from the same IP increments the reputation counter. +# If buckets were not properly initialized, this triggers the crash. +tr2 = Test.AddTestRun("IP reputation: multiple requests increment counters") +tr2.Processes.Default.Command = ( + f'for i in 1 2 3 4 5; do ' + f' curl -sk -o /dev/null -w "%{{http_code}} " ' + f' "https://iprep.example.com:{ts.Variables.ssl_port}/test" ' + f' --resolve "iprep.example.com:{ts.Variables.ssl_port}:127.0.0.1"; ' + f'done; echo ""') +tr2.Processes.Default.ReturnCode = 0 +tr2.Processes.Default.Streams.stdout.Content = Testers.ExcludesExpression( + "000", "No request should get a connection failure (code 000)") + +# Verify ATS didn't crash +ts.Disk.diags_log.Content = Testers.ExcludesExpression("FATAL", "ATS should not crash with ip-rep enabled") diff --git a/tests/gold_tests/pluginTest/rate_limit/rate_limit_sni.test.py b/tests/gold_tests/pluginTest/rate_limit/rate_limit_sni.test.py new file mode 100644 index 00000000000..5389bde7ed8 --- /dev/null +++ b/tests/gold_tests/pluginTest/rate_limit/rate_limit_sni.test.py @@ -0,0 +1,115 @@ +''' +Test rate_limit plugin: SNI queue expiry does not underflow active counter (Finding #109). + +With the bug, when a queued SNI connection expires via max_age, free() is +called on VCONN_CLOSE even though reserve() never succeeded, underflowing +the active counter and crashing on the next reserve() assertion. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Test rate_limit SNI queue expiry: active counter underflow regression (Finding #109). +''' + +Test.ContinueOnFail = True + +server = Test.MakeOriginServer("server", delay=4) +ts = Test.MakeATSProcess("ts", enable_tls=True) + +server.addResponse( + "sessionlog.json", { + "headers": "GET /slow HTTP/1.1\r\nHost: queue-expiry.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\n" + "Content-Length: 4\r\n" + "Connection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "SLOW" + }) + +server.addResponse( + "sessionlog.json", { + "headers": "GET /test HTTP/1.1\r\nHost: queue-expiry.example.com\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\n" + "Content-Length: 2\r\n" + "Connection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "OK" + }) + +ts.addDefaultSSLFiles() + +# SNI selector with limit=1, queue=5, max_age=1s (1000ms). +# The short max_age causes queued connections to expire quickly. +rate_limit_yaml = os.path.join(ts.Variables.CONFIGDIR, 'rate_limit.yaml') +ts.Disk.File( + rate_limit_yaml, typename="ats:config").AddLines( + [ + 'selector:', + ' - sni: queue-expiry.example.com', + ' limit: 1', + ' queue:', + ' size: 5', + ' max_age: 1', + '', + ]) + +ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'rate_limit', + 'proxy.config.http.insert_response_via_str': 0, + 'proxy.config.url_remap.remap_required': 0, + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + }) + +ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + +ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server.Variables.Port}/') + +ts.Disk.plugin_config.AddLine(f'rate_limit.so {rate_limit_yaml}') + +RESOLVE = f"--resolve 'queue-expiry.example.com:{ts.Variables.ssl_port}:127.0.0.1'" +BASE_URL = f"https://queue-expiry.example.com:{ts.Variables.ssl_port}" + +# Test: Queue expiry regression. +# First request holds the slot (4s origin delay), second gets queued and +# expires after max_age (1s). Health check after proves ATS didn't crash. +tr = Test.AddTestRun("Queue expiry: ATS survives without active counter underflow") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Command = ( + f"curl -sk -o /dev/null '{BASE_URL}/slow' {RESOLVE} & " + f"sleep 0.5; " + f"curl -sk -o /dev/null '{BASE_URL}/queued' {RESOLVE} 2>/dev/null; " + f"wait; sleep 0.5; " + f"curl -sk -o /dev/null -w '%{{http_code}}' '{BASE_URL}/test' {RESOLVE}") +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout.Content = Testers.ContainsExpression( + "200", "Health check after queue expiry should succeed (ATS still alive)") + +# Verify ATS didn't crash +ts.Disk.diags_log.Content = Testers.ExcludesExpression("FATAL", "ATS should not crash from active counter underflow") +ts.Disk.diags_log.Content += Testers.ExcludesExpression("ink_release_assert", "No assertion failure from _active underflow") diff --git a/tests/gold_tests/pluginTest/rate_limit/retry_after.sh b/tests/gold_tests/pluginTest/rate_limit/retry_after.sh new file mode 100755 index 00000000000..aac36b60812 --- /dev/null +++ b/tests/gold_tests/pluginTest/rate_limit/retry_after.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test: verify Retry-After header on 429 rejection. +# First request holds the slot, second gets 429 with Retry-After. + +ATS_PORT=$1 +HOST="limit.example.com" + +curl -s -o /dev/null \ + "http://127.0.0.1:${ATS_PORT}/slow" \ + -H "Host: ${HOST}" & + +sleep 0.5 + +curl -s -D - -o /dev/null \ + "http://127.0.0.1:${ATS_PORT}/fast" \ + -H "Host: ${HOST}" + +wait diff --git a/tests/gold_tests/pluginTest/rate_limit/sequential_pass.sh b/tests/gold_tests/pluginTest/rate_limit/sequential_pass.sh new file mode 100755 index 00000000000..3878d8a2924 --- /dev/null +++ b/tests/gold_tests/pluginTest/rate_limit/sequential_pass.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test: sequential requests both pass when slot freed between them. +# With limit=1, serial requests should both get 200 because the +# TXN_CLOSE hook frees the slot before the next request arrives. + +ATS_PORT=$1 +HOST="limit.example.com" + +FIRST=$(curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${ATS_PORT}/slow" \ + -H "Host: ${HOST}") + +SECOND=$(curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${ATS_PORT}/fast" \ + -H "Host: ${HOST}") + +echo "first=${FIRST} second=${SECOND}" diff --git a/tests/gold_tests/pluginTest/redirect_limit/redirect_limit.test.py b/tests/gold_tests/pluginTest/redirect_limit/redirect_limit.test.py new file mode 100644 index 00000000000..bfa921ec63d --- /dev/null +++ b/tests/gold_tests/pluginTest/redirect_limit/redirect_limit.test.py @@ -0,0 +1,134 @@ +''' +Plugin-initiated redirects must honor proxy.config.http.number_of_redirections. + +A plugin that calls TSHttpTxnRedirectUrlSet on every response hook (the test +plugin redirect_rearm does exactly this) must not be able to follow more +redirects than the configured limit. The redirect counter is shared with the +core redirect follower, so a plugin that re-sets the redirect URL on each hop +is still capped at number_of_redirections. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = 'A plugin re-setting the redirect URL each hop must not bypass number_of_redirections' + +Test.ContinueOnFail = True + +server = Test.MakeOriginServer("server") + +# Chain of 5 hops on the same origin server. Use literal 127.0.0.1:{server.port} +# in the Location headers so ATS's redirect follower doesn't have to do DNS. +ORIGIN = "http://127.0.0.1:{0}".format(server.Variables.Port) + +for i in range(1, 5): + server.addResponse( + "sessionlog.json", { + "headers": "GET /r{i} HTTP/1.1\r\nHost: *\r\n\r\n".format(i=i), + "timestamp": "1", + "body": "" + }, { + "headers": "HTTP/1.1 302 Found\r\nLocation: {0}/r{nxt}\r\nContent-Length: 0\r\n\r\n".format(ORIGIN, nxt=i + 1), + "timestamp": "1", + "body": "" + }) + +server.addResponse( + "sessionlog.json", { + "headers": "GET /r5 HTTP/1.1\r\nHost: *\r\n\r\n", + "timestamp": "1", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n", + "timestamp": "1", + "body": "final" + }) + +# A short chain (/s1 -> /s2 -> 200) that stays within number_of_redirections=2. +# This is the positive case: a legitimate plugin-initiated redirect within the +# limit must still be followed all the way to the terminal 200. +server.addResponse( + "sessionlog.json", { + "headers": "GET /s1 HTTP/1.1\r\nHost: *\r\n\r\n", + "timestamp": "1", + "body": "" + }, { + "headers": "HTTP/1.1 302 Found\r\nLocation: {0}/s2\r\nContent-Length: 0\r\n\r\n".format(ORIGIN), + "timestamp": "1", + "body": "" + }) +server.addResponse( + "sessionlog.json", { + "headers": "GET /s2 HTTP/1.1\r\nHost: *\r\n\r\n", + "timestamp": "1", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n", + "timestamp": "1", + "body": "shortfinal" + }) + +ts = Test.MakeATSProcess("ts", enable_cache=False) + +ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'redirect_rearm|http_redirect|http', + 'proxy.config.http.number_of_redirections': 2, + 'proxy.config.http.redirect.actions': 'self:follow,private:follow', + }) + +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'redirect_rearm.so'), ts) + +ts.Disk.remap_config.AddLine('map http://127.0.0.1:{0}/ http://127.0.0.1:{0}/'.format(server.Variables.Port)) + +tr = Test.AddTestRun() +tr.MakeCurlCommand( + '-sS -i -x 127.0.0.1:TSPORT http://127.0.0.1:OPORT/r1'.replace('TSPORT', str(ts.Variables.port)).replace( + 'OPORT', str(server.Variables.Port)), + ts=ts) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.ReturnCode = 0 +# With the fix the limit fires and the client sees a 302 returned (the last +# followed hop's response). Without the fix the plugin re-arms the counter on +# every hop and the client sees the final 200 with body "final". +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "HTTP/1.1 302 ", "Client's terminal response must be a 302 from the limit firing, not the final 200") +# Pin the boundary: with number_of_redirections=2 the follower advances r1 -> r2 +# -> r3 and returns r3's response, whose Location points at /r4. Asserting the +# terminal Location is /r4 proves exactly two hops were followed, so this case +# cannot pass if a regression instead followed zero hops (Location /r2) or the +# whole chain. +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "[Ll]ocation: .*/r4", "Terminal 302 must be the third hop's response, proving exactly two redirects were followed") +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "final", "Client must NOT receive the final body (limit bypassed)") + +# Positive case: a redirect chain within the limit (one hop) must still be +# followed to completion. Guards against the fix over-correcting and refusing +# legitimate plugin-initiated redirects. +tr = Test.AddTestRun() +tr.MakeCurlCommand( + '-sS -i -x 127.0.0.1:TSPORT http://127.0.0.1:OPORT/s1'.replace('TSPORT', str(ts.Variables.port)).replace( + 'OPORT', str(server.Variables.Port)), + ts=ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "HTTP/1.1 200 ", "A within-limit plugin-initiated redirect must reach the terminal 200") +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "shortfinal", "Client must receive the terminal body for a within-limit redirect") diff --git a/tests/gold_tests/pluginTest/slice/shrink_origin.py b/tests/gold_tests/pluginTest/slice/shrink_origin.py new file mode 100644 index 00000000000..6c329d00252 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/shrink_origin.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +''' +Stateful origin server for slice content-shrink underflow test. + +Serves different Content-Range responses based on request sequence to simulate +content shrinking between fetches. + +Usage: python3 shrink_origin.py + +Request sequence for /shrink with blockbytes=7, client requesting bytes=14-20: + 1. GET /shrink Range: bytes=0-6 (reference block, RefType::First) + Response: 206, Content-Range: bytes 0-6/21, Etag: "old", body: 7 bytes + 2. GET /shrink Range: bytes=14-20 (block 2, first client block) + Response: 206, Content-Range: bytes 14-20/10, Etag: "new", body: 7 bytes + -> triggers mismatch (etag differs), m_contentlen updated to 10 + 3. GET /shrink Range: bytes=0-6 (reference refetch) + Response: 206, Content-Range: bytes 0-6/10, Etag: "new", body: 7 bytes + -> blockpos=14 > m_contentlen=10 => underflow guard triggers +''' + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import http.server +import sys +import threading + +# Track how many times each range has been requested +request_counts = {} +lock = threading.Lock() + + +class ShrinkHandler(http.server.BaseHTTPRequestHandler): + + def do_GET(self): + if self.path == '/ruok': + self.send_response(200) + self.send_header('Content-Length', '0') + self.end_headers() + return + + range_hdr = self.headers.get('Range', '') + + with lock: + key = f"{self.path}|{range_hdr}" + request_counts[key] = request_counts.get(key, 0) + 1 + count = request_counts[key] + + if self.path == '/shrink': + self._handle_shrink(range_hdr, count) + elif self.path == '/shrink_mid': + self._handle_shrink_mid(range_hdr, count) + else: + self.send_response(404) + self.send_header('Content-Length', '0') + self.end_headers() + + def _handle_shrink(self, range_hdr, count): + # Parse range: "bytes=START-END" + body = b'x' * 7 # always 7 bytes body for blockbytes=7 + + if range_hdr == 'bytes=0-6': + if count <= 1: + # First request for block 0 (reference): original content + self._send_206('bytes 0-6/21', '"old"', body) + else: + # Second request for block 0 (reference refetch after mismatch): + # Report shrunk content-length=10, new etag + self._send_206('bytes 0-6/10', '"new"', body) + elif range_hdr == 'bytes=14-20': + # Block 2 (interior): shrunk content, different etag => mismatch + self._send_206('bytes 14-20/10', '"new"', body) + else: + self.send_response(416) + self.send_header('Content-Length', '0') + self.end_headers() + + def _handle_shrink_mid(self, range_hdr, count): + """Non-block-aligned range case. + + Client requests bytes=16-20, blockbytes=7. + blockpos=14, but m_req_range.m_beg=16. + Content shrinks to 15: above blockpos but below range start. + """ + body = b'y' * 7 + + if range_hdr == 'bytes=0-6': + if count <= 1: + # Reference: original size + self._send_206('bytes 0-6/21', '"old"', body) + else: + # Reference refetch: shrunk to 15 + self._send_206('bytes 0-6/15', '"new"', body) + elif range_hdr == 'bytes=14-20': + # Block 2 (interior): shrunk content, different etag + self._send_206('bytes 14-20/15', '"new"', body) + else: + self.send_response(416) + self.send_header('Content-Length', '0') + self.end_headers() + + def _send_206(self, content_range, etag, body): + self.send_response(206) + self.send_header('Content-Range', content_range) + self.send_header('Etag', etag) + self.send_header('Accept-Ranges', 'bytes') + self.send_header('Cache-Control', 'max-age=0') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + # Suppress default logging + pass + + +if __name__ == '__main__': + port = int(sys.argv[1]) + server = http.server.HTTPServer(('127.0.0.1', port), ShrinkHandler) + print(f"Shrink origin listening on port {port}", flush=True) + server.serve_forever() diff --git a/tests/gold_tests/pluginTest/slice/slice_content_shrink.test.py b/tests/gold_tests/pluginTest/slice/slice_content_shrink.test.py new file mode 100644 index 00000000000..14402670122 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/slice_content_shrink.test.py @@ -0,0 +1,94 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from ports import get_port + +Test.Summary = ''' +Slice plugin: verify no integer underflow when content shrinks below block position. + +When origin reports Content-Range with total length smaller than the requested +block position (content shrunk between fetches), the plugin must fail gracefully +rather than underflowing m_blockskip which would corrupt the response. +''' + +Test.SkipUnless(Condition.PluginExists('slice.so'),) +Test.ContinueOnFail = False + +# Define ATS - no cache so every slice sub-request goes to origin +ts = Test.MakeATSProcess("ts", enable_cache=False) + +# Test: Request bytes 14-20 via slice plugin (blockbytes=7) +# Slice will: +# 1. Fetch block 0 (reference): gets CL=21, etag "old" +# 2. Fetch block 2 (interior, skips block 1): gets CL=10, etag "new" (MISMATCH! m_contentlen=10) +# 3. Refetch block 0 (reference): gets CL=10, etag "new" (matches new m_contentlen) +# 4. Enters ActiveRef: blockpos=14, m_contentlen=10 => guard triggers, Fail state +# +# Client should get an error/empty response, NOT corrupted data. + +tr = Test.AddTestRun("Request triggering content shrink underflow guard") + +# Copy and start custom origin server +tr.Setup.CopyAs("shrink_origin.py") + +origin = tr.Processes.Process("origin") +origin_port = get_port(origin, 'http_port') +origin.Command = f'{sys.executable} shrink_origin.py {origin_port}' +origin.Ready = When.PortOpenv4(origin_port) + +# Configure remap to point at our custom origin +ts.Disk.remap_config.AddLines( + [ + f'map http://slice/ http://127.0.0.1:{origin_port}/' + ' @plugin=slice.so @pparam=--blockbytes-test=7', + ]) + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'slice', +}) + +ps = tr.Processes.Default +ps.StartBefore(origin) +ps.StartBefore(Test.Processes.ts) +ps.Command = ( + f'curl -s -D /dev/stdout -o /dev/stderr' + f' -x localhost:{ts.Variables.port}' + f' http://slice/shrink -r 14-20' + f' -w "\\nSIZE:%{{size_download}}"') +ps.Streams.stdout = Testers.ContainsExpression(r"SIZE:0\b", "expected zero client-visible body size") +ps.Streams.stderr = Testers.ExcludesExpression(r".", "expected no client-visible response body") +tr.StillRunningAfter = ts + +# Test 2: Non-block-aligned range. blockbytes=7, client requests bytes=16-20. +# firstblock=2, blockpos=14. Content shrinks to 15. +# m_contentlen(15) > blockpos(14) would pass a blockpos-only guard, but +# m_contentlen(15) <= m_req_range.m_beg(16) catches it. +tr = Test.AddTestRun("Mid-block range: content shrinks above blockpos but below range start") +ps = tr.Processes.Default +ps.Command = ( + f'curl -s -D /dev/stdout -o /dev/stderr' + f' -x localhost:{ts.Variables.port}' + f' http://slice/shrink_mid -r 16-20' + f' -w "\\nSIZE:%{{size_download}}"') +tr.StillRunningAfter = ts + +# Verify the error was logged (our new guard message) +ts.Disk.diags_log.Content = Testers.ContainsExpression("shrunk below requested range start", "expected underflow guard error log") diff --git a/tests/gold_tests/pluginTest/stale_response/stale_response.test.py b/tests/gold_tests/pluginTest/stale_response/stale_response.test.py index b073c45760b..fb72c45723f 100644 --- a/tests/gold_tests/pluginTest/stale_response/stale_response.test.py +++ b/tests/gold_tests/pluginTest/stale_response/stale_response.test.py @@ -34,6 +34,7 @@ class OptionType(Enum): DEFAULT_DIRECTIVES = 1 FORCE_SWR = 2 FORCE_SIE = 3 + MAX_MEMORY_USAGE = 4 assert len({option.value for option in OptionType.__members__.values()}) == len(OptionType.__members__) @@ -68,6 +69,9 @@ def __init__(self, option_type: OptionType, is_global: bool) -> None: elif option_type == OptionType.FORCE_SIE: self._replay_file = "stale_response_with_force_sie.replay.yaml" option_description = f"--force-stale-if-error 30: {plugin_type_description}" + elif option_type == OptionType.MAX_MEMORY_USAGE: + self._replay_file = "stale_response_max_memory.replay.yaml" + option_description = f"--max-memory-usage 256: {plugin_type_description}" tr = Test.AddTestRun(f"stale_response.so Options: {option_description}") @@ -108,6 +112,8 @@ def setupTS(self) -> None: plugin_command += ' --force-stale-while-revalidate 30' elif self._option_type == OptionType.FORCE_SIE: plugin_command += ' --force-stale-if-error 30' + elif self._option_type == OptionType.MAX_MEMORY_USAGE: + plugin_command += ' --max-memory-usage 256' ts.Disk.plugin_config.AddLine(plugin_command) else: # Configure the stale_response plugin for the remap rule. @@ -118,6 +124,8 @@ def setupTS(self) -> None: remap_plugin_config += ' @pparam=--force-stale-while-revalidate @pparam=30' elif self._option_type == OptionType.FORCE_SIE: remap_plugin_config += ' @pparam=--force-stale-if-error @pparam=30' + elif self._option_type == OptionType.MAX_MEMORY_USAGE: + remap_plugin_config += ' @pparam=--max-memory-usage @pparam=256' ts.Disk.records_config.update( { @@ -140,6 +148,14 @@ def setupClient(self, tr: 'TestRun') -> None: def verify_plugin_log(self) -> None: """Verify the contents of the stale_response plugin log.""" + if self._option_type == OptionType.MAX_MEMORY_USAGE: + diagnostic = "response exceeded memory limit; sending stale data" + Test.AddAwaitFileContainsTestRun( + "Verify stale_response max-memory diagnostic", self._ts.Disk.traffic_out.Name, diagnostic) + self._ts.Disk.traffic_out.Content += Testers.ContainsExpression( + diagnostic, "Verify max-memory stale-if-error fallback is logged") + return + swr_log_pattern = "stale-while-revalidate:.*stale.jpeg" sie_log_pattern = "stale-if-error:.*error.jpeg" @@ -162,8 +178,10 @@ def expect_log_entry(pattern: str, description: str) -> None: TestStaleResponse(OptionType.DEFAULT_DIRECTIVES, is_global=True) TestStaleResponse(OptionType.FORCE_SWR, is_global=True) TestStaleResponse(OptionType.FORCE_SIE, is_global=True) +TestStaleResponse(OptionType.MAX_MEMORY_USAGE, is_global=True) TestStaleResponse(OptionType.NONE, is_global=False) TestStaleResponse(OptionType.DEFAULT_DIRECTIVES, is_global=False) TestStaleResponse(OptionType.FORCE_SWR, is_global=False) TestStaleResponse(OptionType.FORCE_SIE, is_global=False) +TestStaleResponse(OptionType.MAX_MEMORY_USAGE, is_global=False) diff --git a/tests/gold_tests/pluginTest/stale_response/stale_response_max_memory.replay.yaml b/tests/gold_tests/pluginTest/stale_response/stale_response_max_memory.replay.yaml new file mode 100644 index 00000000000..3b199dd6285 --- /dev/null +++ b/tests/gold_tests/pluginTest/stale_response/stale_response_max_memory.replay.yaml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Verify that stale-if-error origin fetches honor --max-memory-usage. When the +# origin response cannot fit in the plugin buffer, the plugin should fall back +# to the cached stale response instead of buffering the oversized response. + +sessions: + +- transactions: + + - client-request: + method: GET + url: /pictures/memory-cap.jpeg + version: '1.1' + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, first-request ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, image/jpeg ] + - [ Content-Length, 5 ] + - [ Connection, keep-alive ] + - [ Cache-Control, "max-age=1, stale-if-error=30" ] + - [ X-Response, cached-response ] + content: + encoding: plain + data: STALE + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response, { value: cached-response, as: equal } ] + content: + encoding: plain + data: STALE + verify: { as: equal } + + - client-request: + + delay: 2s + + method: GET + url: /pictures/memory-cap.jpeg + version: '1.1' + headers: + fields: + - [ Host, www.example.com ] + - [ uuid, second-request ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Type, image/jpeg ] + - [ Content-Length, 512 ] + - [ Connection, close ] + - [ Cache-Control, "max-age=1" ] + - [ X-Response, oversized-origin-response ] + content: + size: 512 + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response, { value: cached-response, as: equal } ] + content: + encoding: plain + data: STALE + verify: { as: equal } diff --git a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py index 7a0eafb8429..22646795009 100644 --- a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py +++ b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py @@ -69,6 +69,7 @@ f'map http://www.connect_target.com/ http://127.0.0.1:{server.Variables.http_port}', f'map / http://127.0.0.1:{server.Variables.http_port}', ]) +ts.addPrivateConnectAllowYaml(methods='[ CONNECT, GET, POST ]') # Configure traffic_dump. ts.Disk.plugin_config.AddLine( diff --git a/tests/gold_tests/pluginTest/tsapi/test_TSHttpSsnInfo.cc b/tests/gold_tests/pluginTest/tsapi/test_TSHttpSsnInfo.cc index 2c595b1d5ad..c87bee36651 100644 --- a/tests/gold_tests/pluginTest/tsapi/test_TSHttpSsnInfo.cc +++ b/tests/gold_tests/pluginTest/tsapi/test_TSHttpSsnInfo.cc @@ -53,6 +53,13 @@ handle_ssn_close(TSHttpSsn ssn) logFile << "H2 Frames Received:" << "D" << count[0] << "," << "H" << count[1] << "," << "PR" << count[2] << "," << "RS" << count[3] << "," << "S" << count[4] << "," << "PP" << count[5] << "," << "P" << count[6] << "," << "G" << count[7] << "," << "WU" << count[8] << "," << "C" << count[9] << "," << "U" << count[10] << std::endl; + + // Verify OOB sub_key values don't crash and map to the UNKNOWN bucket + TSMgmtInt oob_val = -1; + TSHttpSsnInfoIntGet(ssn, TS_SSN_INFO_RECEIVED_FRAME_COUNT, &oob_val, 11); + logFile << "H2 OOB(11)=" << oob_val; + TSHttpSsnInfoIntGet(ssn, TS_SSN_INFO_RECEIVED_FRAME_COUNT, &oob_val, 1000); + logFile << ",OOB(1000)=" << oob_val << std::endl; } else { TSMgmtInt count[15]; TSHttpSsnInfoIntGet(ssn, TS_SSN_INFO_RECEIVED_FRAME_COUNT, &count[0], 0); diff --git a/tests/gold_tests/pluginTest/tsapi/test_TSHttpSsnInfo.test.py b/tests/gold_tests/pluginTest/tsapi/test_TSHttpSsnInfo.test.py index 3d1dad1a2f3..9e774d05298 100644 --- a/tests/gold_tests/pluginTest/tsapi/test_TSHttpSsnInfo.test.py +++ b/tests/gold_tests/pluginTest/tsapi/test_TSHttpSsnInfo.test.py @@ -100,6 +100,8 @@ f.Content = "test_TSHttpSsnInfo_plugin_log.gold" f.Content += Testers.ContainsExpression( "H2 Frames Received:D1,H1,PR.,RS0,S2,PP0,P0,G1,WU0,C1,U0", "Expected numbers of frames should be received") +f.Content += Testers.ContainsExpression( + "H2 OOB\\(11\\)=0,OOB\\(1000\\)=0", "OOB sub_key values should map to UNKNOWN bucket without crashing") # We cannot test this on H3 now because the test plugin does not work on H3 sessions # f.Content += Testers.ContainsExpression("H3 Frames Received:D1,H1,Ra0,CP0,S1,PP0,Rb0,G0,Rc0,Rd0,UND0,UND0,UND0,MPI0,U0", # "Expected numbers of frames should be received") diff --git a/tests/gold_tests/pluginTest/url_sig/url_sig.gold b/tests/gold_tests/pluginTest/url_sig/url_sig.gold index 9c43b3c41f6..a703398484b 100644 --- a/tests/gold_tests/pluginTest/url_sig/url_sig.gold +++ b/tests/gold_tests/pluginTest/url_sig/url_sig.gold @@ -7,6 +7,9 @@ < HTTP/1.1 403 Forbidden < HTTP/1.1 403 Forbidden < HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden +< HTTP/1.1 403 Forbidden < HTTP/1.1 200 OK < HTTP/1.1 200 OK < HTTP/1.1 200 OK diff --git a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py index 373255cd633..7a497e15384 100644 --- a/tests/gold_tests/pluginTest/url_sig/url_sig.test.py +++ b/tests/gold_tests/pluginTest/url_sig/url_sig.test.py @@ -223,6 +223,36 @@ p.ReturnCode = 0 p.Streams.stdout = Testers.ContainsExpression("HTTP.*403", "Should receive 403 Forbidden") +# With client / MD5 / Only C parameter -- truncated query string. +# +tr = Test.AddTestRun("Truncated query string with only client IP should fail") +p = tr.MakeCurlCommand( + f"--verbose --proxy http://127.0.0.1:{ts.Variables.port} 'http://seven.eight.nine/" + "foo/abcde/qrstuvwxyz?C=127.0.0.1'" + + LogTee, + ts=ts) +p.ReturnCode = 0 +p.Streams.stdout = Testers.ContainsExpression("HTTP.*403", "Should receive 403 Forbidden") + +# With client / MD5 / C parameter last in query -- missing trailing delimiter. +# +tr = Test.AddTestRun("Client IP as final query parameter should fail") +p = tr.MakeCurlCommand( + f"--verbose --proxy http://127.0.0.1:{ts.Variables.port} 'http://seven.eight.nine/" + + "foo/abcde/qrstuvwxyz?E=33046620008&A=2&K=13&P=101&S=d1f352d4f1d931ad2f441013402d93f8&C=127.0.0.1'" + LogTee, + ts=ts) +p.ReturnCode = 0 +p.Streams.stdout = Testers.ContainsExpression("HTTP.*403", "Should receive 403 Forbidden") + +# With client / MD5 / C parameter has empty value. +# +tr = Test.AddTestRun("Empty client IP value should fail") +p = tr.MakeCurlCommand( + f"--verbose --proxy http://127.0.0.1:{ts.Variables.port} 'http://seven.eight.nine/" + + "foo/abcde/qrstuvwxyz?C=&E=33046620008&A=2&K=13&P=101&S=d1f352d4f1d931ad2f441013402d93f8'" + LogTee, + ts=ts) +p.ReturnCode = 0 +p.Streams.stdout = Testers.ContainsExpression("HTTP.*403", "Should receive 403 Forbidden") + # Success tests. # Test excl_regex feature - URLs matching the exclusion regex should bypass signature checks. diff --git a/tests/gold_tests/pluginTest/webp_transform/replay/webp_chunked_cap.replay.yaml b/tests/gold_tests/pluginTest/webp_transform/replay/webp_chunked_cap.replay.yaml new file mode 100644 index 00000000000..978b7174eea --- /dev/null +++ b/tests/gold_tests/pluginTest/webp_transform/replay/webp_chunked_cap.replay.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Origin side for the chunked no-Content-Length cap-trip test. The verifier +# server returns a 20 MiB image/jpeg body framed with Transfer-Encoding: +# chunked and no Content-Length, so webp_transform cannot decline it up front +# and must fall back to the per-transaction cap inside consume(). + +meta: + version: "1.0" + +sessions: +- transactions: + - client-request: + method: "GET" + url: /chunked-huge.jpg + version: "1.1" + headers: + fields: + - [Host, example.com] + - [uuid, chunked-huge] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Type, image/jpeg] + - [Transfer-Encoding, chunked] + content: + encoding: plain + size: 20971520 diff --git a/tests/gold_tests/pluginTest/webp_transform/webp_transform_cache_refused.test.py b/tests/gold_tests/pluginTest/webp_transform/webp_transform_cache_refused.test.py new file mode 100644 index 00000000000..ae9c431b84d --- /dev/null +++ b/tests/gold_tests/pluginTest/webp_transform/webp_transform_cache_refused.test.py @@ -0,0 +1,90 @@ +''' +An over-cap response that webp_transform refuses with a 502 must not poison the +cache. + +The refused path produces a zero-length body and rewrites the client status to +502 in the send-response-headers hook. The cacheable object, however, is driven +by the origin's 200 server response plus the transform's (empty) output, and the +read hook already stamped a transformed Content-Type onto that server response. +If nothing marks the refused response uncacheable, ATS can store a 200 with an +empty body labeled image/webp and serve that poisoned entry to later clients +while the original requester saw a 502. + +This test enables caching (and forces caching of responses without explicit +freshness headers), drives the over-cap chunked body twice through the same URL, +and asserts BOTH requests are refused with a 502 and an empty body. A cached +poisoned 200 would show up as a 200 (and/or a non-zero body) on the second +request. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = 'An over-cap refused (502) response must not poison the cache' + +Test.SkipUnless(Condition.PluginExists('webp_transform.so')) + +Test.ContinueOnFail = True + +server = Test.MakeVerifierServerProcess("server", "replay/webp_chunked_cap.replay.yaml") + +ts = Test.MakeATSProcess("ts", enable_cache=True) +ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'webp_transform', + # Cache aggressively: store 200s even without explicit freshness headers, + # so a refused response that is not marked no-store WOULD be cached. This + # makes the poisoning observable if the fix regresses. + 'proxy.config.http.cache.required_headers': 0, + 'proxy.config.http.cache.ignore_client_cc_max_age': 1, + }) +ts.Disk.plugin_config.AddLine('webp_transform.so convert_to_webp') +ts.Disk.remap_config.AddLine('map http://127.0.0.1:{0}/ http://127.0.0.1:{0}/'.format(server.Variables.http_port)) + +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "response body exceeds cap", "The in-transform cap must trip for the chunked body with no Content-Length") + +# Request 1: origin is contacted, the cap trips mid-stream, client gets a 502. +tr = Test.AddTestRun("first request: over-cap chunked body refused with 502") +tr.MakeCurlCommand( + '-sS -D - -o /dev/null -w "size_download=%{{size_download}}" -x 127.0.0.1:{0} ' + '-H "Accept: image/webp" -H "uuid: chunked-huge" http://127.0.0.1:{1}/chunked-huge.jpg'.format( + ts.Variables.port, server.Variables.http_port), + ts=ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "HTTP/1.1 502", "First over-cap request must be refused with a 502") +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "size_download=0", "First refused response must have an empty body") + +# Request 2: if the refused response was cached, this hit would serve a poisoned +# 200 with an empty (or mislabeled) body. The fix marks the refused response +# uncacheable, so this must also be a fresh 502 with an empty body. +tr2 = Test.AddTestRun("second request: must not be served a cached poisoned 200") +tr2.MakeCurlCommand( + '-sS -D - -o /dev/null -w "size_download=%{{size_download}}" -x 127.0.0.1:{0} ' + '-H "Accept: image/webp" -H "uuid: chunked-huge" http://127.0.0.1:{1}/chunked-huge.jpg'.format( + ts.Variables.port, server.Variables.http_port), + ts=ts) +tr2.Processes.Default.ReturnCode = 0 +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "HTTP/1.1 502", "Second request must also be a 502, not a cached poisoned 200") +tr2.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "HTTP/1.1 200", "Second request must not be served a cached 200 (cache poisoning)") +tr2.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "size_download=0", "Second response body must be empty; no cached oversized/empty image") diff --git a/tests/gold_tests/pluginTest/webp_transform/webp_transform_chunked_cap.test.py b/tests/gold_tests/pluginTest/webp_transform/webp_transform_chunked_cap.test.py new file mode 100644 index 00000000000..5c599c34d44 --- /dev/null +++ b/tests/gold_tests/pluginTest/webp_transform/webp_transform_chunked_cap.test.py @@ -0,0 +1,74 @@ +''' +webp_transform must refuse an over-cap chunked response +rather than forward an oversized, mislabeled body. + +When the origin advertises a Content-Length over the cap, webp_transform +declines the transform up front and the original response passes through. A +chunked response has no Content-Length, so that early check cannot fire and the +per-transaction cap inside consume() is the only bound. The transform produces +nothing until handleInputComplete, so when the cap is exceeded it drops the +buffer, produces no body, and rewrites the status to 502 in the +send-response-headers hook. The client gets a 502 with an empty body rather +than the oversized image. (Transaction::error() cannot be used; it asserts once +the response is in flight, so the status is changed at send time instead.) + +This test drives a 20 MiB image/jpeg framed with Transfer-Encoding: chunked +through the plugin and confirms the cap trips mid-stream, ATS does not buffer +the whole body or crash, and the client receives a 502 with a zero-length body +rather than the full 20 MiB image. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = 'webp_transform returns 502 for an over-cap chunked no-Content-Length response' + +Test.SkipUnless(Condition.PluginExists('webp_transform.so')) + +Test.ContinueOnFail = True + +server = Test.MakeVerifierServerProcess("server", "replay/webp_chunked_cap.replay.yaml") + +ts = Test.MakeATSProcess("ts", enable_cache=False) +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'webp_transform', +}) +ts.Disk.plugin_config.AddLine('webp_transform.so convert_to_webp') +ts.Disk.remap_config.AddLine('map http://127.0.0.1:{0}/ http://127.0.0.1:{0}/'.format(server.Variables.http_port)) + +# No Content-Length, so the up-front decline cannot fire; the cap trips inside +# consume(), which emits this message at ERROR level to diags.log. Asserting it +# both confirms the in-transform cap engaged and tells autest the ERROR line is +# expected (the default check fails on any ERROR in diags.log). +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "response body exceeds cap", "The in-transform cap must trip for a chunked body with no Content-Length") + +tr = Test.AddTestRun("over-cap chunked body is refused with a 502 and no body") +# -w reports the downloaded body size; braces are doubled so str.format leaves +# the curl %{...} variable intact. +tr.MakeCurlCommand( + '-sS -D - -o /dev/null -w "size_download=%{{size_download}}" -x 127.0.0.1:{0} ' + '-H "Accept: image/webp" -H "uuid: chunked-huge" http://127.0.0.1:{1}/chunked-huge.jpg'.format( + ts.Variables.port, server.Variables.http_port), + ts=ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.ReturnCode = 0 +# The client gets a 502 with a zero-length body: the oversized image is refused, +# not forwarded. A full response would download 20 MiB. +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("HTTP/1.1 502", "Over-cap image must be refused with a 502") +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "size_download=0", "Body must be empty; the full 20 MiB image must not be forwarded") diff --git a/tests/gold_tests/pluginTest/webp_transform/webp_transform_decode_limit.test.py b/tests/gold_tests/pluginTest/webp_transform/webp_transform_decode_limit.test.py new file mode 100644 index 00000000000..936a0cd5722 --- /dev/null +++ b/tests/gold_tests/pluginTest/webp_transform/webp_transform_decode_limit.test.py @@ -0,0 +1,117 @@ +''' +A small image that declares dimensions over the ImageMagick decode limits must +not be decoded into a giant pixel buffer; the transform reverts to the original +bytes instead of crashing or exhausting memory. + +The byte cap (max_buffer_size) does not help here: a tiny encoded image can sit +well under the cap yet declare enormous dimensions that decode into gigabytes of +pixels. TSPluginInit installs Magick::ResourceLimits (width/height/area/memory/ +map and disk(0)) so such an image fails as a caught Magick::Error and the plugin +reverts to the original bytes. + +This test serves a minimal, over-wide WebP (VP8L) image: a real RIFF/WEBP/VP8L +signature followed by a bit-packed header declaring 16129x2 (16129 > the +plugin's 16000 px width limit), with no bitstream payload beyond the header. +ImageMagick's decoder reads the declared width/height straight out of that +header and throws before it would ever need pixel data; the plugin catches it, +logs an ImageMagick error, and forwards the original bytes. The test asserts +ATS does not crash (the client still gets a 200), the decode-limit error is +logged, and the original body is returned unchanged (not a converted jpeg). + +The plugin's has_signature_for() guard checks the declared encoding's magic +bytes before ImageMagick ever sees the body, so the served body must carry a +real signature -- an arbitrary/mislabeled body would be caught by that guard +instead and never reach the decode-limit code path this test targets. WebP is +used (rather than PNG or JPEG) because it is the only one of the three +signatures the plugin recognizes that can be built entirely from bytes <=0x7f; +the origin server writes the body via a UTF-8 encode, so any byte over 0x7f +would not survive the round trip unchanged. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct + +Test.Summary = 'An over-dimension image is rejected by the decode limits and reverts to the original' + +Test.SkipUnless(Condition.PluginExists('webp_transform.so')) + +Test.ContinueOnFail = True + +# A minimal WebP (VP8L) image, 16129 px wide by 2 px tall. 16129 > the plugin's +# 16000 px width ResourceLimit, so ImageMagick rejects it on read. There is no +# bitstream data beyond the 5-byte VP8L header (signature byte + packed +# width/height), so the body is a few dozen bytes, far under the buffer cap -- +# the decode-side limit is what must catch it, not the byte cap. 16129x2 is +# also chosen so every byte of the packed header is <=0x7f (see module +# docstring for why that matters). +W, H = 16129, 2 +vp8l_payload = b'\x2f' + struct.pack('jpeg conversion; the body +# carries a real RIFF/WEBP/VP8L signature so has_signature_for() lets it +# through to ImageMagick, which reads the over-limit dimensions from the VP8L +# header. +server.addResponse( + "sessionlog.json", { + "headers": "GET /overwide.webp HTTP/1.1\r\nHost: *\r\n\r\n", + "timestamp": "1", + "body": "" + }, { + "headers": + "HTTP/1.1 200 OK\r\nContent-Type: image/webp\r\nContent-Length: {0}\r\nConnection: close\r\n\r\n".format(WEBP_LEN), + "timestamp": "1", + "body": webp_body + }) + +ts = Test.MakeATSProcess("ts", enable_cache=False) +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'webp_transform', +}) +ts.Disk.plugin_config.AddLine('webp_transform.so convert_to_jpeg') +ts.Disk.remap_config.AddLine('map http://127.0.0.1:{0}/ http://127.0.0.1:{0}/'.format(server.Variables.Port)) + +# The decode-limit failure is caught and logged as an ImageMagick error at ERROR +# level. Asserting it confirms the resource limit engaged (not the byte cap) and +# tells autest the ERROR line is expected. +ts.Disk.diags_log.Content = Testers.ContainsExpression( + "ImageMagick.. error", "The decode-side ResourceLimit must reject the over-dimension image") + +tr = Test.AddTestRun("over-dimension image reverts to original instead of crashing") +tr.MakeCurlCommand( + '-sS -D - -o /dev/null -w "size_download=%{{size_download}}" -x 127.0.0.1:{0} ' + '-H "Accept: image/jpeg" http://127.0.0.1:{1}/overwide.webp'.format(ts.Variables.port, server.Variables.Port), + ts=ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.ReturnCode = 0 +# ATS must stay up and return the original image, not crash and not 502 (the byte +# cap is not exceeded). The reverted body is the original bytes, so its size +# equals the source image, not a smaller converted jpeg. +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression( + "HTTP/1.1 200", "Client must get a 200 (decode failed over the limit, original forwarded), not a crash or 502") +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "size_download={0}".format(WEBP_LEN), "Original bytes must be forwarded unchanged, not a converted jpeg") diff --git a/tests/gold_tests/pluginTest/webp_transform/webp_transform_max_buffer_size.test.py b/tests/gold_tests/pluginTest/webp_transform/webp_transform_max_buffer_size.test.py new file mode 100644 index 00000000000..7cec80ca25e --- /dev/null +++ b/tests/gold_tests/pluginTest/webp_transform/webp_transform_max_buffer_size.test.py @@ -0,0 +1,108 @@ +''' +The max_buffer_size plugin argument overrides the default 16 MiB buffer cap, and +malformed values are rejected so a bad config cannot silently disable the cap. + +Two ATS instances: + - one loads webp_transform with max_buffer_size=1M and is driven with a 2 MiB + Content-Length image, proving the override is parsed (K/M/G suffix) and + applied: the transform is declined at the 1 MiB cap, not the 16 MiB default. + - one loads webp_transform with several malformed max_buffer_size values + (negative, bad suffix, suffix-multiply overflow) and asserts each is rejected + with "keeping default 16777216", proving a bad value falls back to the safe + default rather than wrapping to a huge or unintended cap. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = 'max_buffer_size overrides the cap and rejects malformed values' + +Test.SkipUnless(Condition.PluginExists('webp_transform.so')) + +Test.ContinueOnFail = True + +# ---- Instance 1: a valid 1M override takes effect ---- +server = Test.MakeOriginServer("server") + +# 2 MiB image/jpeg: over the 1 MiB override, well under the 16 MiB default. With +# the override in effect the up-front decline fires at 1 MiB; with the default it +# would not. +TWO_MIB = 2 * 1024 * 1024 +server.addResponse( + "sessionlog.json", { + "headers": "GET /two_mib.jpg HTTP/1.1\r\nHost: *\r\n\r\n", + "timestamp": "1", + "body": "" + }, { + "headers": + "HTTP/1.1 200 OK\r\nContent-Type: image/jpeg\r\nContent-Length: {0}\r\nConnection: close\r\n\r\n".format(TWO_MIB), + "timestamp": "1", + "body": "A" * TWO_MIB + }) + +ts = Test.MakeATSProcess("ts", enable_cache=False) +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'webp_transform', +}) +ts.Disk.plugin_config.AddLine('webp_transform.so convert_to_webp max_buffer_size=1M') +ts.Disk.remap_config.AddLine('map http://127.0.0.1:{0}/ http://127.0.0.1:{0}/'.format(server.Variables.Port)) + +# 1M == 1048576. Seeing the decline name 1048576 (not the 16777216 default) +# proves parse_size parsed the M suffix and the override is what is enforced. +ts.Disk.traffic_out.Content = Testers.ContainsExpression( + "exceeds cap 1048576", "max_buffer_size=1M must override the default and decline at 1 MiB") + +tr = Test.AddTestRun("2 MiB body declined at the 1 MiB override") +tr.MakeCurlCommand( + '-sS -D - -o /dev/null -x 127.0.0.1:{0} -H "Accept: image/webp" http://127.0.0.1:{1}/two_mib.jpg'.format( + ts.Variables.port, server.Variables.Port), + ts=ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("HTTP/1.1 200", "Declined response is a 200 passthrough") +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "[Cc]ontent-[Tt]ype: image/jpeg", "Declined response keeps its original image/jpeg type") + +# ---- Instance 2: malformed values are rejected and keep the default ---- +ts_bad = Test.MakeATSProcess("ts_bad", enable_cache=False) +ts_bad.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'webp_transform', +}) +# Negative (sign guard), bad suffix, and a value that fits in u64 but overflows +# size_t when multiplied by the G suffix (multiply-overflow guard). All must be +# rejected; none may install a cap, so the 16 MiB default must survive. +ts_bad.Disk.plugin_config.AddLine( + 'webp_transform.so convert_to_webp max_buffer_size=-1 max_buffer_size=8X max_buffer_size=20000000000G') +ts_bad.Disk.remap_config.AddLine('map http://127.0.0.1:{0}/ http://127.0.0.1:{0}/'.format(server.Variables.Port)) + +# TSError() writes these to diags.log at ERROR level. Asserting them here both +# confirms parse_size rejected each value and tells autest the ERROR lines are +# expected (the default check fails on any ERROR in diags.log). +ts_bad.Disk.diags_log.Content = Testers.ContainsExpression( + "invalid max_buffer_size=-1, keeping default 16777216", "Negative value must be rejected") +ts_bad.Disk.diags_log.Content += Testers.ContainsExpression( + "invalid max_buffer_size=8X, keeping default 16777216", "Bad suffix must be rejected") +ts_bad.Disk.diags_log.Content += Testers.ContainsExpression( + "invalid max_buffer_size=20000000000G, keeping default 16777216", "Suffix-multiply overflow must be rejected") + +tr_bad = Test.AddTestRun("malformed max_buffer_size values are rejected, default retained") +tr_bad.MakeCurlCommand( + '-sS -o /dev/null -x 127.0.0.1:{0} http://127.0.0.1:{1}/two_mib.jpg'.format(ts_bad.Variables.port, server.Variables.Port), + ts=ts_bad) +tr_bad.Processes.Default.StartBefore(ts_bad) +tr_bad.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/pluginTest/webp_transform/webp_transform_size_cap.test.py b/tests/gold_tests/pluginTest/webp_transform/webp_transform_size_cap.test.py new file mode 100644 index 00000000000..11c2ea326a0 --- /dev/null +++ b/tests/gold_tests/pluginTest/webp_transform/webp_transform_size_cap.test.py @@ -0,0 +1,115 @@ +''' +webp_transform must not buffer unbounded response bodies. + +ImageTransform buffered the entire origin response in memory before handing it +to ImageMagick, so a large or malicious image response could exhaust proxy +memory, and a decode could throw an exception the narrow catch did not handle. + +The fix bounds this two ways: when the origin advertises a Content-Length over +the 16 MiB cap, the transform is declined up front and the original response +passes through untouched and keeps its original Content-Type; bodies without a +usable Content-Length are still bounded by a per-transaction cap inside the +transform. + +This test drives a 20 MiB image/jpeg with a Content-Length through ATS with +webp_transform loaded (convert_to_webp) over both HTTP/1.1 and HTTP/2, and +confirms the client gets a 200 with Content-Type image/jpeg (declined, not a +mislabeled image/webp) rather than ATS buffering the body or crashing. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = 'webp_transform must not buffer unbounded response bodies' + +Test.SkipUnless( + Condition.PluginExists('webp_transform.so'), + Condition.HasCurlFeature('http2'), +) + +Test.ContinueOnFail = True + +server = Test.MakeOriginServer("server") + +# A 20 MiB image/jpeg with a Content-Length over the 16 MiB cap. ImageMagick +# would reject these bytes, so the point is that the transform is declined up +# front and ATS never buffers or decodes them. +BIG = 20 * 1024 * 1024 +body = "A" * BIG +server.addResponse( + "sessionlog.json", { + "headers": "GET /huge.jpg HTTP/1.1\r\nHost: *\r\n\r\n", + "timestamp": "1", + "body": "" + }, { + "headers": "HTTP/1.1 200 OK\r\nContent-Type: image/jpeg\r\nContent-Length: {0}\r\n\r\n".format(BIG), + "timestamp": "1", + "body": body + }) + +ts = Test.MakeATSProcess("ts", enable_tls=True, enable_cache=False) +ts.addDefaultSSLFiles() + +ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + +ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'webp_transform', + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + }) + +ts.Disk.plugin_config.AddLine('webp_transform.so convert_to_webp') + +# Plugin debug goes to traffic.out. Asserting the decline message naming the +# default 16 MiB cap (16777216) proves the plugin actually engaged and declined, +# rather than the response merely passing through untouched. +ts.Disk.traffic_out.Content = Testers.ContainsExpression( + "exceeds cap 16777216", "Plugin must engage and decline at the default 16 MiB cap") + +# Identity rule for the HTTP/1.1 forward-proxy run, plus a catch-all so the +# HTTP/2 reverse-proxy run resolves to the same origin. +ts.Disk.remap_config.AddLine('map http://127.0.0.1:{0}/ http://127.0.0.1:{0}/'.format(server.Variables.Port)) +ts.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}/'.format(server.Variables.Port)) + +# The buffering DoS is in the origin-response transform, so it is independent of +# the client protocol. Exercise both H1 and H2 to confirm the oversized body is +# declined and the client gets a truthful 200 image/jpeg either way. +tr = Test.AddTestRun("HTTP/1.1 client: oversized body declined, original type preserved") +tr.MakeCurlCommandMulti( + '{curl} -sS -D - -o /dev/null -x 127.0.0.1:TSPORT -H "Accept: image/webp" http://127.0.0.1:OPORT/huge.jpg'.replace( + 'TSPORT', str(ts.Variables.port)).replace('OPORT', str(server.Variables.Port)), + ts=ts) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("HTTP/1.1 200", "H1 client must see 200 OK, not a crash") +# The response must keep its truthful image/jpeg type. A mislabeled image/webp +# would mean the original bytes were forwarded under a transformed Content-Type. +tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "[Cc]ontent-[Tt]ype: image/jpeg", "Declined response must keep its original image/jpeg type") +tr.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "image/webp", "Declined response must not be mislabeled image/webp") + +tr2 = Test.AddTestRun("HTTP/2 client: oversized body declined, original type preserved") +tr2.MakeCurlCommand( + '--http2 -k -sS -D - -o /dev/null -H "Accept: image/webp" https://127.0.0.1:{0}/huge.jpg'.format(ts.Variables.ssl_port), ts=ts) +tr2.Processes.Default.ReturnCode = 0 +tr2.Processes.Default.Streams.stdout = Testers.ContainsExpression("HTTP/2 200", "H2 client must see 200 OK, not a crash") +tr2.Processes.Default.Streams.stdout += Testers.ContainsExpression( + "[Cc]ontent-[Tt]ype: image/jpeg", "Declined response must keep its original image/jpeg type") +tr2.Processes.Default.Streams.stdout += Testers.ExcludesExpression( + "image/webp", "Declined response must not be mislabeled image/webp") diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap_long_url/x_remap_long_url.replay.yaml b/tests/gold_tests/pluginTest/xdebug/x_remap_long_url/x_remap_long_url.replay.yaml new file mode 100644 index 00000000000..01b01b28199 --- /dev/null +++ b/tests/gold_tests/pluginTest/xdebug/x_remap_long_url/x_remap_long_url.replay.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: 'xdebug X-Remap injection must not OOB read on long URLs' + + server: + name: 'server' + + client: + name: 'client' + + ats: + name: 'ts' + + plugin_config: + - 'xdebug.so --enable=x-remap' + + remap_config: + - from: 'http://example.com' + to: 'http://127.0.0.1:{SERVER_HTTP_PORT}' + +sessions: +- transactions: + - client-request: + method: GET + version: "1.1" + url: "http://example.com/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + headers: + fields: + - [Host, example.com] + - [X-Debug, X-Remap] + - [Content-Length, "0"] + - [uuid, long-url-1] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, "0"] + + proxy-response: + status: 200 + headers: + fields: + - [X-Remap, {value: "from=", as: contains}] diff --git a/tests/gold_tests/pluginTest/xdebug/x_remap_long_url/x_remap_long_url.test.py b/tests/gold_tests/pluginTest/xdebug/x_remap_long_url/x_remap_long_url.test.py new file mode 100644 index 00000000000..d68f7b148b7 --- /dev/null +++ b/tests/gold_tests/pluginTest/xdebug/x_remap_long_url/x_remap_long_url.test.py @@ -0,0 +1,30 @@ +""" +Verify xdebug X-Remap header injection does not OOB read with long URLs. +""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Send a request with a URL long enough that the combined from/to URL +output exceeds the 2KB stack buffer xdebug uses for the X-Remap header. +Previously, snprintf's would-have-written return value was passed +straight to TSMimeHdrFieldValueStringInsert, causing an OOB read of +adjacent stack memory into the response header. +''' + +Test.SkipUnless(Condition.PluginExists('xdebug.so')) + +Test.ATSReplayTest(replay_file='x_remap_long_url.replay.yaml') diff --git a/tests/gold_tests/proxy_protocol/proxy_protocol.test.py b/tests/gold_tests/proxy_protocol/proxy_protocol.test.py index fc675b9f424..d8fe27b0713 100644 --- a/tests/gold_tests/proxy_protocol/proxy_protocol.test.py +++ b/tests/gold_tests/proxy_protocol/proxy_protocol.test.py @@ -168,6 +168,7 @@ def setupTS(self, tr: 'TestRun') -> None: self._ts.Disk.records_config.update({ "proxy.config.http.connect_ports": f'{self._server.Variables.https_port}', }) + self._ts.addPrivateConnectAllowYaml() self._ts.Disk.sni_yaml.AddLines( [ diff --git a/tests/gold_tests/remap/regex_map_anchor.test.py b/tests/gold_tests/remap/regex_map_anchor.test.py new file mode 100644 index 00000000000..7b27a0036cc --- /dev/null +++ b/tests/gold_tests/remap/regex_map_anchor.test.py @@ -0,0 +1,35 @@ +''' +Verify regex_map performs full-hostname matching. + +A regex_map rule must match the entire request hostname, never a host in +which the configured value appears only as a substring. For a rule covering +"cdn.example.com": + - "prefix.cdn.example.com" must NOT match: rejected by the start anchor. + - "cdn.example.com.evil.com" must NOT match: rejected by full-hostname matching + (start-anchoring alone would still match it). + - "cdn.example.computer" must NOT match: shared prefix, no label boundary. + - "cdn.example.com." must NOT match: trailing-dot FQDN. + - "cdn.example.com" must match (exact), and reach the origin. + - "CDN.EXAMPLE.COM" must match (case-insensitive), and reach the origin. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify regex_map performs full-hostname matching. +''' + +Test.ATSReplayTest(replay_file="replay/regex_map_anchor.replay.yaml") diff --git a/tests/gold_tests/remap/replay/regex_map_anchor.replay.yaml b/tests/gold_tests/remap/replay/regex_map_anchor.replay.yaml new file mode 100644 index 00000000000..9a61d92dba7 --- /dev/null +++ b/tests/gold_tests/remap/replay/regex_map_anchor.replay.yaml @@ -0,0 +1,204 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: '1.0' + +autest: + description: 'Verify regex_map performs full-hostname matching' + + dns: + name: 'dns' + + server: + name: 'server' + + client: + name: 'client' + + ats: + name: 'ts' + + process_config: + enable_cache: false + + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: 'http|url_rewrite_regex' + proxy.config.url_remap.remap_required: 1 + + remap_config: + # The rule must match ONLY the exact host "cdn.example.com", never a host + # in which that string appears as a leading or trailing substring. + - 'regex_map http://cdn\.example\.com/ http://127.0.0.1:{SERVER_HTTP_PORT}/' + +sessions: +- transactions: + + # Leading-prefix attack: the configured host appears only after a prefix. + # Rejected by the start anchor (RE_ANCHORED) alone: the match cannot begin at + # offset 0. ATS must return 404 and never forward to the origin. + - client-request: + method: GET + url: /path/test.html + version: '1.1' + headers: + fields: + - [Host, prefix.cdn.example.com] + - [uuid, leading-prefix-attack] + + proxy-request: + expect: absent + + proxy-response: + status: 404 + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 0] + + # Trailing-content attack: the host starts with the configured value but + # continues past it. Start-anchoring alone would still match this; it is + # rejected only by full-hostname matching (RE_FULL_MATCH). ATS must return 404 + # and never forward to the origin. + - client-request: + method: GET + url: /path/test.html + version: '1.1' + headers: + fields: + - [Host, cdn.example.com.evil.com] + - [uuid, trailing-content-attack] + + proxy-request: + expect: absent + + proxy-response: + status: 404 + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 0] + + # Shared-prefix attack with no label boundary ("cdn.example.com" + "puter"). + # Distinguishes true full-string matching from a naive label-boundary heuristic. + # Rejected by full-hostname matching. ATS must return 404 and not forward. + - client-request: + method: GET + url: /path/test.html + version: '1.1' + headers: + fields: + - [Host, cdn.example.computer] + - [uuid, shared-prefix-attack] + + proxy-request: + expect: absent + + proxy-response: + status: 404 + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 0] + + # Trailing-dot FQDN: "cdn.example.com." is the same name to DNS but carries a + # trailing label separator. With anchored full-hostname matching it does not + # match the rule, so ATS returns 404 and does not forward. This transaction + # pins that behavior against silent drift. + - client-request: + method: GET + url: /path/test.html + version: '1.1' + headers: + fields: + - [Host, cdn.example.com.] + - [uuid, trailing-dot] + + proxy-request: + expect: absent + + proxy-response: + status: 404 + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 0] + + # Exact match: the request host is exactly the configured host. It must match, + # be forwarded to the origin, and return 200. The origin stamps a marker header + # that the client verifies, proving the 200 came from the origin and not from a + # synthesized/error response. + - client-request: + method: GET + url: /path/test.html + version: '1.1' + headers: + fields: + - [Host, cdn.example.com] + - [uuid, exact-match] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 0] + - [X-Origin-Marker, served-by-origin] + + proxy-response: + status: 200 + headers: + fields: + - [X-Origin-Marker, {value: served-by-origin, as: equal}] + + # Case-insensitive exact match: the request host differs only in case. The + # lookup lowercases the host before matching, so this must still match, be + # forwarded, and return the origin's 200 (verified via the marker header). + - client-request: + method: GET + url: /path/test.html + version: '1.1' + headers: + fields: + - [Host, CDN.EXAMPLE.COM] + - [uuid, case-insensitive-match] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [Content-Length, 0] + - [X-Origin-Marker, served-by-origin] + + proxy-response: + status: 200 + headers: + fields: + - [X-Origin-Marker, {value: served-by-origin, as: equal}] diff --git a/tests/gold_tests/timeout/tunnel_active_timeout.test.py b/tests/gold_tests/timeout/tunnel_active_timeout.test.py index 80c5840acea..d6bf18e9c2f 100644 --- a/tests/gold_tests/timeout/tunnel_active_timeout.test.py +++ b/tests/gold_tests/timeout/tunnel_active_timeout.test.py @@ -51,6 +51,7 @@ }) ts.Disk.remap_config.AddLine(f'map / https://127.0.0.1:{server.Variables.SSL_Port}') +ts.addPrivateConnectAllowYaml() # Configure custom log format to capture squid code ts.Disk.logging_yaml.AddLines( diff --git a/tests/gold_tests/tls/gold/tls-tunnel-metrics.gold b/tests/gold_tests/tls/gold/tls-tunnel-metrics.gold index 8c3cfa1dc90..708f7cc79d3 100644 --- a/tests/gold_tests/tls/gold/tls-tunnel-metrics.gold +++ b/tests/gold_tests/tls/gold/tls-tunnel-metrics.gold @@ -1,6 +1,6 @@ -proxy.process.http.total_incoming_connections 13 -proxy.process.http.total_client_connections 13 -proxy.process.http.total_client_connections_ipv4 13 +proxy.process.http.total_incoming_connections 14 +proxy.process.http.total_client_connections 14 +proxy.process.http.total_client_connections_ipv4 14 proxy.process.http.total_client_connections_ipv6 0 proxy.process.http.total_server_connections 0 proxy.process.http2.total_client_connections 2 diff --git a/tests/gold_tests/tls/proxy_protocol_addressless.test.py b/tests/gold_tests/tls/proxy_protocol_addressless.test.py new file mode 100644 index 00000000000..e135aa08947 --- /dev/null +++ b/tests/gold_tests/tls/proxy_protocol_addressless.test.py @@ -0,0 +1,81 @@ +''' +Verify handling of addressless PROXY protocol headers. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +Test.Summary = ''' +Verify listeners handle addressless PROXY protocol headers. +''' + +host = 'addressless.proxy.protocol.test' + + +def add_addressless_proxy_protocol_run(protocol_version: int, description: str, use_tls: bool) -> None: + """Add a test run for an addressless PROXY header.""" + mode = 'tls' if use_tls else 'http' + ts = Test.MakeATSProcess(f'ts_{mode}_v{protocol_version}', enable_tls=use_tls, enable_cache=False, enable_proxy_protocol=True) + server = Test.MakeOriginServer(f'server_{mode}_v{protocol_version}') + server.ReturnCode = 0 + + request_header = { + "headers": f"GET /proxy_protocol HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + } + response_header = {"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": "ok"} + server.addResponse("sessionlog.json", request_header, response_header) + + if use_tls: + ts.addDefaultSSLFiles() + ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + + ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server.Variables.Port}/') + records_config = { + 'proxy.config.http.proxy_protocol_allowlist': '127.0.0.1', + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'proxyprotocol', + } + if use_tls: + records_config.update( + { + 'proxy.config.ssl.server.cert.path': ts.Variables.SSLDir, + 'proxy.config.ssl.server.private_key.path': ts.Variables.SSLDir, + }) + ts.Disk.records_config.update(records_config) + + tr = Test.AddTestRun(description) + tr.TimeOut = 10 + tr.Setup.Copy('proxy_protocol_client.py') + port = ts.Variables.proxy_protocol_ssl_port if use_tls else ts.Variables.proxy_protocol_port + tr.Processes.Default.Command = ( + f'{sys.executable} proxy_protocol_client.py 127.0.0.1 {port} {host} ' + f'127.0.0.1 127.0.0.1 60123 {server.Variables.Port} ' + f'{protocol_version} --addressless') + if use_tls: + tr.Processes.Default.Command += ' --https' + tr.Processes.Default.StartBefore(server) + tr.Processes.Default.StartBefore(ts) + tr.ReturnCode = 0 + tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Verify a successful response is received") + + +add_addressless_proxy_protocol_run(1, 'PROXY v1 UNKNOWN before HTTP/1', use_tls=False) +add_addressless_proxy_protocol_run(2, 'PROXY v2 LOCAL before HTTP/1', use_tls=False) +add_addressless_proxy_protocol_run(1, 'PROXY v1 UNKNOWN before TLS', use_tls=True) +add_addressless_proxy_protocol_run(2, 'PROXY v2 LOCAL before TLS', use_tls=True) diff --git a/tests/gold_tests/tls/proxy_protocol_client.py b/tests/gold_tests/tls/proxy_protocol_client.py index 5c7c8dc9883..36c04426fd1 100644 --- a/tests/gold_tests/tls/proxy_protocol_client.py +++ b/tests/gold_tests/tls/proxy_protocol_client.py @@ -37,6 +37,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("proxy_src_port", type=int, help="The source port in the PROXY message.") parser.add_argument("proxy_dest_port", type=int, help="The destination port in the PROXY message.") parser.add_argument("protocol_version", type=int, choices=[1, 2], help="the proxy protocol version(either 1 or 2).") + parser.add_argument("--addressless", action="store_true", help="Send a PROXY header without source or destination addresses.") parser.add_argument("--https", action="store_true", help="Send https data after the PROXY message.") return parser.parse_args() @@ -52,6 +53,11 @@ def construct_proxy_header_v1(src_addr: tuple, dst_addr: tuple) -> bytes: return f"PROXY TCP4 {src_addr[0]} {dst_addr[0]} {src_addr[1]} {dst_addr[1]}\r\n".encode() +def construct_proxy_header_v1_unknown() -> bytes: + """Construct a PROXY protocol v1 UNKNOWN header.""" + return b"PROXY UNKNOWN\r\n" + + def construct_proxy_header_v2(src_addr: tuple, dst_addr: tuple) -> bytes: """Construct a PROXY protocol v2 header. @@ -74,8 +80,19 @@ def construct_proxy_header_v2(src_addr: tuple, dst_addr: tuple) -> bytes: return header +def construct_proxy_header_v2_local() -> bytes: + """Construct a PROXY protocol v2 LOCAL header.""" + return VERSION_2_SIGNATURE + b'\x20\x00\x00\x00' + + def send_proxy_header( - socket: socket.socket, src_ip: str, src_port: str, dest_ip: int, dest_port: int, proxy_protocol_version: int) -> None: + socket: socket.socket, + src_ip: str, + src_port: int, + dest_ip: str, + dest_port: int, + proxy_protocol_version: int, + addressless: bool = False) -> None: """Send the specified PROXY protocol header. :param socket: The socket to send the header on. @@ -84,9 +101,14 @@ def send_proxy_header( :param dest_ip: The destination IP address. :param dest_port: The destination port. :param proxy_protocol_version: The PROXY protocol version. + :param addressless: Whether to send a valid header with no address data. """ logging.info(f'Sending PROXY protocol version {proxy_protocol_version}') - if proxy_protocol_version == 1: + if addressless and proxy_protocol_version == 1: + header = construct_proxy_header_v1_unknown() + elif addressless and proxy_protocol_version == 2: + header = construct_proxy_header_v2_local() + elif proxy_protocol_version == 1: header = construct_proxy_header_v1((src_ip, src_port), (dest_ip, dest_port)) elif proxy_protocol_version == 2: header = construct_proxy_header_v2((src_ip, src_port), (dest_ip, dest_port)) @@ -102,7 +124,7 @@ def send_and_receive_http(sock: socket.socket, host: str) -> None: :param sock: The socket to send and receive data on. :param host: The Host header value to send in the HTTP request. """ - request = f"GET /proxy_protocol HTTP/1.1\r\nHost: {host}\r\n\r\n" + request = f"GET /proxy_protocol HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n" logging.info("Sending:") logging.info(f'\n{request}') sock.sendall(request.encode()) @@ -123,7 +145,8 @@ def main() -> None: with socket.create_connection((args.server_address, args.server_port)) as sock: # send the PROXY header send_proxy_header( - sock, args.proxy_src_ip, args.proxy_src_port, args.proxy_dest_ip, args.proxy_dest_port, args.protocol_version) + sock, args.proxy_src_ip, args.proxy_src_port, args.proxy_dest_ip, args.proxy_dest_port, args.protocol_version, + args.addressless) if args.https: # https context = ssl.create_default_context() diff --git a/tests/gold_tests/tls/replay/outbound_sni_server_name_plain.replay.yaml b/tests/gold_tests/tls/replay/outbound_sni_server_name_plain.replay.yaml new file mode 100644 index 00000000000..b13506b0ed6 --- /dev/null +++ b/tests/gold_tests/tls/replay/outbound_sni_server_name_plain.replay.yaml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +autest: + description: "Plain inbound transaction with outbound SNI server_name policy" + dns: + name: "dns" + server: + name: "server" + client: + name: "client" + ats: + name: "ts" + process_config: + enable_cache: false + records_config: + proxy.config.diags.debug.enabled: 1 + proxy.config.diags.debug.tags: "http|ssl" + proxy.config.ssl.client.sni_policy: "server_name" + proxy.config.ssl.client.verify.server.policy: "DISABLED" + remap_config: + - from: "http://plain.example.com/" + to: "https://origin.example.com:{SERVER_HTTPS_PORT}/" + +sessions: +- protocol: + stack: http + transactions: + - client-request: + method: GET + url: /server-name-policy + version: "1.1" + headers: + fields: + - [ Host, plain.example.com ] + - [ uuid, outbound-sni-server-name-plain ] + - [ X-Test, outbound-sni-server-name-plain ] + - [ Content-Length, 0 ] + + proxy-request: + protocol: + stack: https + tls: + sni: origin.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + headers: + fields: + - [ X-Test, { value: outbound-sni-server-name-plain, as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 0 ] + - [ X-Response, outbound-sni-server-name-plain ] + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response, { value: outbound-sni-server-name-plain, as: equal } ] diff --git a/tests/gold_tests/tls/tls_accept_timeout_crash.replay.yaml b/tests/gold_tests/tls/tls_accept_timeout_crash.replay.yaml new file mode 100644 index 00000000000..54524fcdf28 --- /dev/null +++ b/tests/gold_tests/tls/tls_accept_timeout_crash.replay.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +sessions: +- protocol: + stack: https + transactions: + - client-request: + method: "GET" + version: "1.1" + url: / + headers: + fields: + - [ Host, example.com ] + - [ uuid, 1 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 0 ] + + proxy-response: + status: 200 + +- protocol: + stack: http2 + transactions: + - client-request: + headers: + fields: + - [ :method, GET ] + - [ :scheme, https ] + - [ :authority, example.com ] + - [ :path, / ] + - [ uuid, 2 ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ Content-Length, 0 ] + + proxy-response: + status: 200 diff --git a/tests/gold_tests/tls/tls_accept_timeout_crash.test.py b/tests/gold_tests/tls/tls_accept_timeout_crash.test.py new file mode 100644 index 00000000000..f98ac18215e --- /dev/null +++ b/tests/gold_tests/tls/tls_accept_timeout_crash.test.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' +Regression test: stale TLS accept-timeout events must not crash the session acceptors. + +With a pending TS_HTTP_SSN_START_HOOK whose callout defers, a stale TLS +handshake or accept-no-activity inactivity timer could fire after the read VIO +had been reassigned to a session acceptor but before the session took +ownership, triggering a release_assert in the acceptor's mainEvent and aborting +traffic_server. + +Reproduction: set the TLS handshake and accept-no-activity timeouts to 1 +second, load a plugin that delays the SSN_START hook by 3 seconds, then drive +TLS requests over HTTP/1.1 and HTTP/2. Before the fix, traffic_server aborts +during the delay window. After the fix, the client receives normal 200 +responses. +''' + +import os + +Test.Summary = 'TLS accept timeout must not crash HttpSessionAccept when SSN_START hook defers.' + +replay_file = "tls_accept_timeout_crash.replay.yaml" + +server = Test.MakeVerifierServerProcess("server", replay_file) + +ts = Test.MakeATSProcess("ts", enable_tls=True, enable_cache=False) +ts.addDefaultSSLFiles() + +ts.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.handshake_timeout_in': 1, + 'proxy.config.http.accept_no_activity_timeout': 1, + 'proxy.config.http2.accept_no_activity_timeout': 1, + 'proxy.config.url_remap.remap_required': 0, + }) + +ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + +ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{server.Variables.http_port}/') + +# Delay TS_HTTP_SSN_START_HOOK reenable by 3s — longer than both +# handshake_timeout_in and accept_no_activity_timeout (1s each), shorter than +# the verifier's 5s read timeout. Either timer firing during the hook window +# lands a VC_EVENT_INACTIVITY_TIMEOUT on the session acceptor's VIO cont. +Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'hook_add_plugin.so'), ts, '-delay-ms=3000') + +# Assert traffic_server never hits the release_assert landmine in the session +# acceptors when an inactivity timer fires during the SSN_START hook window. +ts.Disk.traffic_out.Content += Testers.ExcludesExpression( + r'failed assertion `event == NET_EVENT_ACCEPT', 'session acceptors must not abort on stale inactivity timeouts') +ts.Disk.diags_log.Content += Testers.ExcludesExpression( + r'FATAL.*Assertion', 'traffic_server must not abort on a stale accept-path inactivity timeout') + +tr = Test.AddTestRun("TLS request through delayed SSN_START hook") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.AddVerifierClientProcess("client", replay_file, https_ports=[ts.Variables.ssl_port]) +tr.StillRunningAfter = ts +tr.StillRunningAfter = server diff --git a/tests/gold_tests/tls/tls_engine_abort.py b/tests/gold_tests/tls/tls_engine_abort.py new file mode 100644 index 00000000000..04d3cf4b108 --- /dev/null +++ b/tests/gold_tests/tls/tls_engine_abort.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Abort TLS handshakes while the async_handshake plugin's job is mid-pause. + +The async_handshake plugin pauses each handshake for two seconds (-delay-ms=2000). +This client opens a TLS connection, lets ATS enter that async pause (so the handshake +eventfd is registered on the poller with the SSLNetVConnection as its target), +then closes the socket before the pause finishes. On that disconnect the +SSLNetVConnection must deregister the eventfd before it is freed, otherwise the +poller is left with a live registration pointing at a freed connection. Under +ASan a server that fails to deregister reports an error here; a correct server +stays clean. +""" + +import socket +import ssl +import sys +import time + +port = int(sys.argv[1]) +iterations = int(sys.argv[2]) if len(sys.argv) > 2 else 25 + +ctx = ssl._create_unverified_context() + +for _ in range(iterations): + try: + raw = socket.create_connection(("127.0.0.1", port), timeout=5) + except OSError: + continue + try: + tls = ctx.wrap_socket(raw, do_handshake_on_connect=False, server_hostname="example.com") + # Drive the handshake far enough that ATS enters the async pause, then + # bail out quickly so the close lands inside the async job's 2s window. + tls.settimeout(0.4) + try: + tls.do_handshake() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError, ssl.SSLError, socket.timeout, OSError): + pass + # Abort: close hard while the async job is still in flight. + try: + tls.close() + except OSError: + pass + except OSError: + try: + raw.close() + except OSError: + pass + time.sleep(0.05) + +print("sent {0} aborted handshakes".format(iterations)) diff --git a/tests/gold_tests/tls/tls_engine_abort.test.py b/tests/gold_tests/tls/tls_engine_abort.test.py new file mode 100644 index 00000000000..f2756e55ce3 --- /dev/null +++ b/tests/gold_tests/tls/tls_engine_abort.test.py @@ -0,0 +1,113 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +Test.Summary = ''' +Abort TLS handshakes while an OpenSSL async job is mid-pause, to exercise the +SSLNetVConnection teardown path. When a handshake returns SSL_ERROR_WANT_ASYNC +an eventfd is registered on the poller with the connection as its target, and a +connection torn down before the async job finishes must deregister that eventfd +so the poller is not left pointing at a freed connection. Built under ASan the +server must survive the abort barrage with no sanitizer error and still serve a +normal request. +''' + +async_handshake = os.path.join(Test.Variables.AtsTestPluginsDir, 'async_handshake.so') + +Test.SkipUnless( + Condition.HasOpenSSLVersion('1.1.1'), + Condition.IsOpenSSL(), + Condition(lambda: os.path.isfile(async_handshake), async_handshake + " not found."), +) + +ts = Test.MakeATSProcess("ts", enable_tls=True) +server = Test.MakeOriginServer("server") + +# A wide pause window (well beyond the abort client's 0.4s handshake attempt) +# so the abort reliably lands while the async job is still in flight. +Test.PrepareTestPlugin(async_handshake, ts, '-delay-ms=2000') + +server.addResponse( + "sessionlog.json", { + "headers": "GET / HTTP/1.1\r\nuuid: basic\r\n\r\n", + "timestamp": "1469733493.993", + "body": "" + }, { + "headers": + "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 2\r\n\r\n", + "timestamp": "1469733493.993", + "body": "ok" + }) + +ts.addSSLfile("ssl/server.pem") +ts.addSSLfile("ssl/server.key") + +ts.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port)) + +ts.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') + +ts.Disk.records_config.update( + { + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.exec_thread.autoconfig.scale': 1.0, + 'proxy.config.ssl.async.handshake.enabled': 1, + 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.tags': 'ssl' + }) + +# Fire a barrage of handshakes that abort while the async job is mid-pause. Correct +# teardown is validated by ATS surviving this with no crash and, under ASan, no +# sanitizer error. Without deregistration the connection is freed while its +# eventfd still has a live poller registration. +abort_client = os.path.join(Test.TestDirectory, 'tls_engine_abort.py') + +tr = Test.AddTestRun("abort-during-async-handshake") +tr.Processes.Default.Command = "{0} {1} {2} 30".format(sys.executable, abort_client, ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port)) +tr.Processes.Default.Streams.All = Testers.ContainsExpression("sent 30 aborted handshakes", "Abort client ran") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server + +# After the abort barrage, a normal request must still succeed: the server is +# healthy, not crashed or wedged. +tr2 = Test.AddTestRun("normal-request-after-aborts") +tr2.MakeCurlCommand("-k -v -H uuid:basic -H host:example.com https://127.0.0.1:{0}/".format(ts.Variables.ssl_port), ts=ts) +tr2.ReturnCode = 0 +tr2.Processes.Default.Streams.All = Testers.ContainsExpression(r"HTTP/(2|1\.1) 200", "Request succeeds after the abort barrage") +tr2.StillRunningAfter = ts +tr2.StillRunningAfter = server + +# The abort barrage must actually drive handshakes into the async pause, +# otherwise the eventfd is never registered and the test proves nothing. The +# async_handshake plugin's wake thread prints this to stderr (-> traffic.out) +# when it signals the eventfd at the end of its pause; that only happens if a +# handshake entered the WANT_ASYNC path and armed the eventfd, so its presence +# confirms the teardown path was exercised (and that the async job completed +# after the abort -- exactly the use-after-free window this fix closes). +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + "sent async wake signal to", "Async job engaged on at least one handshake") + +# The server process must not have reported an AddressSanitizer error. ASan +# writes to stderr, which the harness binds to traffic.out -- not diags.log -- +# so the exclusion has to be checked against traffic.out to catch the UAF. +ts.Disk.traffic_out.Content += Testers.ExcludesExpression("AddressSanitizer", "No ASan error in the server") diff --git a/tests/gold_tests/tls/tls_forward_nonhttp.test.py b/tests/gold_tests/tls/tls_forward_nonhttp.test.py index e2ba698ede0..fd7bfad8002 100644 --- a/tests/gold_tests/tls/tls_forward_nonhttp.test.py +++ b/tests/gold_tests/tls/tls_forward_nonhttp.test.py @@ -52,6 +52,7 @@ 'proxy.config.dns.nameservers': f"127.0.0.1:{nameserver.Variables.Port}", 'proxy.config.dns.resolv_conf': 'NULL' }) +ts.addPrivateConnectAllowYaml() # foo.com should not terminate. Just tunnel to server_foo # bar.com should terminate. Forward its tcp stream to server_bar diff --git a/tests/gold_tests/tls/tls_outbound_sni_server_name_plain.test.py b/tests/gold_tests/tls/tls_outbound_sni_server_name_plain.test.py new file mode 100644 index 00000000000..989255ba773 --- /dev/null +++ b/tests/gold_tests/tls/tls_outbound_sni_server_name_plain.test.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' +Verify outbound SNI server_name policy on plain inbound traffic. +''' + +Test.Summary = ''' +Verify that plain inbound transactions do not crash when outbound SNI is +configured to use the inbound TLS server name. +''' + +Test.ATSReplayTest(replay_file='replay/outbound_sni_server_name_plain.replay.yaml') diff --git a/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py b/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py index eab7b9fca10..93b2adfa41f 100644 --- a/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py +++ b/tests/gold_tests/tls/tls_partial_blind_tunnel.test.py @@ -51,6 +51,7 @@ 'proxy.config.dns.nameservers': f"127.0.0.1:{nameserver.Variables.Port}", 'proxy.config.dns.resolv_conf': 'NULL' }) +ts.addPrivateConnectAllowYaml() # foo.com should terminate. and reconnect via TLS upstream to bar.com ts.Disk.sni_yaml.AddLines( diff --git a/tests/gold_tests/tls/tls_sni_host_policy.test.py b/tests/gold_tests/tls/tls_sni_host_policy.test.py index 6a2e7477e61..aba882b05f2 100644 --- a/tests/gold_tests/tls/tls_sni_host_policy.test.py +++ b/tests/gold_tests/tls/tls_sni_host_policy.test.py @@ -68,6 +68,14 @@ ' host_sni_policy: PERMISSIVE', '- fqdn: bOb', ' verify_client: STRICT', + '- fqdn: bob.bar.com', + ' verify_client: STRICT', + '- fqdn: dave.bob', + ' verify_client: STRICT', + '- fqdn: noipallow.example.com', + ' http2: off', + '- fqdn: ipallow_nomatch.example.com', + ' ip_allow: 192.168.1.1', ]) # case 1 @@ -181,6 +189,86 @@ tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Access Denied", "Check response") +# case 10 +# sni=bob.bar.com and host=bob. Do provide client cert. SNI is longer than host but shares the +# same prefix. Should fail due to sni-host mismatch. +tr = Test.AddTestRun("Connect with SNI longer than host sharing prefix") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.MakeCurlCommand( + "-v --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key -H 'host:bob' --resolve 'bob.bar.com:{0}:127.0.0.1' https://bob.bar.com:{0}/case1" + .format(ts.Variables.ssl_port), + ts=ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("Access Denied", "Check response") + +# case 11 +# sni=bob and host=bob.bar.com. Do provide client cert. Host is longer than SNI but shares the +# same prefix. Should fail due to sni-host mismatch. +tr = Test.AddTestRun("Connect with host longer than SNI sharing prefix") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.MakeCurlCommand( + "-v --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key -H 'host:bob.bar.com' --resolve 'bob:{0}:127.0.0.1' https://bob:{0}/case1" + .format(ts.Variables.ssl_port), + ts=ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("Access Denied", "Check response") + +# case 12 +# sni=bob and host=dave.bob. Do provide client cert. Host ends with the SNI value but is a +# different hostname. Should fail due to sni-host mismatch. +tr = Test.AddTestRun("Connect with host ending with SNI value") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.MakeCurlCommand( + "-v --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key -H 'host:dave.bob' --resolve 'bob:{0}:127.0.0.1' https://bob:{0}/case1" + .format(ts.Variables.ssl_port), + ts=ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("Access Denied", "Check response") + +# case 13 +# sni=dave.bob and host=bob. Do provide client cert. SNI ends with the host value but is a +# different hostname. Should fail due to sni-host mismatch. +tr = Test.AddTestRun("Connect with SNI ending with host value") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.MakeCurlCommand( + "-v --tls-max 1.2 -k --cert ./signed-foo.pem --key ./signed-foo.key -H 'host:bob' --resolve 'dave.bob:{0}:127.0.0.1' https://dave.bob:{0}/case1" + .format(ts.Variables.ssl_port), + ts=ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("Access Denied", "Check response") + +# case 14 +# sni=other.example.com and host=ipallow_nomatch.example.com. Host header matches SNI entry but +# client IP is NOT in ip_allow list. TestClientSNIAction should still return true (because ip_addrs is +# non-empty), which means host_sni_policy IS enforced and the mismatch triggers "Access Denied". +tr = Test.AddTestRun("Connect with ip_allow SNI entry not matching client IP should still enforce host_sni_policy") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.MakeCurlCommand( + "-v --tls-max 1.2 -k -H 'host:ipallow_nomatch.example.com' --resolve 'other.example.com:{0}:127.0.0.1' https://other.example.com:{0}/case1" + .format(ts.Variables.ssl_port), + ts=ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ContainsExpression("Access Denied", "Should get 403 due to host_sni_policy enforcement") + +# case 15 +# sni=other.example.com and host=noipallow.example.com. Host header matches SNI +# entry that only has http2: off configured (no verify_client, no ip_allow). This should +# NOT trigger host_sni_policy enforcement because the only action is a no-op. +tr = Test.AddTestRun("Connect with SNI entry having no ip_allow should not enforce host_sni_policy") +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.MakeCurlCommand( + "-v --tls-max 1.2 -k -H 'host:noipallow.example.com' --resolve 'other.example.com:{0}:127.0.0.1' https://other.example.com:{0}/case1" + .format(ts.Variables.ssl_port), + ts=ts) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.All = Testers.ExcludesExpression("Access Denied", "No 403 for non-ip_allow SNI entry") + # Wait for the error.log entry to be written. test_run = Test.AddAwaitFileContainsTestRun( 'Await SNI mismatch error log entry.', @@ -192,8 +280,19 @@ "WARNING: SNI/hostname mismatch sni=dave host=bob action=terminate", "Should have warning on mismatch") ts.Disk.diags_log.Content += Testers.ContainsExpression( "WARNING: SNI/hostname mismatch sni=ellen host=Boblite action=continue", "Should have warning on mismatch") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: SNI/hostname mismatch sni=bob.bar.com host=bob action=terminate", "Should have warning on prefix mismatch") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: SNI/hostname mismatch sni=bob host=bob.bar.com action=terminate", "Should have warning on prefix mismatch") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: SNI/hostname mismatch sni=bob host=dave.bob action=terminate", "Should have warning on suffix mismatch") +ts.Disk.diags_log.Content += Testers.ContainsExpression( + "WARNING: SNI/hostname mismatch sni=dave.bob host=bob action=terminate", "Should have warning on suffix mismatch") ts.Disk.diags_log.Content += Testers.ExcludesExpression( "WARNING: SNI/hostname mismatch sni=ellen host=fran", "Should not have warning on mismatch with non-policy host") +ts.Disk.diags_log.Content += Testers.ExcludesExpression( + "WARNING: SNI/hostname mismatch sni=other.example.com host=noipallow.example.com", + "Should not have warning for SNI entry with no ip_allow") test_run.Processes.Default.ReturnCode = 0 ts.Disk.error_log.Content += Testers.ContainsExpression( diff --git a/tests/gold_tests/tls/tls_sni_ip_allow.test.py b/tests/gold_tests/tls/tls_sni_ip_allow.test.py index 84cb9f6b9f3..319853925d6 100644 --- a/tests/gold_tests/tls/tls_sni_ip_allow.test.py +++ b/tests/gold_tests/tls/tls_sni_ip_allow.test.py @@ -138,6 +138,7 @@ def _configure_trafficserver(self, tr: 'TestRun', connect_type: int, dns: 'Proce ts.Disk.records_config.update({ 'proxy.config.http.connect_ports': f"{server.Variables.https_port}", }) + ts.addPrivateConnectAllowYaml() return ts def _configure_client( diff --git a/tests/gold_tests/tls/tls_sni_with_port.test.py b/tests/gold_tests/tls/tls_sni_with_port.test.py index 3f107d6765a..798b191cb6c 100644 --- a/tests/gold_tests/tls/tls_sni_with_port.test.py +++ b/tests/gold_tests/tls/tls_sni_with_port.test.py @@ -120,6 +120,7 @@ def _configure_traffic_server(self, tr: "TestRun", server_one: "Process", server }) ts.Disk.remap_config.AddLine(f"map / http://127.0.0.1:{server_three.Variables.http_port}") + ts.addPrivateConnectAllowYaml(methods='[ CONNECT, GET ]') ts.Disk.sni_yaml.AddLines( [ diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index e2de6524af6..251defe4c39 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -93,6 +93,7 @@ 'proxy.config.dns.nameservers': f'127.0.0.1:{dns.Variables.Port}', 'proxy.config.dns.resolv_conf': 'NULL' }) +ts.addPrivateConnectAllowYaml() # foo.com should not terminate. Just tunnel to server_foo # bar.com should terminate. Forward its tcp stream to server_bar @@ -286,6 +287,25 @@ "Verify the tunnel destination is expanded correctly.") tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Verify a successful response is received") +# Regression: a tunnel_route that combines $N match groups with a port variable +# must still enforce connect_ports. An earlier bug let MATCH_GROUPS clear the +# dynamic-port flag set by MAP_WITH_PROXY_PROTOCOL_PORT, bypassing the check. +tr = Test.AddTestRun("test wildcard with proxy_protocol_port - not in connect_ports") +tr.TimeOut = 5 +tr.Setup.Copy('proxy_protocol_client.py') +wildcard_rejected_port = server_forbidden.Variables.SSL_Port +tr.Processes.Default.Command = ( + f'{sys.executable} proxy_protocol_client.py ' + f'127.0.0.1 {ts.Variables.proxy_protocol_ssl_port} wildcard.with.proxy.protocol.port.com ' + f'127.0.0.1 127.0.0.1 60123 {wildcard_rejected_port} ' + f'2 --https') +tr.ReturnCode = 1 +tr.StillRunningAfter = ts +tr.Processes.Default.Streams.All += Testers.ContainsExpression("ssl.SSL.*Error:.*EOF", "Verify the handshake failed") +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + f"Rejected a tunnel to port {wildcard_rejected_port} not in connect_ports", + "Verify the tunnel was rejected even though the route uses a $N match group") + # Update sni file and reload tr = Test.AddTestRun("Update config files") # Update the SNI config diff --git a/tests/gold_tests/tls/tls_tunnel_forward.test.py b/tests/gold_tests/tls/tls_tunnel_forward.test.py index 472797d10ae..9ac0f306031 100644 --- a/tests/gold_tests/tls/tls_tunnel_forward.test.py +++ b/tests/gold_tests/tls/tls_tunnel_forward.test.py @@ -72,6 +72,7 @@ 'proxy.config.dns.nameservers': f"127.0.0.1:{nameserver.Variables.Port}", 'proxy.config.dns.resolv_conf': 'NULL' }) +ts.addPrivateConnectAllowYaml() # foo.com should not terminate. Just tunnel to server_foo # bar.com should terminate. Forward its tcp stream to server_bar diff --git a/tests/gold_tests/tunnel/tunnel_transform.test.py b/tests/gold_tests/tunnel/tunnel_transform.test.py index dabb0692b44..2749186a69f 100644 --- a/tests/gold_tests/tunnel/tunnel_transform.test.py +++ b/tests/gold_tests/tunnel/tunnel_transform.test.py @@ -27,6 +27,7 @@ ''' Test.SkipIf(Condition.CurlUsingUnixDomainSocket()) + # Define default ATS. Disable the cache to simplify the test. ts = Test.MakeATSProcess("ts", enable_cache=False, enable_tls=True) ts.addSSLfile("../tls/ssl/server.pem") @@ -65,6 +66,7 @@ '- fqdn: tunnel-test', " tunnel_route: localhost:{0}".format(server.Variables.SSL_Port), ]) +ts.addPrivateConnectAllowYaml() # Set up simple forwarding proxy to keep track of TLS bytes for both # directions diff --git a/tests/gold_tests/tunnel/txn_type.test.py b/tests/gold_tests/tunnel/txn_type.test.py index 8c79e0a6798..437d3f2d40d 100644 --- a/tests/gold_tests/tunnel/txn_type.test.py +++ b/tests/gold_tests/tunnel/txn_type.test.py @@ -24,6 +24,7 @@ ''' Test.SkipIf(Condition.CurlUsingUnixDomainSocket()) + # Define default ATS. Disable the cache to simplify the test. ts = Test.MakeATSProcess("ts", enable_cache=False, enable_tls=True) ts.addSSLfile("../tls/ssl/server.pem") @@ -73,6 +74,7 @@ '- fqdn: tunnel-test', " tunnel_route: localhost:{0}".format(server.Variables.SSL_Port), ]) +ts.addPrivateConnectAllowYaml(methods='[ CONNECT, GET ]') # Add connection close to ensure that the client connection closes promptly after completing the transaction cmd_http = '-k --http1.1 -H "Connection: close" -vs --resolve "http-test:{0}:127.0.0.1" https://http-test:{0}/'.format( diff --git a/tests/gold_tests/uds/uds_socket_perm.test.py b/tests/gold_tests/uds/uds_socket_perm.test.py new file mode 100644 index 00000000000..0aebcf15907 --- /dev/null +++ b/tests/gold_tests/uds/uds_socket_perm.test.py @@ -0,0 +1,57 @@ +'''Verify the listening UDS socket mode honors uds-perm and defaults to 0666.''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Verify proxy.config.http.server_ports honors the uds-perm option on UDS +listeners and defaults to 0666. +''' + + +def assert_socket_mode(socket_path: str, expected_octal: str) -> str: + return ( + f'python3 -c "import os, stat, sys; ' + f'sys.exit(0 if stat.S_IMODE(os.stat(sys.argv[1]).st_mode) == int(sys.argv[2], 8) else 1)" ' + f'{socket_path} {expected_octal}') + + +# +# Default UDS permission should be 0666 (no uds-perm option specified). +# +ts_default = Test.MakeATSProcess("ts_default") +ts_default.Disk.records_config.update( + { + 'proxy.config.http.server_ports': f"{ts_default.Variables.port} {ts_default.Variables.uds_path}", + }) + +tr = Test.AddTestRun("UDS default permission is 0666") +tr.Processes.Default.Command = assert_socket_mode(ts_default.Variables.uds_path, '0666') +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts_default) + +# +# uds-perm=0660 should be honored. +# +ts_custom = Test.MakeATSProcess("ts_custom") +ts_custom.Disk.records_config.update( + { + 'proxy.config.http.server_ports': f"{ts_custom.Variables.port} {ts_custom.Variables.uds_path}:uds-perm=0660", + }) + +tr = Test.AddTestRun("UDS custom permission 0660 honored") +tr.Processes.Default.Command = assert_socket_mode(ts_custom.Variables.uds_path, '0660') +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.StartBefore(ts_custom) diff --git a/tests/tools/plugins/CMakeLists.txt b/tests/tools/plugins/CMakeLists.txt index a058eea6c63..a909cf59d1b 100644 --- a/tests/tools/plugins/CMakeLists.txt +++ b/tests/tools/plugins/CMakeLists.txt @@ -20,6 +20,7 @@ add_autest_plugin(conf_remap_stripped conf_remap_stripped.cc) add_autest_plugin(continuations_verify continuations_verify.cc) add_autest_plugin(cont_schedule cont_schedule.cc) add_autest_plugin(custom204plugin custom204plugin.cc) +add_autest_plugin(delay_txn_start delay_txn_start.cc) add_autest_plugin(emergency_shutdown emergency_shutdown.cc) add_autest_plugin(fatal_shutdown fatal_shutdown.cc) add_autest_plugin(hook_add_plugin hook_add_plugin.cc) @@ -39,6 +40,7 @@ add_autest_plugin(user_args user_args.cc) add_autest_plugin(hook_tunnel_plugin hook_tunnel_plugin.cc) add_autest_plugin(tunnel_transform tunnel_transform.cc) add_autest_plugin(http2_close_connection http2_close_connection.cc) +add_autest_plugin(redirect_rearm redirect_rearm.cc) target_link_libraries(continuations_verify PRIVATE OpenSSL::SSL) target_link_libraries(ssl_client_verify_test PRIVATE OpenSSL::SSL) diff --git a/tests/tools/plugins/async_handshake.cc b/tests/tools/plugins/async_handshake.cc index 3747ce17eb6..23c8998a20c 100644 --- a/tests/tools/plugins/async_handshake.cc +++ b/tests/tools/plugins/async_handshake.cc @@ -29,6 +29,8 @@ #include #include #include +#include +#include #include #include @@ -39,6 +41,7 @@ namespace { DbgCtl dbg_ctl{PLUGIN_NAME}; char async_hook_key; +int delay_ms = 100; void wait_cleanup(ASYNC_WAIT_CTX * /* ctx ATS_UNUSED */, const void * /* key ATS_UNUSED */, OSSL_ASYNC_FD read_fd, void *write_fd_ptr) @@ -56,7 +59,7 @@ wake_async_job(void *arg) auto signal_fd = static_cast(reinterpret_cast(arg)); char buf = 'X'; - usleep(100 * 1000); + usleep(delay_ms * 1000); if (write(signal_fd, &buf, sizeof(buf)) < 0) { fprintf(stderr, PCP "failed to send async wake signal to %d, errno=%d\n", signal_fd, errno); } else { @@ -158,7 +161,7 @@ handle_ssl_cert(TSCont /* cont ATS_UNUSED */, TSEvent event, void *edata) } // namespace void -TSPluginInit(int /* argc ATS_UNUSED */, const char ** /* argv ATS_UNUSED */) +TSPluginInit(int argc, const char **argv) { TSPluginRegistrationInfo info; @@ -171,6 +174,19 @@ TSPluginInit(int /* argc ATS_UNUSED */, const char ** /* argv ATS_UNUSED */) return; } + for (int i = 1; i < argc; ++i) { + if (strncmp(argv[i], "-delay-ms=", 10) == 0) { + const char *delay_arg = argv[i] + 10; + char *end = nullptr; + long parsed = strtol(delay_arg, &end, 10); + if (end == delay_arg || *end != '\0' || parsed < 0) { + TSError(PCP "invalid -delay-ms value '%s', keeping default %d", delay_arg, delay_ms); + } else { + delay_ms = static_cast(parsed); + } + } + } + TSCont cb_cert = TSContCreate(handle_ssl_cert, TSMutexCreate()); if (cb_cert == nullptr) { TSError(PCP "failed to create SSL cert hook"); diff --git a/tests/tools/plugins/delay_txn_start.cc b/tests/tools/plugins/delay_txn_start.cc new file mode 100644 index 00000000000..a4bb25a3447 --- /dev/null +++ b/tests/tools/plugins/delay_txn_start.cc @@ -0,0 +1,122 @@ +/** @file + + Delay transaction start hook completion for HTTP/2 read gating tests. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include + +namespace +{ +char const PLUGIN_NAME[] = "delay_txn_start"; + +DbgCtl dbg_ctl{PLUGIN_NAME}; + +int delay_ms = 500; + +struct TxnState { + TSHttpTxn txn = nullptr; + TSAction action = nullptr; + bool reenabled = false; +}; + +int +txn_handler(TSCont contp, TSEvent event, void *edata) +{ + auto *state = static_cast(TSContDataGet(contp)); + + switch (event) { + case TS_EVENT_TIMEOUT: + state->action = nullptr; + state->reenabled = true; + Dbg(dbg_ctl, "delayed TXN_START reenable"); + TSHttpTxnReenable(state->txn, TS_EVENT_HTTP_CONTINUE); + break; + case TS_EVENT_HTTP_READ_REQUEST_HDR: { + if (!state->reenabled) { + TSError("[%s] READ_REQUEST_HDR before delayed TXN_START reenable", PLUGIN_NAME); + TSReleaseAssert(state->reenabled); + } + Dbg(dbg_ctl, "READ_REQUEST_HDR after delayed TXN_START reenable"); + auto txnp = static_cast(edata); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + break; + } + case TS_EVENT_HTTP_TXN_CLOSE: { + auto txnp = static_cast(edata); + + if (state->action != nullptr && !TSActionDone(state->action)) { + TSActionCancel(state->action); + } + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + TSContDataSet(contp, nullptr); + delete state; + TSContDestroy(contp); + break; + } + default: + TSError("[%s] unexpected event: %d", PLUGIN_NAME, event); + break; + } + + return 0; +} + +int +global_handler(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edata) +{ + if (event == TS_EVENT_HTTP_TXN_START) { + auto txnp = static_cast(edata); + auto contp = TSContCreate(txn_handler, TSMutexCreate()); + auto state = new TxnState; + + state->txn = txnp; + TSContDataSet(contp, state); + TSHttpTxnHookAdd(txnp, TS_HTTP_READ_REQUEST_HDR_HOOK, contp); + TSHttpTxnHookAdd(txnp, TS_HTTP_TXN_CLOSE_HOOK, contp); + state->action = TSContScheduleOnPool(contp, delay_ms, TS_THREAD_POOL_TASK); + } + + return 0; +} +} // namespace + +void +TSPluginInit(int argc, const char **argv) +{ + TSPluginRegistrationInfo info; + + info.plugin_name = const_cast(PLUGIN_NAME); + info.vendor_name = const_cast("Apache Software Foundation"); + info.support_email = const_cast("dev@trafficserver.apache.org"); + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("[%s] plugin registration failed", PLUGIN_NAME); + return; + } + + if (argc > 1) { + delay_ms = std::atoi(argv[1]); + } + + TSHttpHookAdd(TS_HTTP_TXN_START_HOOK, TSContCreate(global_handler, TSMutexCreate())); +} diff --git a/tests/tools/plugins/hook_add_plugin.cc b/tests/tools/plugins/hook_add_plugin.cc index e1eef902746..1dc2d8dda35 100644 --- a/tests/tools/plugins/hook_add_plugin.cc +++ b/tests/tools/plugins/hook_add_plugin.cc @@ -21,7 +21,8 @@ limitations under the License. */ #include -#include +#include +#include #define PLUGIN_TAG "test" @@ -30,8 +31,9 @@ namespace DbgCtl dbg_ctl{PLUGIN_TAG}; } -// Number of seconds to reschedule to a task thread and delay +// Whether to reschedule the SSN_START reenable to a task thread, and by how many ms. int DelayStart = 0; +int DelayMs = 500; int transactionHandler(TSCont continuation, TSEvent event, void *d) @@ -120,7 +122,7 @@ globalHandler(TSCont /* continuation ATS_UNUSED */, TSEvent event, void *data) TSHttpSsnReenable(session, TS_EVENT_HTTP_CONTINUE); } else { TSContDataSet(cont, session); - TSContScheduleOnPool(cont, 500, TS_THREAD_POOL_TASK); + TSContScheduleOnPool(cont, DelayMs, TS_THREAD_POOL_TASK); } } @@ -141,10 +143,20 @@ TSPluginInit(int argc, const char **argv) return; } - if (argc >= 2) { - Dbg(dbg_ctl, "Argument %s", argv[1]); - if (strcmp(argv[1], "-delay") == 0) { + for (int i = 1; i < argc; ++i) { + Dbg(dbg_ctl, "Argument %s", argv[i]); + if (strcmp(argv[i], "-delay") == 0) { DelayStart = 1; + } else if (strncmp(argv[i], "-delay-ms=", 10) == 0) { + DelayStart = 1; + const char *delay_arg = argv[i] + 10; + char *end = nullptr; + long delay_ms = strtol(delay_arg, &end, 10); + if (end == delay_arg || *end != '\0' || delay_ms < 0) { + TSError("[" PLUGIN_TAG "] invalid -delay-ms value '%s', keeping default %d", delay_arg, DelayMs); + } else { + DelayMs = static_cast(delay_ms); + } } } TSCont continuation = TSContCreate(globalHandler, TSMutexCreate()); diff --git a/tests/tools/plugins/redirect_rearm.cc b/tests/tools/plugins/redirect_rearm.cc new file mode 100644 index 00000000000..b7e8de41cfd --- /dev/null +++ b/tests/tools/plugins/redirect_rearm.cc @@ -0,0 +1,112 @@ +/** @file + + Test plugin that re-sets the redirect URL on every response hop. + + On TS_HTTP_READ_RESPONSE_HDR_HOOK, for every response with a Location + header (i.e. every redirect hop), call TSHttpTxnRedirectUrlSet with + that Location value. This exercises a plugin that calls + TSHttpTxnRedirectUrlSet on every response hook, and verifies that such + a plugin cannot follow more redirects than + proxy.config.http.number_of_redirections allows. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include + +#define PLUGIN_TAG "redirect_rearm" + +namespace +{ +DbgCtl dbg{PLUGIN_TAG}; + +int +handle_read_response_hdr(TSCont /* contp */, TSEvent event, void *edata) +{ + if (event != TS_EVENT_HTTP_READ_RESPONSE_HDR) { + TSError("[" PLUGIN_TAG "] unexpected event %d", event); + TSHttpTxnReenable(static_cast(edata), TS_EVENT_HTTP_CONTINUE); + return 0; + } + + TSHttpTxn txnp = static_cast(edata); + TSMBuffer bufp; + TSMLoc hdr; + + if (TSHttpTxnServerRespGet(txnp, &bufp, &hdr) != TS_SUCCESS) { + Dbg(dbg, "no server response header, skipping hop"); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + TSHttpStatus status = TSHttpHdrStatusGet(bufp, hdr); + if (status < 300 || status >= 400) { + Dbg(dbg, "status %d is not a redirect, skipping hop", status); + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + TSMLoc field = TSMimeHdrFieldFind(bufp, hdr, "Location", 8); + if (field == TS_NULL_MLOC) { + Dbg(dbg, "no Location header, skipping hop"); + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + int vlen = 0; + const char *vstr = TSMimeHdrFieldValueStringGet(bufp, hdr, field, 0, &vlen); + if (vstr != nullptr && vlen > 0) { + char *url = static_cast(TSmalloc(vlen + 1)); + memcpy(url, vstr, vlen); + url[vlen] = '\0'; + Dbg(dbg, "setting redirect URL from Location: %.*s", vlen, vstr); + // TSHttpTxnRedirectUrlSet takes ownership of the buffer, which the core + // later frees with ats_free, so it must come from the ATS allocator + // (TSmalloc / ats_malloc), not plain malloc or a stack buffer. + TSHttpTxnRedirectUrlSet(txnp, url, vlen); + } else { + Dbg(dbg, "empty Location value, skipping hop"); + } + + TSHandleMLocRelease(bufp, hdr, field); + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr); + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; +} +} // namespace + +void +TSPluginInit(int /* argc */, const char * /* argv */[]) +{ + TSPluginRegistrationInfo info; + info.plugin_name = const_cast(PLUGIN_TAG); + info.vendor_name = const_cast("Apache"); + info.support_email = const_cast("dev@trafficserver.apache.org"); + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("[" PLUGIN_TAG "] registration failed"); + return; + } + + TSCont c = TSContCreate(handle_read_response_hdr, nullptr); + TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, c); +}