diff --git a/pcapkit/protocols/application/ftp.py b/pcapkit/protocols/application/ftp.py index 09319f9e33..7c33af52a0 100644 --- a/pcapkit/protocols/application/ftp.py +++ b/pcapkit/protocols/application/ftp.py @@ -31,7 +31,7 @@ from pcapkit.utilities.exceptions import ProtocolError, UnsupportedCall if TYPE_CHECKING: - from typing import Any, NoReturn, Optional + from typing import Any, Iterable, NoReturn, Optional from typing_extensions import Literal @@ -110,11 +110,24 @@ def read(self, length: 'Optional[int]' = None, **kwargs: 'Any') -> 'Data_FTP': schema = self.__header__ data = schema.data - if (match := FTP_REQUEST.match(data)) is not None: - cmmd = match.group('cmmd').decode() + # NOTE: A multi-line reply (:rfc:`959#section-4`) may arrive whole in one + # segment. Its first line is parsed as any reply line, and the lines + # after it are kept as received in ``lines``. A segment holding anything + # else -- no closing line, or more after it -- is refused as before. + line, lines = data, () # type: bytes, tuple[bytes, ...] + if FTP_RESPONSE.match(data) is None and data.endswith(b'\r\n'): + first, _, rest = data.partition(b'\r\n') + if rest and (match := FTP_RESPONSE.match(first + b'\r\n')) is not None \ + and match.group('more'): + rest_lines = tuple(rest[:-2].split(b'\r\n')) + if self._test_lines(match.group('code'), rest_lines): + line, lines = first + b'\r\n', rest_lines + + if (match := FTP_REQUEST.match(line)) is not None: + raw_cmmd = match.group('cmmd') args = match.group('args') - cmmd_val = Enum_Command.get(cmmd) + cmmd_val = Enum_Command.get(raw_cmmd.decode()) args_val, charset = self._decode_args(args) ftp = Data_Request( @@ -122,8 +135,9 @@ def read(self, length: 'Optional[int]' = None, **kwargs: 'Any') -> 'Data_FTP': cmmd=cmmd_val, args=args_val, charset=charset, + raw_cmmd=raw_cmmd, ) # type: Data_FTP - elif (match := FTP_RESPONSE.match(data)) is not None: + elif (match := FTP_RESPONSE.match(line)) is not None: code = int(match.group('code')) more = bool(match.group('more')) args = match.group('text') if more else match.group('args') @@ -137,6 +151,7 @@ def read(self, length: 'Optional[int]' = None, **kwargs: 'Any') -> 'Data_FTP': more=more, args=args_val, charset=charset, + lines=lines, ) else: raise ProtocolError('FTP: invalid packet format') @@ -148,6 +163,8 @@ def make(self, args: 'Optional[str | bytes]' = None, more: 'bool' = False, charset: 'Optional[str]' = None, + raw_cmmd: 'Optional[bytes]' = None, + lines: 'Iterable[bytes]' = (), **kwargs: 'Any') -> 'Schema_FTP': """Make (construct) packet data. @@ -156,17 +173,30 @@ def make(self, the command or code, except after the hyphen of a multi-line reply, which the text follows immediately (:rfc:`959#section-4`). + Commands are case-insensitive (:rfc:`959#section-5`), so ``raw_cmmd`` + is written in place of ``cmmd`` whenever the two differ only in case, + which keeps the case the command was read in. + Args: cmmd: FTP command. code: FTP status code. args: Optional FTP command arguments and/or status messages. more: More status messages to follow for response packets. charset: Encoding for :obj:`str` arguments, **UTF-8** if not given. + raw_cmmd: FTP command as received, in its original case. + lines: Lines of a multi-line reply after the first, each written + verbatim and terminated with CRLF; the last, and only the last, + is the closing line, `` SP ``. **kwargs: Arbitrary keyword arguments. Returns: Constructed packet data. + Raises: + ProtocolError: If neither or both of ``cmmd`` and ``code`` are + given, or if ``lines`` are given other than as the rest of a + multi-line reply. + """ if cmmd is not None and code is None: if isinstance(cmmd, bytes): @@ -175,6 +205,8 @@ def make(self, prefix = cmmd.encode() else: prefix = cmmd.value + if raw_cmmd is not None and raw_cmmd.upper() == prefix.upper(): + prefix = raw_cmmd mf = b'' elif cmmd is None and code is not None: @@ -185,6 +217,10 @@ def make(self, else: raise ProtocolError('FTP: invalid packet type') + lines = tuple(lines) + if lines and not (mf and self._test_lines(prefix, lines)): + raise ProtocolError('FTP: lines must close a multi-line reply') + if args is None: suffix = b'' elif isinstance(args, bytes): @@ -195,7 +231,7 @@ def make(self, # the separating SP, absent with no arguments and after a hyphen sep = b'' if args is None or mf else b' ' return Schema_FTP( - data=b'%s%s%s%s\r\n' % (prefix, mf, sep, suffix), + data=b'%s%s%s%s\r\n' % (prefix, mf, sep, suffix) + b''.join(b'%s\r\n' % line for line in lines), ) ########################################################################## @@ -219,8 +255,32 @@ def _make_data(cls, data: 'Data_FTP') -> 'dict[str, Any]': # type: ignore[overr 'args': getattr(data, 'args', None), 'more': getattr(data, 'more', False), 'charset': getattr(data, 'charset', None), + 'raw_cmmd': getattr(data, 'raw_cmmd', None), + 'lines': getattr(data, 'lines', ()), } + @staticmethod + def _test_lines(code: 'bytes', lines: 'tuple[bytes, ...]') -> 'bool': + """Whether ``lines`` are the rest of a multi-line reply with ``code``. + + A multi-line reply ends at the first line that starts with its code + followed by SP (:rfc:`959#section-4`). The lines before that one are + free text, which may begin with digits -- the code with a hyphen + included -- but, as in any reply line, carry no LF. + + Args: + code: Reply code of the first line. + lines: Lines after the first, CRLF excluded. + + Returns: + Whether the last line, and only the last, is the closing line. + + """ + closing = code + b' ' + if not lines or not all(isinstance(line, bytes) and b'\n' not in line for line in lines): + return False + return lines[-1].startswith(closing) and not any(line.startswith(closing) for line in lines[:-1]) + def _decode_args(self, args: 'Optional[bytes]') -> 'tuple[Optional[str], Optional[str]]': """Decode arguments, recording how to encode them back. diff --git a/pcapkit/protocols/application/httpv1.py b/pcapkit/protocols/application/httpv1.py index e85b5cf120..2fb0e04f8c 100644 --- a/pcapkit/protocols/application/httpv1.py +++ b/pcapkit/protocols/application/httpv1.py @@ -46,7 +46,7 @@ if TYPE_CHECKING: from enum import IntEnum as StdlibEnum - from typing import Any, Optional + from typing import Any, Optional, Sequence from typing import Type as _Type from aenum import IntEnum as AenumEnum @@ -242,6 +242,7 @@ def read(self, length: 'Optional[int]' = None, **kwargs: 'Any') -> 'Data_HTTP': receipt=header_line, header=header_unpacked, body=body_unpacked, + raw_header=self._split_field_lines(header.partition(b'\r\n')[2]), ) self._receipt = header_line.type self._version = header_line.version # type: ignore[attr-defined] @@ -261,9 +262,21 @@ def make(self, # type: ignore[override] headers: 'Optional[OrderedMultiDict[str, str]]' = None, body: 'bytes' = b'', charset: 'Optional[str]' = None, + raw_line: 'Optional[bytes]' = None, + raw_header: 'Sequence[bytes]' = (), **kwargs: 'Any') -> 'Schema_HTTP': """Make (construct) packet data. + The start line and the field lines are built from the parsed values + with single SPs, ``name: value`` and UTF-8 field values. ``raw_line`` + is written in place of the start line, and each ``raw_header`` entry in + place of the header field at the same position, whenever reading it + back gives the same values -- so a message rebuilds byte for byte, + whitespace, ``obs-fold`` and field-value charset included, while a + value that has been changed is written from the change. Field lines + are matched to ``headers`` by content and in order, so adding or + removing a field leaves the others as received. + Args: http_version: HTTP version. method: HTTP method. @@ -277,11 +290,122 @@ def make(self, # type: ignore[override] body: HTTP body. charset: Encoding for a :obj:`str` request URI or status message, **UTF-8** if not given. + raw_line: HTTP start line as received, CRLF excluded. + raw_header: HTTP field lines as received, CRLF excluded, one per + header field and in the order of ``headers``. **kwargs: Arbitrary keyword arguments. Returns: Constructed packet data. + Raises: + ProtocolError: If the values are neither a request's nor a + response's, or if ``raw_line`` or a ``raw_header`` entry that + reads back as the given values carries a CR or LF that the + values do not (other than the CRLF of an ``obs-fold``). + + """ + header_line = self._make_start_line(http_version, method=method, uri=uri, status=status, + status_default=status_default, + status_namespace=status_namespace, + status_reversed=status_reversed, + message=message, charset=charset) + if raw_line is not None: + try: + received = self._read_start_line(raw_line) + except ProtocolError: + pass + else: + if self._make_start_line(received.version, # type: ignore[attr-defined] + method=getattr(received, 'method', None), + uri=getattr(received, 'uri', None), + status=getattr(received, 'status', None), + message=getattr(received, 'message', None), + charset=received.charset) == header_line: # type: ignore[attr-defined] + text = getattr(received, 'uri', getattr(received, 'message', '')) # type: str + self._check_raw(raw_line, text, fold=False) + header_line = raw_line + b'\r\n' + + # NOTE: Field lines are matched to the header fields by content, in + # order, rather than by position, so that adding or removing a field + # leaves the field lines around it as they were received. + received_fields = [] # type: list[Optional[tuple[str, str]]] + for raw_field in raw_header: + try: + received_fields.append(self._read_field_line(raw_field)) + except ProtocolError: + received_fields.append(None) + + header_fields = [] # type: list[bytes] + start = 0 + for key, value in headers.items(multi=True) if headers is not None else (): + for index in range(start, len(received_fields)): + if received_fields[index] == (key, value): + self._check_raw(raw_header[index], key, value, fold=True) + header_fields.append(raw_header[index] + b'\r\n') + start = index + 1 + break + else: + header_fields.append(b'%s: %s\r\n' % (key.encode(), value.encode())) + + return Schema_HTTP( + data=header_line + b''.join(header_fields) + b'\r\n' + body, + ) + + @staticmethod + def _check_raw(raw: 'bytes', *values: 'str', fold: 'bool') -> 'None': + """Refuse a raw line that would write a line break its values lack. + + Reading a line back strips the whitespace around its tokens, and that + includes a lone CR or LF, so a raw line can read back as values that + carry no line break while writing one to the wire. + + Args: + raw: Raw start line or field line, CRLF excluded. + *values: The values ``raw`` reads back as. + fold: Whether the CRLF of an ``obs-fold``, followed by SP or HTAB, + is allowed in ``raw``. + + Raises: + ProtocolError: If ``raw`` carries a CR or LF that ``values`` do not. + + """ + if fold: + raw = re.sub(rb'\r\n(?=[ \t])', b'', raw) + for char in ('\r', '\n'): + if raw.count(char.encode()) != sum(value.count(char) for value in values): + raise ProtocolError('HTTP: raw line carries a CR or LF its values do not') + + def _make_start_line(self, http_version: 'str | bytes', *, + method: 'Optional[Enum_Method | str | bytes]' = None, + uri: 'Optional[str | bytes]' = None, + status: 'Optional[Enum_StatusCode | str | bytes | int]' = None, + status_default: 'Optional[int]' = None, + status_namespace: 'Optional[dict[str, int] | dict[int, str] | _Type[StdlibEnum] | _Type[AenumEnum]]' = None, # pylint: disable=line-too-long + status_reversed: 'bool' = False, + message: 'Optional[str | bytes]' = None, + charset: 'Optional[str]' = None) -> 'bytes': + """Make an HTTP/1.* start line from its values. + + Args: + http_version: HTTP version. + method: HTTP method. + uri: HTTP request URI. + status: HTTP status code. + status_default: Default HTTP status code. + status_namespace: Namespace of HTTP status code. + status_reversed: Whether to reverse the namespace. + message: HTTP status message. + charset: Encoding for a :obj:`str` request URI or status message, + **UTF-8** if not given. + + Returns: + The start line, CRLF included. + + Raises: + ProtocolError: If the values are neither a request's nor a + response's. + """ version = http_version.encode() if isinstance(http_version, str) else http_version if method is not None and status is None: @@ -314,16 +438,7 @@ def make(self, # type: ignore[override] header_line = b'HTTP/%s %s %s\r\n' % (version, str(status_code_val).encode(), msg) else: raise ProtocolError('HTTP packet must be either request or response.') - - header_fields = [] # type: list[bytes] - if headers is not None: - header_fields = [] - for key, value in headers.items(multi=True): - header_fields.append(b'%s: %s\r\n' % (key.encode(), value.encode())) - - return Schema_HTTP( - data=header_line + b''.join(header_fields) + b'\r\n' + body, - ) + return header_line @classmethod def id(cls) -> 'tuple[Literal["HTTP"], Literal["HTTPv1"]]': # type: ignore[override] @@ -362,6 +477,8 @@ def _make_data(cls, data: 'Data_HTTP') -> 'dict[str, Any]': # type: ignore[over 'headers': data.header, 'body': b'' if data.body is None else data.body, 'charset': getattr(data.receipt, 'charset', None), + 'raw_line': getattr(data.receipt, 'raw_line', None), + 'raw_header': getattr(data, 'raw_header', ()), } def _decode_text(self, text: 'bytes') -> 'tuple[str, Optional[str]]': @@ -422,55 +539,108 @@ def _read_http_header(self, header: 'bytes') -> 'tuple[Data_Header, OrderedMulti if header == _HTTP2_PREFACE_HEADER: raise ProtocolError('HTTP: invalid format') startline, _, headerfield = header.partition(b'\r\n') + header_line = self._read_start_line(startline) - # NOTE: Short for a start line of fewer than three whitespace-separated - # tokens. Raised as ``ProtocolError`` for the reason ``read`` gives - # above, with the same message this method uses below for a start line - # it cannot recognise. - try: - para1, para2, para3 = re.split(rb'\s+', startline, maxsplit=2) - except ValueError as error: - raise ProtocolError('HTTP: invalid format') from error + header_fields = OrderedMultiDict() # type: OrderedMultiDict[str, str] + for raw_field in self._split_field_lines(headerfield): + header_fields.add(*self._read_field_line(raw_field)) + + return header_line, header_fields + + @staticmethod + def _split_field_lines(headerfield: 'bytes') -> 'tuple[bytes, ...]': + """Split the field lines of an HTTP/1.* header. + + Args: + headerfield: The header after its start line and that line's CRLF. + Returns: + One field line per header field, as received and CRLF excluded. + + Raises: + ProtocolError: If the first field line is an ``obs-fold`` + continuation. + + """ # NOTE: A field line beginning with SP or HTAB is an ``obs-fold`` - # continuation of the line before it (:rfc:`9112#section-5.2`), and is - # unfolded here -- the RFC's own remedy -- rather than treated as a field - # line of its own. Deprecated, but present in real captures. Treating a - # continuation as a field line of its own goes wrong twice: one carrying - # no colon fails the colon check below, so a valid message is refused; - # one that happens to contain a colon parses silently into a spurious - # extra field (``X-Long: a`` plus ``b: c``, for a folded ``X-Long: a b``). - # Unfolded, a folded message parses to the field it actually carries. - fields = [] # type: list[bytes] + # continuation of the line before it (:rfc:`9112#section-5.2`), so it is + # kept with that line rather than treated as a field line of its own, + # and unfolded by ``_read_field_line``. Deprecated, but present in real + # captures. Treating a continuation as a field line of its own goes wrong + # twice: one carrying no colon fails the colon check, so a valid message + # is refused; one that happens to contain a colon parses silently into a + # spurious extra field (``X-Long: a`` plus ``b: c``, for a folded + # ``X-Long: a b``). + raw_fields = [] # type: list[bytes] for line in headerfield.split(b'\r\n') if headerfield else (): if line.startswith((b' ', b'\t')): # A continuation with nothing to continue -- the first field line # folded -- is malformed rather than unfoldable. - if not fields: + if not raw_fields: raise ProtocolError('HTTP: invalid format') - # NOTE: The accumulator is right-stripped as well as the - # continuation, because the production is ``obs-fold = OWS CRLF - # RWS`` and it is the *whole* obs-fold that is replaced by a - # single space -- the OWS before the CRLF belongs to the fold, - # not to the value. Stripping only the continuation would leave - # that OWS in place, so ``X: a \t\r\n\tb`` would unfold to - # ``'a \t b'`` rather than ``'a b'``, keeping a HTAB where the - # RFC prescribes SP. - fields[-1] = fields[-1].rstrip() + b' ' + line.strip() + raw_fields[-1] += b'\r\n' + line continue - fields.append(line) + raw_fields.append(line) + return tuple(raw_fields) + + def _read_field_line(self, line: 'bytes') -> 'tuple[str, str]': + """Read one HTTP/1.* field line. + + Args: + line: Field line as received, CRLF excluded, with any ``obs-fold`` + continuations still in it. + + Returns: + The field name and the field value, the latter unfolded and without + the optional whitespace around it. + + Raises: + ProtocolError: If the line is not a field line. + + """ + # NOTE: The accumulator is right-stripped as well as the continuation, + # because the production is ``obs-fold = OWS CRLF RWS`` and it is the + # *whole* obs-fold that is replaced by a single space -- the RFC's own + # remedy (:rfc:`9112#section-5.2`). The OWS before the CRLF belongs to + # the fold, not to the value, so ``X: a \t\r\n\tb`` unfolds to ``'a b'``. + field, *continuations = line.split(b'\r\n') + for continuation in continuations: + if not continuation.startswith((b' ', b'\t')): + raise ProtocolError('HTTP: invalid format') + field = field.rstrip() + b' ' + continuation.strip() # NOTE: Checked rather than left to ``item[1]``, and refused rather than # skipped: a field line with no colon is not a header field, and dropping # it would hand back a message whose fields are quietly not the ones on - # the wire. ``ProtocolError`` for the reason the start-line split above - # gives, and to the same message. - lists = [] # type: list[list[bytes]] - for field in fields: - item = re.split(rb'\s*:\s*', field, maxsplit=1) - if len(item) != 2: - raise ProtocolError('HTTP: invalid format') - lists.append(item) + # the wire. ``ProtocolError`` for the reason ``read`` gives, and to the + # same message. + item = re.split(rb'\s*:\s*', field, maxsplit=1) + if len(item) != 2: + raise ProtocolError('HTTP: invalid format') + return self.decode(item[0].strip()), self.decode(item[1].strip()) + + def _read_start_line(self, startline: 'bytes') -> 'Data_Header': + """Read an HTTP/1.* start line. + + Args: + startline: Start line as received, CRLF excluded. + + Returns: + Parsed start line. + + Raises: + ProtocolError: If the line is neither a request line nor a status + line. + + """ + # NOTE: Short for a start line of fewer than three whitespace-separated + # tokens. Raised as ``ProtocolError`` for the reason ``read`` gives, + # with the same message this method uses below for a start line it + # cannot recognise. + try: + para1, para2, para3 = re.split(rb'\s+', startline, maxsplit=2) + except ValueError as error: + raise ProtocolError('HTTP: invalid format') from error if TYPE_CHECKING: header_line: 'Data_Header' @@ -487,6 +657,7 @@ def _read_http_header(self, header: 'bytes') -> 'tuple[Data_Header, OrderedMulti uri=uri, version=self.decode(match2.group('version')), charset=charset, + raw_line=startline, ) elif match3 and match4: message, charset = self._decode_text(para3) @@ -496,17 +667,11 @@ def _read_http_header(self, header: 'bytes') -> 'tuple[Data_Header, OrderedMulti status=Enum_StatusCode.get(int(para2)), message=message, charset=charset, + raw_line=startline, ) else: raise ProtocolError('HTTP: invalid format') - - header_fields = OrderedMultiDict() # type: OrderedMultiDict[str, str] - for item in lists: - key = self.decode(item[0].strip()) - value = self.decode(item[1].strip()) - header_fields.add(key, value) - - return header_line, header_fields + return header_line def _read_http_body(self, body: 'bytes', *, headers: 'OrderedMultiDict[str, str]') -> 'Any': diff --git a/pcapkit/protocols/data/application/ftp.py b/pcapkit/protocols/data/application/ftp.py index 0e94014144..5381a4906a 100644 --- a/pcapkit/protocols/data/application/ftp.py +++ b/pcapkit/protocols/data/application/ftp.py @@ -41,9 +41,12 @@ class Request(FTP): args: 'Optional[str]' #: Charset to encode :attr:`args` with, :obj:`None` for UTF-8. charset: 'Optional[str]' + #: Command as received, in its original case; :attr:`cmmd` is the + #: case-insensitive lookup of it. + raw_cmmd: 'bytes' if TYPE_CHECKING: - def __init__(self, type: 'Literal[FTP_Type.REQUEST]', cmmd: 'Command', args: 'Optional[str]', charset: 'Optional[str]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, type: 'Literal[FTP_Type.REQUEST]', cmmd: 'Command', args: 'Optional[str]', charset: 'Optional[str]', raw_cmmd: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -60,6 +63,10 @@ class Response(FTP): more: 'bool' #: Charset to encode :attr:`args` with, :obj:`None` for UTF-8. charset: 'Optional[str]' + #: Lines of a multi-line reply after the first, as received and without + #: their CRLF, ending with the closing `` SP`` line; empty unless the + #: whole reply arrived at once. + lines: 'tuple[bytes, ...]' if TYPE_CHECKING: - def __init__(self, type: 'Literal[FTP_Type.RESPONSE]', code: 'ReturnCode', args: 'Optional[str]', more: 'bool', charset: 'Optional[str]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, type: 'Literal[FTP_Type.RESPONSE]', code: 'ReturnCode', args: 'Optional[str]', more: 'bool', charset: 'Optional[str]', lines: 'tuple[bytes, ...]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin diff --git a/pcapkit/protocols/data/application/httpv1.py b/pcapkit/protocols/data/application/httpv1.py index d1c8c19a52..70d6125c91 100644 --- a/pcapkit/protocols/data/application/httpv1.py +++ b/pcapkit/protocols/data/application/httpv1.py @@ -33,9 +33,12 @@ class HTTP(Protocol): header: 'OrderedMultiDict[str, str]' #: HTTP body. body: 'Any' + #: HTTP field lines as received, CRLF excluded and ``obs-fold`` + #: continuations kept: one per field in :attr:`header`, in order. + raw_header: 'tuple[bytes, ...]' if TYPE_CHECKING: - def __init__(self, receipt: 'Header', header: 'OrderedMultiDict[str, str]', body: 'Any') -> None: ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, receipt: 'Header', header: 'OrderedMultiDict[str, str]', body: 'Any', raw_header: 'tuple[bytes, ...]') -> None: ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin class Header(Data): @@ -59,9 +62,11 @@ class RequestHeader(Header): version: 'str' #: Charset to encode :attr:`uri` with, :obj:`None` for UTF-8. charset: 'Optional[str]' + #: HTTP request line as received, CRLF excluded. + raw_line: 'bytes' if TYPE_CHECKING: - def __init__(self, type: 'HTTP_Type', method: 'Enum_Method', uri: 'str', version: 'str', charset: 'Optional[str]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, type: 'HTTP_Type', method: 'Enum_Method', uri: 'str', version: 'str', charset: 'Optional[str]', raw_line: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -78,6 +83,8 @@ class ResponseHeader(Header): message: 'str' #: Charset to encode :attr:`message` with, :obj:`None` for UTF-8. charset: 'Optional[str]' + #: HTTP status line as received, CRLF excluded. + raw_line: 'bytes' if TYPE_CHECKING: - def __init__(self, type: 'HTTP_Type', version: 'str', status: 'Enum_StatusCode', message: 'str', charset: 'Optional[str]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, type: 'HTTP_Type', version: 'str', status: 'Enum_StatusCode', message: 'str', charset: 'Optional[str]', raw_line: 'bytes') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin diff --git a/tests/protocols/application/test_ftp_http1_raw_unit.py b/tests/protocols/application/test_ftp_http1_raw_unit.py new file mode 100644 index 0000000000..7dd5ea3aa3 --- /dev/null +++ b/tests/protocols/application/test_ftp_http1_raw_unit.py @@ -0,0 +1,221 @@ +# -*- coding: utf-8 -*- +"""FTP and HTTP/1.* keep what they normalise. C.f. #1302, #1303, #1333. + +- #1302: HTTP/1.* field values decoded with a guessed charset were encoded back + as UTF-8. The field lines are now kept as received (``raw_header``). +- #1303: a multi-line FTP reply arriving whole in one segment did not parse. The + lines after the first are now kept as received (``lines``). +- #1333: the FTP command case and the HTTP/1.* whitespace -- around start-line + tokens, around the field colon, and in ``obs-fold`` -- were normalised on + rebuild. The parsed values stay normalised for lookup, and the original + token or line is kept beside them (``raw_cmmd``, ``raw_line``, + ``raw_header``) and written back whenever it still reads as those values. + +""" + +from __future__ import annotations + +import importlib.util +import unittest + +from tests._support import reimport_once_per_class + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) + +#: HTTP/1.* messages: name, octets, the normalised values they parse to. +HTTP1_MESSAGES = ( + ('latin-1 field value', b'HTTP/1.1 200 OK\r\nX-Name: caf\xe9\r\n\r\n', # 1302 + {'message': 'OK'}, [('X-Name', 'café')]), + ('leading HTAB in reason phrase', b'HTTP/1.1 200 \tOK\r\n\r\n', # 1333 + {'message': 'OK'}, []), + ('leading SP in reason phrase', b'HTTP/1.1 200 OK\r\n\r\n', + {'message': 'OK'}, []), + ('trailing HTAB in request URI', b'GET /a\t HTTP/1.1\r\n\r\n', + {'uri': '/a'}, []), + ('trailing FF in request URI', b'GET /a\x0c HTTP/1.1\r\n\r\n', + {'uri': '/a'}, []), + ('no space after colon', b'GET / HTTP/1.1\r\nHost:a\r\n\r\n', + {'uri': '/'}, [('Host', 'a')]), + ('obs-fold', b'GET / HTTP/1.1\r\nX: a\r\n b\r\n\r\n', + {'uri': '/'}, [('X', 'a b')]), + ('whitespace around field', b'GET / HTTP/1.1\r\nX : a \r\nY:\tb\r\n\r\nbody', + {'uri': '/'}, [('X', 'a'), ('Y', 'b')]), +) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class FTPRawTests(unittest.TestCase): + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_command_case_rebuilds_byte_exactly(self) -> None: + from pcapkit.const.ftp.command import Command + from pcapkit.protocols.application.ftp import FTP + + for raw, command in ((b'quit\r\n', Command.QUIT), (b'Retr file.txt\r\n', Command.RETR)): + with self.subTest(raw=raw): + request = FTP(raw, len(raw)).info + self.assertIs(request.cmmd, command) + self.assertEqual(request.raw_cmmd, raw.split(b' ')[0].rstrip()) + self.assertEqual(FTP.from_data(request).data, raw) + + def test_changed_command_is_written_from_the_change(self) -> None: + from pcapkit.protocols.application.ftp import FTP + + ftp = object.__new__(FTP) + self.assertEqual(ftp.make(cmmd='USER', raw_cmmd=b'quit').data, b'USER\r\n') + self.assertEqual(ftp.make(cmmd='QUIT').data, b'QUIT\r\n') + + def test_whole_multi_line_reply_parses_and_rebuilds(self) -> None: + from pcapkit.protocols.application.ftp import FTP + + for raw, lines in ((b'220-a\r\n220 b\r\n', (b'220 b',)), + (b'230-a\r\n free text\r\n230-b\r\n230 end\r\n', + (b' free text', b'230-b', b'230 end'))): + with self.subTest(raw=raw): + reply = FTP(raw, len(raw)).info + self.assertEqual(int(reply.code), int(raw[:3])) + self.assertTrue(reply.more) + self.assertEqual(reply.args, 'a') + self.assertEqual(reply.lines, lines) + self.assertEqual(FTP.from_data(reply).data, raw) + + single = FTP(b'220 a\r\n', 7).info + self.assertEqual(single.lines, ()) + + def test_multi_line_reply_must_end_at_its_closing_line(self) -> None: + from pcapkit.protocols.application.ftp import FTP + from pcapkit.utilities.exceptions import ProtocolError + + ftp = object.__new__(FTP) + ftp.__cached__ = {} + for raw in (b'220-a\r\nfoo\r\n', # no closing line + b'220-a\r\n221 b\r\n', # a different code + b'220-a\r\n220 b\r\n331 x\r\n', # the next reply after it + b'220-a\r\n220-b\r\n220 c\r\n220 d\r\n'): # past the first closing line + with self.subTest(raw=raw): + ftp.__header__ = type('Header', (), {'data': raw})() + with self.assertRaises(ProtocolError): + ftp.read(length=len(raw)) + code, _, rest = raw.partition(b'-') + with self.assertRaises(ProtocolError): + ftp.make(code=int(code), args='a', more=True, + lines=tuple(rest.split(b'\r\n')[1:-1])) + + raw = b'220-a\r\n220-b\r\n221 digits\r\n220 c\r\n' + reply = FTP(raw, len(raw)).info + self.assertEqual(reply.lines, (b'220-b', b'221 digits', b'220 c')) + self.assertEqual(FTP.from_data(reply).data, raw) + + def test_lines_must_be_bytes(self) -> None: + from pcapkit.protocols.application.ftp import FTP + from pcapkit.utilities.exceptions import ProtocolError + + for lines in ('220 b', ('220 b',), (b'220 b\n',)): + with self.subTest(lines=lines): + with self.assertRaises(ProtocolError): + object.__new__(FTP).make(code=220, args='a', more=True, lines=lines) + + def test_lines_follow_only_a_multi_line_reply(self) -> None: + from pcapkit.protocols.application.ftp import FTP + from pcapkit.utilities.exceptions import ProtocolError + + ftp = object.__new__(FTP) + ftp.__cached__ = {} + for raw in (b'220 a\r\n221 b\r\n', b'USER a\r\nPASS b\r\n'): + with self.subTest(raw=raw): + ftp.__header__ = type('Header', (), {'data': raw})() + with self.assertRaises(ProtocolError): + ftp.read(length=len(raw)) + with self.assertRaises(ProtocolError): + ftp.make(code=220, args='a', lines=(b'220 b',)) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class HTTP1RawTests(unittest.TestCase): + def setUp(self) -> None: + reimport_once_per_class(self) + + def test_parse_then_rebuild_is_byte_exact(self) -> None: + from pcapkit.protocols.application.http import HTTP + from pcapkit.protocols.application.httpv1 import HTTP as HTTPv1 + + for cls in (HTTPv1, HTTP): + for name, raw, _, _ in HTTP1_MESSAGES: + with self.subTest(cls=cls.__module__, name=name): + self.assertEqual(cls.from_data(cls(raw, len(raw)).info).data, raw) + + def test_parsed_values_stay_normalised(self) -> None: + from pcapkit.protocols.application.httpv1 import HTTP + + for name, raw, receipt, fields in HTTP1_MESSAGES: + with self.subTest(name=name): + info = HTTP(raw, len(raw)).info + for key, value in receipt.items(): + self.assertEqual(getattr(info.receipt, key), value) + self.assertEqual(list(info.header.items(multi=True)), fields) + self.assertEqual(info.receipt.raw_line, raw.split(b'\r\n')[0]) + self.assertEqual(len(info.raw_header), len(fields)) + + def test_changed_values_are_written_from_the_change(self) -> None: + from pcapkit.corekit.multidict import OrderedMultiDict + from pcapkit.protocols.application.httpv1 import HTTP + + raw = b'HTTP/1.1 200 OK\r\nA:1\r\nB:\tcaf\xe9\r\n\r\n' + info = HTTP(raw, len(raw)).info + http = object.__new__(HTTP) + + headers = OrderedMultiDict([('A', '2'), ('B', 'café')]) + self.assertEqual( + http.make(**{**HTTP._make_data(info), 'headers': headers}).data, + b'HTTP/1.1 200 OK\r\nA: 2\r\nB:\tcaf\xe9\r\n\r\n', + ) + self.assertEqual( + http.make(**{**HTTP._make_data(info), 'message': 'Fine'}).data, + b'HTTP/1.1 200 Fine\r\nA:1\r\nB:\tcaf\xe9\r\n\r\n', + ) + + def test_raw_lines_cannot_write_a_line_break_their_values_lack(self) -> None: + from pcapkit.corekit.multidict import OrderedMultiDict + from pcapkit.protocols.application.httpv1 import HTTP + from pcapkit.utilities.exceptions import ProtocolError + + http = object.__new__(HTTP) + for raw in (b'X: a\r', b'X: a\n', b'\nX: a', b'X: a\r\n \nInjected: b'): + with self.subTest(raw_header=raw): + headers = OrderedMultiDict([http._read_field_line(raw)]) + with self.assertRaises(ProtocolError): + http.make(status=200, message='OK', headers=headers, raw_header=(raw,)) + for raw in (b'HTTP/1.1 200\r OK', b'HTTP/1.1 200\nOK', b'HTTP/1.1\r\n 200 OK'): + with self.subTest(raw_line=raw): + with self.assertRaises(ProtocolError): + http.make(status=200, message='OK', raw_line=raw) + + def test_line_breaks_the_values_carry_still_rebuild(self) -> None: + from pcapkit.protocols.application.httpv1 import HTTP + + for raw in (b'GET / HTTP/1.1\r\nX: a\nb\r\n\r\n', b'GET / HTTP/1.1\r\nX: a\r\n\tb\r\n\r\n'): + with self.subTest(raw=raw): + self.assertEqual(HTTP.from_data(HTTP(raw, len(raw)).info).data, raw) + + def test_field_lines_are_matched_by_content(self) -> None: + from pcapkit.corekit.multidict import OrderedMultiDict + from pcapkit.protocols.application.httpv1 import HTTP + + raw = b'GET / HTTP/1.1\r\nA:1\r\nB: 2\r\nC:3\r\n\r\n' + data = HTTP._make_data(HTTP(raw, len(raw)).info) + http = object.__new__(HTTP) + for headers, expected in ( + ([('B', '2'), ('C', '3')], b'B: 2\r\nC:3\r\n'), + ([('A', '1'), ('N', 'new'), ('B', '2'), ('C', '3')], b'A:1\r\nN: new\r\nB: 2\r\nC:3\r\n'), + ): + with self.subTest(headers=headers): + self.assertEqual( + http.make(**{**data, 'headers': OrderedMultiDict(headers)}).data, + b'GET / HTTP/1.1\r\n' + expected + b'\r\n', + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/protocols/application/test_ftp_unit.py b/tests/protocols/application/test_ftp_unit.py index 54f575cf0f..23c6f313e1 100644 --- a/tests/protocols/application/test_ftp_unit.py +++ b/tests/protocols/application/test_ftp_unit.py @@ -202,18 +202,20 @@ def test_ftp_make_data_preserves_request_and_response_fields(self) -> None: from pcapkit.const.ftp.return_code import ReturnCode from pcapkit.protocols.application.ftp import FTP - request = DummyData(cmmd=Command.USER, code=None, args='anonymous', more=False, charset=None) + request = DummyData(cmmd=Command.USER, code=None, args='anonymous', more=False, charset=None, + raw_cmmd=None, lines=()) response = DummyData(cmmd=None, code=ReturnCode.CODE_220, args='Ready', more=True, - charset='latin-1') + charset='latin-1', raw_cmmd=None, lines=()) self.assertEqual( FTP._make_data(request), - {'cmmd': Command.USER, 'code': None, 'args': 'anonymous', 'more': False, 'charset': None}, + {'cmmd': Command.USER, 'code': None, 'args': 'anonymous', 'more': False, 'charset': None, + 'raw_cmmd': None, 'lines': ()}, ) self.assertEqual( FTP._make_data(response), {'cmmd': None, 'code': ReturnCode.CODE_220, 'args': 'Ready', 'more': True, - 'charset': 'latin-1'}, + 'charset': 'latin-1', 'raw_cmmd': None, 'lines': ()}, ) def test_ftp_data_channel_name_is_stable(self) -> None: diff --git a/tests/protocols/application/test_http_unit.py b/tests/protocols/application/test_http_unit.py index a1a6f45d0e..5765ef1ab0 100644 --- a/tests/protocols/application/test_http_unit.py +++ b/tests/protocols/application/test_http_unit.py @@ -1608,6 +1608,7 @@ def test_httpv1_id_make_data_and_request_construction(self) -> None: receipt=SimpleNamespace(version='1.1', method=Method.GET, uri='/index.html'), header=headers, body=b'body', + raw_header=(), ) self.assertEqual(HTTPv1.id(), ('HTTP', 'HTTPv1'))