fix(ftp,httpv1): keep the raw command case, HTTP/1 whitespace and field octets, and parse whole multi-line FTP replies

pcapkit/protocols/application/ftp.py

@@ -31,7 +31,7 @@
 from pcapkit.utilities.exceptions import ProtocolError, UnsupportedCall
 
 if TYPE_CHECKING:
-    from typing import Any, NoReturn, Optional
+    from typing import Any, Iterable, NoReturn, Optional
 
     from typing_extensions import Literal
 
@@ -110,20 +110,34 @@ def read(self, length: 'Optional[int]' = None, **kwargs: 'Any') -> 'Data_FTP':
         schema = self.__header__
 
         data = schema.data
-        if (match := FTP_REQUEST.match(data)) is not None:
-            cmmd = match.group('cmmd').decode()
+        # NOTE: A multi-line reply (:rfc:`959#section-4`) may arrive whole in one
+        # segment. Its first line is parsed as any reply line, and the lines
+        # after it are kept as received in ``lines``. A segment holding anything
+        # else -- no closing line, or more after it -- is refused as before.
+        line, lines = data, ()  # type: bytes, tuple[bytes, ...]
+        if FTP_RESPONSE.match(data) is None and data.endswith(b'\r\n'):
+            first, _, rest = data.partition(b'\r\n')
+            if rest and (match := FTP_RESPONSE.match(first + b'\r\n')) is not None \
+                    and match.group('more'):
+                rest_lines = tuple(rest[:-2].split(b'\r\n'))
+                if self._test_lines(match.group('code'), rest_lines):
+                    line, lines = first + b'\r\n', rest_lines
+
+        if (match := FTP_REQUEST.match(line)) is not None:
+            raw_cmmd = match.group('cmmd')
             args = match.group('args')
 
-            cmmd_val = Enum_Command.get(cmmd)
+            cmmd_val = Enum_Command.get(raw_cmmd.decode())
             args_val, charset = self._decode_args(args)
 
             ftp = Data_Request(
                 type=Type.REQUEST,
                 cmmd=cmmd_val,
                 args=args_val,
                 charset=charset,
+                raw_cmmd=raw_cmmd,
             )  # type: Data_FTP
-        elif (match := FTP_RESPONSE.match(data)) is not None:
+        elif (match := FTP_RESPONSE.match(line)) is not None:
             code = int(match.group('code'))
             more = bool(match.group('more'))
             args = match.group('text') if more else match.group('args')
@@ -137,6 +151,7 @@ def read(self, length: 'Optional[int]' = None, **kwargs: 'Any') -> 'Data_FTP':
                 more=more,
                 args=args_val,
                 charset=charset,
+                lines=lines,
             )
         else:
             raise ProtocolError('FTP: invalid packet format')
@@ -148,6 +163,8 @@ def make(self,
              args: 'Optional[str | bytes]' = None,
              more: 'bool' = False,
              charset: 'Optional[str]' = None,
+             raw_cmmd: 'Optional[bytes]' = None,
+             lines: 'Iterable[bytes]' = (),
              **kwargs: 'Any') -> 'Schema_FTP':
         """Make (construct) packet data.
 
@@ -156,17 +173,30 @@ def make(self,
         the command or code, except after the hyphen of a multi-line reply,
         which the text follows immediately (:rfc:`959#section-4`).
 
+        Commands are case-insensitive (:rfc:`959#section-5`), so ``raw_cmmd``
+        is written in place of ``cmmd`` whenever the two differ only in case,
+        which keeps the case the command was read in.
+
         Args:
             cmmd: FTP command.
             code: FTP status code.
             args: Optional FTP command arguments and/or status messages.
             more: More status messages to follow for response packets.
             charset: Encoding for :obj:`str` arguments, **UTF-8** if not given.
+            raw_cmmd: FTP command as received, in its original case.
+            lines: Lines of a multi-line reply after the first, each written
+                verbatim and terminated with CRLF; the last, and only the last,
+                is the closing line, ``<code> SP <text>``.
             **kwargs: Arbitrary keyword arguments.
 
         Returns:
             Constructed packet data.
 
+        Raises:
+            ProtocolError: If neither or both of ``cmmd`` and ``code`` are
+                given, or if ``lines`` are given other than as the rest of a
+                multi-line reply.
+
         """
         if cmmd is not None and code is None:
             if isinstance(cmmd, bytes):
@@ -175,6 +205,8 @@ def make(self,
                 prefix = cmmd.encode()
             else:
                 prefix = cmmd.value
+            if raw_cmmd is not None and raw_cmmd.upper() == prefix.upper():
+                prefix = raw_cmmd
 
             mf = b''
         elif cmmd is None and code is not None:
@@ -185,6 +217,10 @@ def make(self,
         else:
             raise ProtocolError('FTP: invalid packet type')
 
+        lines = tuple(lines)
+        if lines and not (mf and self._test_lines(prefix, lines)):
+            raise ProtocolError('FTP: lines must close a multi-line reply')
+
         if args is None:
             suffix = b''
         elif isinstance(args, bytes):
@@ -195,7 +231,7 @@ def make(self,
         # the separating SP, absent with no arguments and after a hyphen
         sep = b'' if args is None or mf else b' '
         return Schema_FTP(
-            data=b'%s%s%s%s\r\n' % (prefix, mf, sep, suffix),
+            data=b'%s%s%s%s\r\n' % (prefix, mf, sep, suffix) + b''.join(b'%s\r\n' % line for line in lines),
         )
 
     ##########################################################################
@@ -219,8 +255,32 @@ def _make_data(cls, data: 'Data_FTP') -> 'dict[str, Any]':  # type: ignore[overr
             'args': getattr(data, 'args', None),
             'more': getattr(data, 'more', False),
             'charset': getattr(data, 'charset', None),
+            'raw_cmmd': getattr(data, 'raw_cmmd', None),
+            'lines': getattr(data, 'lines', ()),
         }
 
+    @staticmethod
+    def _test_lines(code: 'bytes', lines: 'tuple[bytes, ...]') -> 'bool':
+        """Whether ``lines`` are the rest of a multi-line reply with ``code``.
+
+        A multi-line reply ends at the first line that starts with its code
+        followed by SP (:rfc:`959#section-4`). The lines before that one are
+        free text, which may begin with digits -- the code with a hyphen
+        included -- but, as in any reply line, carry no LF.
+
+        Args:
+            code: Reply code of the first line.
+            lines: Lines after the first, CRLF excluded.
+
+        Returns:
+            Whether the last line, and only the last, is the closing line.
+
+        """
+        closing = code + b' '
+        if not lines or not all(isinstance(line, bytes) and b'\n' not in line for line in lines):
+            return False
+        return lines[-1].startswith(closing) and not any(line.startswith(closing) for line in lines[:-1])
+
     def _decode_args(self, args: 'Optional[bytes]') -> 'tuple[Optional[str], Optional[str]]':
         """Decode arguments, recording how to encode them back.