diff --git a/pixi.toml b/pixi.toml index ab89f17ab..22c35e15e 100644 --- a/pixi.toml +++ b/pixi.toml @@ -18,7 +18,6 @@ parmed = "*" pyarrow = "*" py3dmol = "*" pydot = "*" -pygtail = "*" pyyaml = "*" rdkit = "*" diff --git a/recipes/biosimspace/recipe.yaml b/recipes/biosimspace/recipe.yaml index 73bf91b07..2682967e5 100644 --- a/recipes/biosimspace/recipe.yaml +++ b/recipes/biosimspace/recipe.yaml @@ -37,7 +37,6 @@ requirements: - pyarrow - py3dmol - pydot - - pygtail - python - pyyaml - rdkit diff --git a/src/BioSimSpace/Process/_amber.py b/src/BioSimSpace/Process/_amber.py index f8dd37342..ca3ac81ab 100644 --- a/src/BioSimSpace/Process/_amber.py +++ b/src/BioSimSpace/Process/_amber.py @@ -26,11 +26,6 @@ __all__ = ["Amber"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from . import _process @@ -2712,7 +2707,7 @@ def stdout(self, n=10): self._is_header = False # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) line = line.strip() diff --git a/src/BioSimSpace/Process/_gromacs.py b/src/BioSimSpace/Process/_gromacs.py index 16c8e6f68..85cf5714c 100644 --- a/src/BioSimSpace/Process/_gromacs.py +++ b/src/BioSimSpace/Process/_gromacs.py @@ -26,11 +26,6 @@ __all__ = ["Gromacs"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from . import _process @@ -2104,7 +2099,7 @@ def stdout(self, n=10): raise ValueError("The number of lines must be positive!") # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) # Get the current number of lines. diff --git a/src/BioSimSpace/Process/_namd.py b/src/BioSimSpace/Process/_namd.py index a4e780451..105b0d16d 100644 --- a/src/BioSimSpace/Process/_namd.py +++ b/src/BioSimSpace/Process/_namd.py @@ -26,11 +26,6 @@ __all__ = ["Namd"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from . import _process @@ -2127,7 +2122,7 @@ def stdout(self, n=10): raise ValueError("The number of lines must be positive!") # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) # Split the record using whitespace. diff --git a/src/BioSimSpace/Process/_openmm.py b/src/BioSimSpace/Process/_openmm.py index b399f8ac6..2c1a04a04 100644 --- a/src/BioSimSpace/Process/_openmm.py +++ b/src/BioSimSpace/Process/_openmm.py @@ -26,11 +26,6 @@ __all__ = ["OpenMM"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from . import _process @@ -1988,7 +1983,7 @@ def stdout(self, n=10): raise ValueError("The number of lines must be positive!") # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) # Get the current number of lines. @@ -2295,7 +2290,7 @@ def _update_stdout_dict(self): lines = [] # Append any new lines. - for line in _pygtail.Pygtail(self._log_file): + for line in self._tail(self._log_file): lines.append(line) # Append any new records to the stdout dictionary. diff --git a/src/BioSimSpace/Process/_plumed.py b/src/BioSimSpace/Process/_plumed.py index efd8cd14c..d4bb0c0f8 100644 --- a/src/BioSimSpace/Process/_plumed.py +++ b/src/BioSimSpace/Process/_plumed.py @@ -26,12 +26,8 @@ __all__ = ["Plumed"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from .. import Types as _Types +from .._Utils import Tail as _Tail class Plumed: @@ -104,6 +100,8 @@ def __init__(self, work_dir): # Set the location of the HILLS and COLVAR files. self._hills_file = _os.path.join(str(self._work_dir), "HILLS") self._colvar_file = _os.path.join(str(self._work_dir), "COLVAR") + self._hills_tail = _Tail(self._hills_file) + self._colvar_tail = _Tail(self._colvar_file) # The number of collective variables and total number of components. self._num_colvar = 0 @@ -247,15 +245,9 @@ def _createMetadynamicsConfig(self, system, protocol, property_map={}): self._config = [] self._aux_files = [] - # Always remove pygtail offset files. - try: - _os.remove(_os.path.join(str(self._work_dir), "COLVAR.offset")) - except: - pass - try: - _os.remove(_os.path.join(str(self._work_dir), "HILLS.offset")) - except: - pass + # Reset the incremental file readers. + self._colvar_tail = _Tail(self._colvar_file) + self._hills_tail = _Tail(self._hills_file) # Restart if existing HILLS and COLVAR files are present. if _os.path.isfile(self._colvar_file) and _os.path.isfile(self._hills_file): @@ -959,11 +951,8 @@ def _createSteeringConfig(self, system, protocol, property_map={}): self._config = [] self._aux_files = [] - # Always remove pygtail offset files. - try: - _os.remove(_os.path.join(str(self._work_dir), "COLVAR.offset")) - except: - pass + # Reset the incremental file reader. + self._colvar_tail = _Tail(self._colvar_file) # Restart if an existing COLVAR files is present. if _os.path.isfile(self._colvar_file): @@ -1610,7 +1599,7 @@ def _update_colvar_dict(self): # Parse the HILLS file for OpenMM. if self._use_hills: # Loop over all new lines in the file. - for line in _pygtail.Pygtail(self._hills_file): + for line in self._hills_tail: # Is this a header line. If so, store the keys. if line[3:9] == "FIELDS": self._colvar_keys = line[10:].split()[: self._num_components + 1] @@ -1623,7 +1612,7 @@ def _update_colvar_dict(self): else: # Loop over all new lines in the file. - for line in _pygtail.Pygtail(self._colvar_file): + for line in self._colvar_tail: # Is this a header line. If so, store the keys. if line[3:9] == "FIELDS": self._colvar_keys = line[10:].split() @@ -1643,7 +1632,7 @@ def _update_hills_dict(self): return # Loop over all new lines in the file. - for line in _pygtail.Pygtail(self._hills_file): + for line in self._hills_tail: # Is this a header line. If so, store the keys. if line[3:9] == "FIELDS": self._hills_keys = line[10:].split() diff --git a/src/BioSimSpace/Process/_process.py b/src/BioSimSpace/Process/_process.py index 496f14d13..a3be37954 100644 --- a/src/BioSimSpace/Process/_process.py +++ b/src/BioSimSpace/Process/_process.py @@ -27,11 +27,6 @@ __all__ = ["Process"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from .. import Units as _Units from .. import _is_notebook @@ -332,9 +327,6 @@ def __repr__(self): def _clear_output(self): """Reset stdout and stderr.""" - import glob as _glob - import os as _os - # Create the files. This makes sure that the 'stdout' and 'stderr' # methods can be called when the files are empty. open(self._stdout_file, "a").close() @@ -344,19 +336,31 @@ def _clear_output(self): self._stdout = [] self._stderr = [] - # Clean up any existing offset files. - offset_files = _glob.glob(_os.path.join(str(self._work_dir), "*.offset")) + # Reset the incremental file readers. + self._tails = {} - # Remove any HILLS or COLVAR files from the list. These will be dealt - # with by the PLUMED interface. - try: - offset_files.remove(_os.path.join(str(self._work_dir), "COLVAR.offset")) - offset_files.remove(_os.path.join(str(self._work_dir), "HILLS.offset")) - except: - pass + def _tail(self, filename): + """ + Return the lines appended to a file since it was last read. + + Parameters + ---------- + + filename : str + The path to the file. + + Returns + ------- + + lines : iterable + An iterable over the new lines. + """ + from .._Utils import Tail as _Tail + + if filename not in self._tails: + self._tails[filename] = _Tail(filename) - for file in offset_files: - _os.remove(file) + return self._tails[filename] def _getPlumedConfig(self): """ @@ -1018,7 +1022,7 @@ def stdout(self, n=10): raise ValueError("The number of lines must be positive!") # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) # Get the current number of lines. @@ -1050,7 +1054,7 @@ def stderr(self, n=10): raise ValueError("The number of lines must be positive!") # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stderr_file): + for line in self._tail(self._stderr_file): self._stderr.append(line.rstrip()) # Get the current number of lines. @@ -1126,7 +1130,7 @@ def getStdout(self, block="AUTO"): self.wait() # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) return self._stdout.copy() @@ -1155,7 +1159,7 @@ def getStderr(self, block="AUTO"): self.wait() # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stderr_file): + for line in self._tail(self._stderr_file): self._stderr.append(line.rstrip()) return self._stderr.copy() diff --git a/src/BioSimSpace/Process/_somd.py b/src/BioSimSpace/Process/_somd.py index 47f785eb9..56a89e175 100644 --- a/src/BioSimSpace/Process/_somd.py +++ b/src/BioSimSpace/Process/_somd.py @@ -26,10 +26,6 @@ __all__ = ["Somd"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - import string as _string from . import _process @@ -876,7 +872,7 @@ def getGradient(self, time_series=False, block="AUTO"): return None # Append any new lines to the gradients list. - for line in _pygtail.Pygtail(self._gradient_file): + for line in self._tail(self._gradient_file): # Ignore comments. if line[0] != "#": self._gradients.append(float(line.rstrip().split()[-1])) diff --git a/src/BioSimSpace/Sandpit/Exscientia/Process/_amber.py b/src/BioSimSpace/Sandpit/Exscientia/Process/_amber.py index 36a52762b..b85fcb989 100644 --- a/src/BioSimSpace/Sandpit/Exscientia/Process/_amber.py +++ b/src/BioSimSpace/Sandpit/Exscientia/Process/_amber.py @@ -27,11 +27,6 @@ __all__ = ["Amber"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from .._Utils import _have_imported, _try_import # alchemlyb isn't available on all variants of Python that we support, so we @@ -2795,7 +2790,7 @@ def stdout(self, n=10): self._is_header = False # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) line = line.strip() @@ -3053,9 +3048,6 @@ def _init_stdout_dict(self): into the working directory, start and wait again. In this case, the result will be a combination of both runs. This function ensures that the results are regenerated from the new output file.""" - import os - from pathlib import Path as _Path - # Initialise dictionaries to hold stdout records for all possible # degrees of freedom. For regular simulations there will be one, # for free-energy simulations there will be three, i.e. one for @@ -3080,9 +3072,9 @@ def _init_stdout_dict(self): self._finished_results = False self._is_header = False - # Initiate the pytails. - for file in _Path(self.workDir()).glob("*.out.offset"): - os.remove(file) + # Reset the incremental readers for the output files. + for file in [f for f in self._tails if f.endswith(".out")]: + del self._tails[file] def _saveMetric( self, filename="metric.parquet", u_nk="u_nk.parquet", dHdl="dHdl.parquet" diff --git a/src/BioSimSpace/Sandpit/Exscientia/Process/_gromacs.py b/src/BioSimSpace/Sandpit/Exscientia/Process/_gromacs.py index b7125b955..2038a84ce 100644 --- a/src/BioSimSpace/Sandpit/Exscientia/Process/_gromacs.py +++ b/src/BioSimSpace/Sandpit/Exscientia/Process/_gromacs.py @@ -27,11 +27,6 @@ __all__ = ["Gromacs"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from .._Utils import _have_imported, _try_import # alchemlyb isn't available on all variants of Python that we support, so we @@ -2187,7 +2182,7 @@ def stdout(self, n=10): raise ValueError("The number of lines must be positive!") # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) # Get the current number of lines. diff --git a/src/BioSimSpace/Sandpit/Exscientia/Process/_namd.py b/src/BioSimSpace/Sandpit/Exscientia/Process/_namd.py index 130e1d400..12727b004 100644 --- a/src/BioSimSpace/Sandpit/Exscientia/Process/_namd.py +++ b/src/BioSimSpace/Sandpit/Exscientia/Process/_namd.py @@ -26,11 +26,6 @@ __all__ = ["Namd"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from . import _process @@ -2103,7 +2098,7 @@ def stdout(self, n=10): raise ValueError("The number of lines must be positive!") # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) # Split the record using whitespace. diff --git a/src/BioSimSpace/Sandpit/Exscientia/Process/_openmm.py b/src/BioSimSpace/Sandpit/Exscientia/Process/_openmm.py index 9480a0bf5..51b311c44 100644 --- a/src/BioSimSpace/Sandpit/Exscientia/Process/_openmm.py +++ b/src/BioSimSpace/Sandpit/Exscientia/Process/_openmm.py @@ -26,11 +26,6 @@ __all__ = ["OpenMM"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from . import _process @@ -1935,7 +1930,7 @@ def stdout(self, n=10): raise ValueError("The number of lines must be positive!") # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) # Get the current number of lines. @@ -2229,7 +2224,7 @@ def _update_stdout_dict(self): lines = [] # Append any new lines. - for line in _pygtail.Pygtail(self._log_file): + for line in self._tail(self._log_file): lines.append(line) # Append any new records to the stdout dictionary. diff --git a/src/BioSimSpace/Sandpit/Exscientia/Process/_plumed.py b/src/BioSimSpace/Sandpit/Exscientia/Process/_plumed.py index efd8cd14c..d4bb0c0f8 100644 --- a/src/BioSimSpace/Sandpit/Exscientia/Process/_plumed.py +++ b/src/BioSimSpace/Sandpit/Exscientia/Process/_plumed.py @@ -26,12 +26,8 @@ __all__ = ["Plumed"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from .. import Types as _Types +from .._Utils import Tail as _Tail class Plumed: @@ -104,6 +100,8 @@ def __init__(self, work_dir): # Set the location of the HILLS and COLVAR files. self._hills_file = _os.path.join(str(self._work_dir), "HILLS") self._colvar_file = _os.path.join(str(self._work_dir), "COLVAR") + self._hills_tail = _Tail(self._hills_file) + self._colvar_tail = _Tail(self._colvar_file) # The number of collective variables and total number of components. self._num_colvar = 0 @@ -247,15 +245,9 @@ def _createMetadynamicsConfig(self, system, protocol, property_map={}): self._config = [] self._aux_files = [] - # Always remove pygtail offset files. - try: - _os.remove(_os.path.join(str(self._work_dir), "COLVAR.offset")) - except: - pass - try: - _os.remove(_os.path.join(str(self._work_dir), "HILLS.offset")) - except: - pass + # Reset the incremental file readers. + self._colvar_tail = _Tail(self._colvar_file) + self._hills_tail = _Tail(self._hills_file) # Restart if existing HILLS and COLVAR files are present. if _os.path.isfile(self._colvar_file) and _os.path.isfile(self._hills_file): @@ -959,11 +951,8 @@ def _createSteeringConfig(self, system, protocol, property_map={}): self._config = [] self._aux_files = [] - # Always remove pygtail offset files. - try: - _os.remove(_os.path.join(str(self._work_dir), "COLVAR.offset")) - except: - pass + # Reset the incremental file reader. + self._colvar_tail = _Tail(self._colvar_file) # Restart if an existing COLVAR files is present. if _os.path.isfile(self._colvar_file): @@ -1610,7 +1599,7 @@ def _update_colvar_dict(self): # Parse the HILLS file for OpenMM. if self._use_hills: # Loop over all new lines in the file. - for line in _pygtail.Pygtail(self._hills_file): + for line in self._hills_tail: # Is this a header line. If so, store the keys. if line[3:9] == "FIELDS": self._colvar_keys = line[10:].split()[: self._num_components + 1] @@ -1623,7 +1612,7 @@ def _update_colvar_dict(self): else: # Loop over all new lines in the file. - for line in _pygtail.Pygtail(self._colvar_file): + for line in self._colvar_tail: # Is this a header line. If so, store the keys. if line[3:9] == "FIELDS": self._colvar_keys = line[10:].split() @@ -1643,7 +1632,7 @@ def _update_hills_dict(self): return # Loop over all new lines in the file. - for line in _pygtail.Pygtail(self._hills_file): + for line in self._hills_tail: # Is this a header line. If so, store the keys. if line[3:9] == "FIELDS": self._hills_keys = line[10:].split() diff --git a/src/BioSimSpace/Sandpit/Exscientia/Process/_process.py b/src/BioSimSpace/Sandpit/Exscientia/Process/_process.py index 1dbee6800..d8fa1a6b2 100644 --- a/src/BioSimSpace/Sandpit/Exscientia/Process/_process.py +++ b/src/BioSimSpace/Sandpit/Exscientia/Process/_process.py @@ -27,11 +27,6 @@ __all__ = ["Process"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") - - from .. import Units as _Units from .. import _is_notebook from ..Types import Time as _Time @@ -334,9 +329,6 @@ def __repr__(self): def _clear_output(self): """Reset stdout and stderr.""" - import glob as _glob - import os as _os - # Create the files. This makes sure that the 'stdout' and 'stderr' # methods can be called when the files are empty. open(self._stdout_file, "a").close() @@ -346,19 +338,31 @@ def _clear_output(self): self._stdout = [] self._stderr = [] - # Clean up any existing offset files. - offset_files = _glob.glob("%s/*.offset" % self._work_dir) + # Reset the incremental file readers. + self._tails = {} - # Remove any HILLS or COLVAR files from the list. These will be dealt - # with by the PLUMED interface. - try: - offset_files.remove("%s/COLVAR.offset" % self._work_dir) - offset_files.remove("%s/HILLS.offset" % self._work_dir) - except: - pass + def _tail(self, filename): + """ + Return the lines appended to a file since it was last read. + + Parameters + ---------- + + filename : str + The path to the file. + + Returns + ------- + + lines : iterable + An iterable over the new lines. + """ + from .._Utils import Tail as _Tail + + if filename not in self._tails: + self._tails[filename] = _Tail(filename) - for file in offset_files: - _os.remove(file) + return self._tails[filename] def _getPlumedConfig(self): """ @@ -1064,7 +1068,7 @@ def stdout(self, n=10): raise ValueError("The number of lines must be positive!") # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) # Get the current number of lines. @@ -1096,7 +1100,7 @@ def stderr(self, n=10): raise ValueError("The number of lines must be positive!") # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stderr_file): + for line in self._tail(self._stderr_file): self._stderr.append(line.rstrip()) # Get the current number of lines. @@ -1172,7 +1176,7 @@ def getStdout(self, block="AUTO"): self.wait() # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stdout_file): + for line in self._tail(self._stdout_file): self._stdout.append(line.rstrip()) return self._stdout.copy() @@ -1201,7 +1205,7 @@ def getStderr(self, block="AUTO"): self.wait() # Append any new lines to the stdout list. - for line in _pygtail.Pygtail(self._stderr_file): + for line in self._tail(self._stderr_file): self._stderr.append(line.rstrip()) return self._stderr.copy() diff --git a/src/BioSimSpace/Sandpit/Exscientia/Process/_somd.py b/src/BioSimSpace/Sandpit/Exscientia/Process/_somd.py index 34b8456e0..36c384696 100644 --- a/src/BioSimSpace/Sandpit/Exscientia/Process/_somd.py +++ b/src/BioSimSpace/Sandpit/Exscientia/Process/_somd.py @@ -27,9 +27,6 @@ __all__ = ["Somd"] -from .._Utils import _try_import - -_pygtail = _try_import("pygtail") import string as _string from . import _process @@ -868,7 +865,7 @@ def getGradient(self, time_series=False, block="AUTO"): return None # Append any new lines to the gradients list. - for line in _pygtail.Pygtail(self._gradient_file): + for line in self._tail(self._gradient_file): # Ignore comments. if line[0] != "#": self._gradients.append(float(line.rstrip().split()[-1])) diff --git a/src/BioSimSpace/Sandpit/Exscientia/_Utils/__init__.py b/src/BioSimSpace/Sandpit/Exscientia/_Utils/__init__.py index c09b89d72..0b0a5221b 100644 --- a/src/BioSimSpace/Sandpit/Exscientia/_Utils/__init__.py +++ b/src/BioSimSpace/Sandpit/Exscientia/_Utils/__init__.py @@ -28,6 +28,7 @@ .. autosummary:: :toctree: generated/ + Tail WorkDir Context managers @@ -54,4 +55,5 @@ from ._command_split import * from ._contextmanagers import * from ._module_stub import * +from ._tail import * from ._workdir import * diff --git a/src/BioSimSpace/Sandpit/Exscientia/_Utils/_tail.py b/src/BioSimSpace/Sandpit/Exscientia/_Utils/_tail.py new file mode 100644 index 000000000..729c87e12 --- /dev/null +++ b/src/BioSimSpace/Sandpit/Exscientia/_Utils/_tail.py @@ -0,0 +1,82 @@ +###################################################################### +# BioSimSpace: Making biomolecular simulation a breeze! +# +# Copyright: 2017-2025 +# +# Authors: Lester Hedges +# +# BioSimSpace is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# BioSimSpace is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with BioSimSpace. If not, see . +##################################################################### + +"""Incremental reading of files that are being appended to.""" + +__author__ = "Lester Hedges" +__email__ = "lester.hedges@gmail.com" + +__all__ = ["Tail"] + + +class Tail: + """ + Read a file incrementally, yielding only the lines appended since the + previous read. Only complete (newline-terminated) lines are returned; + a partial trailing line is left for the next read. + """ + + def __init__(self, filename): + """ + Constructor. + + Parameters + ---------- + + filename : str + The path to the file. + """ + if not isinstance(filename, str): + raise TypeError("'filename' must be of type 'str'") + + self._filename = filename + self._offset = 0 + self._inode = None + + def __iter__(self): + import os as _os + + try: + stat = _os.stat(self._filename) + except FileNotFoundError: + return + + # Start again if the file has been replaced or truncated. + if stat.st_ino != self._inode or stat.st_size < self._offset: + self._inode = stat.st_ino + self._offset = 0 + + with open(self._filename, "rb") as file: + file.seek(self._offset) + + while True: + line = file.readline() + + if not line.endswith(b"\n"): + break + + self._offset = file.tell() + + # Normalise Windows line endings. + if line.endswith(b"\r\n"): + line = line[:-2] + b"\n" + + yield line.decode(errors="replace") diff --git a/src/BioSimSpace/_Utils/__init__.py b/src/BioSimSpace/_Utils/__init__.py index c09b89d72..0b0a5221b 100644 --- a/src/BioSimSpace/_Utils/__init__.py +++ b/src/BioSimSpace/_Utils/__init__.py @@ -28,6 +28,7 @@ .. autosummary:: :toctree: generated/ + Tail WorkDir Context managers @@ -54,4 +55,5 @@ from ._command_split import * from ._contextmanagers import * from ._module_stub import * +from ._tail import * from ._workdir import * diff --git a/src/BioSimSpace/_Utils/_tail.py b/src/BioSimSpace/_Utils/_tail.py new file mode 100644 index 000000000..729c87e12 --- /dev/null +++ b/src/BioSimSpace/_Utils/_tail.py @@ -0,0 +1,82 @@ +###################################################################### +# BioSimSpace: Making biomolecular simulation a breeze! +# +# Copyright: 2017-2025 +# +# Authors: Lester Hedges +# +# BioSimSpace is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# BioSimSpace is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with BioSimSpace. If not, see . +##################################################################### + +"""Incremental reading of files that are being appended to.""" + +__author__ = "Lester Hedges" +__email__ = "lester.hedges@gmail.com" + +__all__ = ["Tail"] + + +class Tail: + """ + Read a file incrementally, yielding only the lines appended since the + previous read. Only complete (newline-terminated) lines are returned; + a partial trailing line is left for the next read. + """ + + def __init__(self, filename): + """ + Constructor. + + Parameters + ---------- + + filename : str + The path to the file. + """ + if not isinstance(filename, str): + raise TypeError("'filename' must be of type 'str'") + + self._filename = filename + self._offset = 0 + self._inode = None + + def __iter__(self): + import os as _os + + try: + stat = _os.stat(self._filename) + except FileNotFoundError: + return + + # Start again if the file has been replaced or truncated. + if stat.st_ino != self._inode or stat.st_size < self._offset: + self._inode = stat.st_ino + self._offset = 0 + + with open(self._filename, "rb") as file: + file.seek(self._offset) + + while True: + line = file.readline() + + if not line.endswith(b"\n"): + break + + self._offset = file.tell() + + # Normalise Windows line endings. + if line.endswith(b"\r\n"): + line = line[:-2] + b"\n" + + yield line.decode(errors="replace") diff --git a/tests/_Utils/test_tail.py b/tests/_Utils/test_tail.py new file mode 100644 index 000000000..b411ae9e8 --- /dev/null +++ b/tests/_Utils/test_tail.py @@ -0,0 +1,93 @@ +import os + +from BioSimSpace._Utils import Tail + + +def test_tail_incremental(tmp_path): + """Only lines appended since the previous read are returned.""" + path = str(tmp_path / "log") + + with open(path, "w") as f: + f.write("a\nb\n") + + tail = Tail(path) + assert list(tail) == ["a\n", "b\n"] + assert list(tail) == [] + + with open(path, "a") as f: + f.write("c\n") + + assert list(tail) == ["c\n"] + + +def test_tail_partial_line(tmp_path): + """A partial trailing line is held back until it is complete.""" + path = str(tmp_path / "log") + + with open(path, "w") as f: + f.write("a\nb") + + tail = Tail(path) + assert list(tail) == ["a\n"] + + with open(path, "a") as f: + f.write("c\n") + + assert list(tail) == ["bc\n"] + + +def test_tail_truncated(tmp_path): + """Reading restarts from the beginning if the file shrinks.""" + path = str(tmp_path / "log") + + with open(path, "w") as f: + f.write("a\nb\nc\n") + + tail = Tail(path) + assert len(list(tail)) == 3 + + with open(path, "w") as f: + f.write("d\n") + + assert list(tail) == ["d\n"] + + +def test_tail_replaced(tmp_path): + """Reading restarts from the beginning if the file is replaced.""" + path = str(tmp_path / "log") + + with open(path, "w") as f: + f.write("a\nb\n") + + tail = Tail(path) + assert len(list(tail)) == 2 + + # Rename over the original so the new file has a different inode. + with open(path + ".new", "w") as f: + f.write("c\nd\n") + os.replace(path + ".new", path) + + assert list(tail) == ["c\n", "d\n"] + + +def test_tail_missing(tmp_path): + """A missing file yields nothing and is picked up once created.""" + path = str(tmp_path / "log") + + tail = Tail(path) + assert list(tail) == [] + + with open(path, "w") as f: + f.write("a\n") + + assert list(tail) == ["a\n"] + + +def test_tail_crlf(tmp_path): + """Windows line endings are normalised.""" + path = str(tmp_path / "log") + + with open(path, "wb") as f: + f.write(b"a\r\nb\r\n") + + assert list(Tail(path)) == ["a\n", "b\n"]