From 6cb83a1fc65aa979711c0d2ba20718bcebd93e98 Mon Sep 17 00:00:00 2001 From: Mark Teese Date: Mon, 31 Aug 2026 12:22:27 +0200 Subject: [PATCH 1/6] Fix input file path handling, overwrite flag and figure legend run_weighslide read the file extension via infile.name, which only works when a Path is passed. The command line passed a normalised string, so `weighslide -i ` raised AttributeError before reading any data, as did the string-path form documented in the README. The path is now converted to a Path on entry, so both call styles behave the same. The command line parsed -o/--overwrite but never passed it on, so a second run against the same input raised FileExistsError whatever the flag was set to. The plotted lines carried no labels, so the legend was drawn empty and matplotlib warned on every run. Slicing indexed the padded series by label while treating the index value as a position, which produced wrong slices for any input whose index was not a zero-based range. Slicing is now positional throughout, and the caller's index is restored on the returned series. Also moves the argument parser out of module scope, where it was built on every import, into build_parser() and main(); replaces the inplace replace() and positional Series assignment that are deprecated in pandas 2 and later; and collects the slice frames in lists rather than concatenating inside the loop, which was quadratic in input length. --- weighslide/__init__.py | 3 +- weighslide/weighslide.py | 481 ++++++++++++++++++++------------------- 2 files changed, 253 insertions(+), 231 deletions(-) diff --git a/weighslide/__init__.py b/weighslide/__init__.py index e6b7f08..4469deb 100644 --- a/weighslide/__init__.py +++ b/weighslide/__init__.py @@ -1,2 +1 @@ -from weighslide.weighslide import run_weighslide -from weighslide.weighslide import calculate_weighted_windows +from weighslide.weighslide import calculate_weighted_windows, run_weighslide diff --git a/weighslide/weighslide.py b/weighslide/weighslide.py index 430e2f3..f5508b8 100644 --- a/weighslide/weighslide.py +++ b/weighslide/weighslide.py @@ -1,17 +1,21 @@ -from pathlib import Path -from typing import Union +# Defers evaluation of the annotations below, so that `Path | str` (PEP 604 syntax, +# which is only valid at runtime on python 3.10+) does not raise on older versions. +from __future__ import annotations -import numpy as np -import pandas as pd -import os -import matplotlib.pyplot as plt import argparse import ast import sys +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + +STATISTICS = ("mean", "std", "sum") -def run_weighslide(infile: Union[Path, str], window: Union[list, str], statistic: str, **kwargs): - """ Runs the weighslide algorithm using data from an input file, and saves the output files and figures +def run_weighslide(infile: Path | str, window: list | str, statistic: str, **kwargs): + """Runs the weighslide algorithm using data from an input file, and saves the output files and figures Weighslide takes as an input a 1D array (list) of numerical data, and applies a user-defined weighting and algorithm in a sliding-window fashion across the data. @@ -43,7 +47,7 @@ def run_weighslide(infile: Union[Path, str], window: Union[list, str], statistic Parameters ---------- - infile : string + infile : Path or string Path to csv or excel file containing the data to be analysed. window : list or string The user-defined window that determines the size of the slices in the array, and the weight of each value in @@ -102,22 +106,18 @@ def run_weighslide(infile: Union[Path, str], window: Union[list, str], statistic """ print("Starting weighslide analysis.") + # accept either a string or a Path, so that the command line and the python API behave the same + infile = Path(infile) + filetype = infile.suffix.lstrip(".").lower() + # if the infile ends in .xls or .xlsx, open with excel_kwargs, if available - filetype = str(infile.name).split(".")[-1] - if filetype == "xlsx" or filetype == "xls": - if "excel_kwargs" in kwargs.keys(): - df = pd.read_excel(infile, **kwargs["excel_kwargs"]) - else: - df = pd.read_excel(infile) + if filetype in ("xlsx", "xls"): + excel_kwargs = kwargs.get("excel_kwargs") or {} + df = pd.read_excel(infile, **excel_kwargs) # if the infile ends in .csv, open with csv_kwargs, if available elif filetype == "csv": - if "csv_kwargs" in kwargs: - if kwargs["csv_kwargs"] is not None: - df = pd.read_csv(infile, **kwargs["csv_kwargs"]) - else: - df = pd.read_csv(infile) - else: - df = pd.read_csv(infile) + csv_kwargs = kwargs.get("csv_kwargs") or {} + df = pd.read_csv(infile, **csv_kwargs) else: raise ValueError("Filetype must be excel or csv, and have an .xlsx, .xls, or .csv extension.") @@ -126,62 +126,66 @@ def run_weighslide(infile: Union[Path, str], window: Union[list, str], statistic data_series = df.iloc[:, 0] # if the dataframe has multiple columns, search in the kwargs for the appropriate column name for input data elif df.shape[1] > 1: - if "column" in kwargs.keys() and kwargs["column"] is not None: + if kwargs.get("column") is not None: column = kwargs["column"] + if column not in df.columns: + raise ValueError( + f'Column "{column}" is not in the input file "{infile}". ' + f"The available columns are {list(df.columns)}." + ) # select data column data_series = df[column] else: - raise ValueError(r'No column name provided. The input file "{}" appears to have multiple columns, and ' - r'therefore the column name with data needs to be input as a column ' - r'variable.'.format(infile)) + raise ValueError( + f'No column name provided. The input file "{infile}" appears to have multiple columns, and ' + f"therefore the column name with data needs to be input as a column variable." + ) else: raise ValueError(f"Input data not found. Imported {filetype} file '{infile}' has {df.shape[1]} columns") # get the path of the input file - split_input_filepath = os.path.split(infile) - inpath = split_input_filepath[0] + inpath = infile.parent # get the sample/experiment name from input variables, otherwise use the first 20 characters of filename - if "name" in kwargs.keys(): + if "name" in kwargs: out_name = kwargs["name"][:20] else: - out_name = split_input_filepath[1][:20] + out_name = infile.name[:20] # if the window is a string, use first 20 characters in output filenames - if type(window) == str: + if isinstance(window, str): window_str = window[:20] else: # the list of weightings is probably not suitable to include in a filename. Use an empty string. window_str = "" - # create a base name for the output files. Greate directory. Create output paths for excel, csv and png files. - outpath = os.path.join(inpath, "weighslide_output") - if not os.path.isdir(outpath): - os.mkdir(outpath) - out_basename = os.path.join(outpath, out_name + window_str) - out_excelfile = out_basename + ".xlsx" - out_csv_slice = out_basename + "_sliced" + ".csv" - out_csv_mult = out_basename + "_mult" + ".csv" - out_csv_statistic = out_basename + "_{}".format(statistic) + ".csv" - out_png = out_basename + ".png" + # create a base name for the output files. Create directory. Create output paths for excel, csv and png files. + outpath = inpath / "weighslide_output" + outpath.mkdir(parents=True, exist_ok=True) + out_basename = outpath / (out_name + window_str) + out_excelfile = out_basename.with_name(out_basename.name + ".xlsx") + out_csv_slice = out_basename.with_name(out_basename.name + "_sliced.csv") + out_csv_mult = out_basename.with_name(out_basename.name + "_mult.csv") + out_csv_statistic = out_basename.with_name(f"{out_basename.name}_{statistic}.csv") + out_png = out_basename.with_name(out_basename.name + ".png") # determine the user variable "overwrite" - if "overwrite" in kwargs.keys(): - overwrite = kwargs["overwrite"] - else: - overwrite = False + overwrite = kwargs.get("overwrite", False) # check if output files exist. Raise error if they exist, and "overwrite" is not True list_check_if_existing = [out_excelfile, out_csv_slice, out_csv_mult, out_csv_statistic, out_png] - for filepath in list_check_if_existing: - if os.path.exists(filepath): - if not overwrite: - raise FileExistsError('\nOutput files already exist. To overwrite files, please change the' - ' "overwrite" variable to True.') + if not overwrite: + for filepath in list_check_if_existing: + if filepath.exists(): + raise FileExistsError( + "\nOutput files already exist. To overwrite files, please change the" + ' "overwrite" variable to True.' + ) # run the algorithm to calculate the weighted windows - window_array, df_orig_sliced, df_multiplied, output_series = calculate_weighted_windows(data_series, - window, statistic) + window_array, df_orig_sliced, df_multiplied, output_series = calculate_weighted_windows( + data_series, window, statistic + ) # save output files to csv df_orig_sliced.to_csv(out_csv_slice) @@ -196,7 +200,7 @@ def run_weighslide(infile: Union[Path, str], window: Union[list, str], statistic with pd.ExcelWriter(out_excelfile) as writer: df_orig_sliced.to_excel(writer, sheet_name="orig_data_sliced") df_multiplied.to_excel(writer, sheet_name="data_multipled") - output_series.to_frame(name="window_{}".format(statistic)).to_excel(writer, sheet_name="window_{}".format(statistic)) + output_series.to_frame(name=f"window_{statistic}").to_excel(writer, sheet_name=f"window_{statistic}") # print dot showing progress sys.stdout.write(".") @@ -209,30 +213,53 @@ def run_weighslide(infile: Union[Path, str], window: Union[list, str], statistic ############################################################ fig, ax = plt.subplots() - ax.plot(data_series) - ax.plot(output_series) + # the labels are what ax.legend() below picks up; without them the legend is empty + ax.plot(data_series.index, data_series.to_numpy(), label="original data") + ax.plot(output_series.index, output_series.to_numpy(), label=str(output_series.name)) ax.set_xlabel("position") ax.set_ylabel("value") window_string = str(window) dots = "..." if len(window_string) > 20 else "" - ax.set_title("weighslide output for window {}{}".format(window_string[:20], dots)) + ax.set_title(f"weighslide output for window {window_string[:20]}{dots}") max_value = max(data_series.max(), output_series.max()) min_value = min(data_series.min(), output_series.min()) ax.set_ylim(min_value * 0.8, max_value * 1.2) - lgd = ax.legend() + ax.legend() plt.tight_layout() - fig.savefig(out_png, format='png', dpi=200) - if "showfig" in kwargs.keys(): - if kwargs["showfig"] == True: - plt.show() + fig.savefig(out_png, format="png", dpi=200) + if kwargs.get("showfig"): + plt.show() + # close explicitly, otherwise repeated calls accumulate open figures until matplotlib warns + plt.close(fig) print("\nWeighslide analysis is finished.") - if len(inpath) > 1: - print("\nLocation of output files:\n\t{}".format(outpath)) + print(f"\nLocation of output files:\n\t{outpath}") + + +def _window_to_array(window: list | str) -> np.ndarray: + """Convert a user-supplied window into a 1D float array, with "x" positions as np.nan. + + A string window uses a 0-9 shorthand for the weighting, which is shifted and scaled onto + 0.1-1.0. A list window is taken as literal weights. + """ + if isinstance(window, str): + # split into a list of single characters, e.g. "4x4" -> ["4", "x", "4"] + window_series = pd.Series(list(window)) + # replace x with np.nan, then change dtype to float + window_series = window_series.replace("x", np.nan).astype(float) + # convert 0-9 scale to 1-10, divide by 10 to give a relative weighting + window_series = (window_series + 1) / 10 + elif isinstance(window, list): + # replace x with np.nan + window_series = pd.Series(window, dtype=object).replace("x", np.nan).astype(float) + else: + raise TypeError("The input variable 'window' is neither a string nor a list.") + + return np.asarray(window_series, dtype=float) def calculate_weighted_windows(data_series, window, statistic, full_output=True): - """ Apply the weighslide algorithm to an input series. + """Apply the weighslide algorithm to an input series. Parameters ---------- @@ -262,193 +289,191 @@ def calculate_weighted_windows(data_series, window, statistic, full_output=True) Pandas Series containing the output data. This is the result after slicing, applying the window, and applying a statistic (mean, std or sum). The series indexb is the range of the original data. The dtype is float. """ + if statistic not in STATISTICS: + raise ValueError( + "The 'statistic' variable is not recognised. \nPlease check that the variable " + "is either 'mean', 'std', or 'sum'." + ) - data_series.name = "original data" + data_series = pd.Series(data_series).rename("original data") - if type(window) == str: - # determine length of the window from the user input window - window_length = len(window) - # split into a list, convert to float, divide by 10 to yield a proportion - window_series: pd.Series = pd.Series(list(window)) - # replace x with np.nan - window_series.replace("x", np.nan, inplace=True) - # change dtype to float - window_series: pd.Series = window_series.astype(float) - # convert 0-9 scale to 1-10, divide by 10 to give a relative weighting - window_series = (window_series + 1) / 10 - # convert the series to a numpy array - window_array = np.array(window_series).astype(float) - - elif type(window) == list: - window_length = len(window) - # convert the list or series to a numpy array - window_series = pd.Series(window) - # replace x with np.nan - window_series.replace("x", np.nan, inplace=True) - # convert the series to a numpy array - window_array = np.array(window_series).astype(float) - - else: - raise TypeError("The input variable 'window' is neither a string nor a list.") + window_array = _window_to_array(window) + window_length = len(window_array) if window_length == 0: raise ValueError("Window length is 0. Please check the 'window' input variable.") elif window_length % 2 == 0: - raise ValueError("Window length ({}) is even. Please check the window input variable. Only odd-length windows " - "are accepted, so that the result of the sliding " - "window analysis centres around a single non-ambiguous original position.".format( - window_length)) + raise ValueError( + f"Window length ({window_length}) is even. Please check the window input variable. Only odd-length " + "windows are accepted, so that the result of the sliding window analysis centres around a single " + "non-ambiguous original position." + ) elif window_length > 100: - raise ValueError("Window length ({}) is too long. Weighslide has not been optimised for large windows " - "or datasets. To run code anyway, convert this elif statement to a comment.".format(window_length)) - - # count the number of positions on either side of the central position - extension_each_side = int((window_length - 1) / 2) - # extend the original series with zeroes on either side - # define positive and negative ranges - neg_range = list(range(-extension_each_side, 0)) - pos_range = list(np.arange(1, 6) + len(data_series) - 1) - # create the index (e.g. -5,-4,-3,-2,-1,0,ORIG DATA, end+1, end+2...end+5) - s_index = neg_range + list(data_series.index) + pos_range - # reindex the series so that it is padded with np.nan on either side, as in the s_index - extend_series = data_series.reindex(s_index) - - # create output series for the final window-averaged data - output_series = pd.Series(index=data_series.index, dtype=float) - # create output dataframes for saving all the slices etc. - df_orig_sliced = pd.DataFrame() - df_multiplied = pd.DataFrame() + raise ValueError( + f"Window length ({window_length}) is too long. Weighslide has not been optimised for large windows " + "or datasets. To run code anyway, convert this elif statement to a comment." + ) # get length of data series data_series_len = len(data_series) # show a warning if the data series is quite long if data_series_len > 1000: - print("Warning. Input data length is {}. Weighslide performance may be slow.".format(data_series_len)) + print(f"Warning. Input data length is {data_series_len}. Weighslide performance may be slow.") if data_series_len > 10000: # abort if data series is very long. - raise ValueError("Program aborted due to long input sequence. " - "Weighslide performance has not been optimised for large datasets. " - "To run code anyway, convert this 'if statement' to a comment".format(data_series_len)) + raise ValueError( + f"Program aborted due to long input sequence (length {data_series_len}). " + "Weighslide performance has not been optimised for large datasets. " + "To run code anyway, convert this 'if statement' to a comment" + ) + + # the slicing below indexes the data by position, so an arbitrary input index (e.g. a + # gene name, or a non-zero-based range) would silently produce the wrong slices. + # Work on a 0..n-1 index throughout, and restore the caller's index on the output. + orig_index = data_series.index + data_series = data_series.reset_index(drop=True) + + # count the number of positions on either side of the central position + extension_each_side = (window_length - 1) // 2 + # extend the original series with np.nan on either side, so that the windows centred on the + # first and last positions still have a full-length slice to work with + neg_range = list(range(-extension_each_side, 0)) + pos_range = list(range(data_series_len, data_series_len + extension_each_side)) + # create the index (e.g. -5,-4,-3,-2,-1,0,ORIG DATA, end+1, end+2...end+5) + s_index = neg_range + list(data_series.index) + pos_range + # reindex the series so that it is padded with np.nan on either side, as in the s_index + extend_series = data_series.reindex(s_index) + + # collect the slices in lists and concatenate once. Concatenating inside the loop copies the + # whole growing frame on every iteration, which is quadratic in the length of the input. + orig_slices = [] + multiplied_slices = [] + statistic_values = np.empty(data_series_len, dtype=float) - # for i in extend_series.index[:len(extend_series)-extension_each_side-1]: for i in range(data_series_len): - if data_series_len > 100: - if i % 100 == 0: - sys.stdout.write(".") - sys.stdout.flush() - start = extend_series.index[i] - end = start + window_length - data_range = list(range(start, end)) + if data_series_len > 100 and i % 100 == 0: + sys.stdout.write(".") + sys.stdout.flush() + start = i - extension_each_side + data_range = list(range(start, start + window_length)) # slice out the original data from the window (e.g. 11 residues) - # orig_sliced = extend_series.loc[i:i+window_length-1] - # orig_sliced = extend_series[extend_series.index.isin(data_range)] orig_sliced = extend_series.reindex(data_range) - orig_sliced.name = "window {}".format(i) - orig_sliced_for_excel = orig_sliced.fillna("nodata") - df_orig_sliced = pd.concat([df_orig_sliced, orig_sliced_for_excel], axis=1, names=[start]) + orig_sliced.name = f"window {i}" + orig_slices.append(orig_sliced.fillna("nodata")) # double-check that the sliced window, and the values have the same length - assert (len(orig_sliced) == len(window_array)) + assert len(orig_sliced) == len(window_array) # multiply by the window value multiplier for each position win_multiplied = orig_sliced * window_array - win_multiplied_for_excel = win_multiplied.fillna("") - df_multiplied = pd.concat([df_multiplied, win_multiplied_for_excel], axis=1, names=start) + multiplied_slices.append(win_multiplied.fillna("")) if statistic == "mean": # calculate the mean of the values, representing the relative value of that window - win_mult_statistic = win_multiplied.mean() + statistic_values[i] = win_multiplied.mean() elif statistic == "std": # calculate the standard deviation of the values, representing the relative value of that window - win_mult_statistic = win_multiplied.std() - elif statistic == "sum": - # calculate the standard deviation of the values, representing the relative value of that window - win_mult_statistic = win_multiplied.sum() + statistic_values[i] = win_multiplied.std() else: - raise ValueError("The 'statistic' variable is not recognised. \nPlease check that the variable " - "is either 'mean', 'std', or 'sum'.") - # add to output series - output_series[i] = win_mult_statistic + # calculate the sum of the values, representing the relative value of that window + statistic_values[i] = win_multiplied.sum() + + df_orig_sliced = pd.concat(orig_slices, axis=1) + df_multiplied = pd.concat(multiplied_slices, axis=1) + output_series = pd.Series(statistic_values, index=orig_index, dtype=float) output_series.index.name = "position" - output_series.name = "{} over window".format(statistic) + output_series.name = f"{statistic} over window" - if full_output == True: + if full_output: return window_array, df_orig_sliced, df_multiplied, output_series else: return output_series -# create a parser object to read user inputs from the command line -parser = argparse.ArgumentParser() - -# add command-line options -parser.add_argument("w", # "--window", - help="Sliding weighted window. Can be either a python list " - "(e.g. [0.3,1.0,0.3,0,0.3,1.0,0.3,0,0.3,1.0,0.3]), or a list of numbers that will be " - "converted to a python list (e.g. 393x393x393), where x represents positions that are ignored" - "and 9 represents positions that are most highly weighted.") -parser.add_argument("s", # "--statistic", - default="mean", - type=str, choices=["mean", "std", "sum"], - help="The choices are mean, std or sum. Desired method to reduce the weighted values in the to a " - "single value at the central position.") -parser.add_argument("-r", # "--rawdata", - default=None, - help='Raw data input in the command line. Should be a python list of integers (e.g. "[1,3,5,7,2,4]")' - ' or floats (e.g. "[1.1,3.4,5.2,7.8,2.7,4.5]")') -parser.add_argument("-i", # "-infile", - default=None, - help=r'Full path of file containing original data in csv or excel format.' - r'E.g. "C:/Path/to/your/file.xlsx"') -parser.add_argument("-n", # "--name", - default="", - help="Name of dataset. Should not be longer than 20 characters. Used in output filenames.") -parser.add_argument("-c", # "--column", - default=None, - help='Column name in input file that should be used for analysis. E.g. "data values"') -parser.add_argument("-o", # "--overwrite", - type=str, default="False", - help='If True, existing files will be overwritten.') -parser.add_argument("-e", # "--excel_kwargs", - default="None", - help="Keyword arguments in python dictionary format to be used when opening " - "an excel file using the python pandas module. (E.g. {'sheet_name':'orig_data'}") -parser.add_argument("-k", # "--csv_kwargs", - default=None, - help="Keyword arguments in python dictionary format to be used when opening " - "your csv file using the python pandas module. (E.g. {'delimeter':',','header'='infer'}") - -# if weighslide.py is run as the main python script, obtain the options from the command line. -if __name__ == '__main__': - - print("\nTo view the help:\npython weighslide.py -h\n\nTo test the weighslide module:\n" - "python weighslide.py [0.5,1.0,0.5] mean -r [1,3,5,7,2,4,3,5,7,2,4]\n\n") +def build_parser() -> argparse.ArgumentParser: + """Build the command-line parser for the weighslide console script.""" + parser = argparse.ArgumentParser( + prog="weighslide", + description="Sliding window analysis of a list of numerical values, using flexible user-defined windows.", + ) + + # add command-line options + parser.add_argument( + "w", # "--window", + help="Sliding weighted window. Can be either a python list " + "(e.g. [0.3,1.0,0.3,0,0.3,1.0,0.3,0,0.3,1.0,0.3]), or a list of numbers that will be " + "converted to a python list (e.g. 393x393x393), where x represents positions that are ignored" + "and 9 represents positions that are most highly weighted.", + ) + parser.add_argument( + "s", # "--statistic", + default="mean", + type=str, + choices=list(STATISTICS), + help="The choices are mean, std or sum. Desired method to reduce the weighted values in the to a " + "single value at the central position.", + ) + parser.add_argument( + "-r", # "--rawdata", + default=None, + help='Raw data input in the command line. Should be a python list of integers (e.g. "[1,3,5,7,2,4]")' + ' or floats (e.g. "[1.1,3.4,5.2,7.8,2.7,4.5]")', + ) + parser.add_argument( + "-i", # "-infile", + default=None, + help=r"Full path of file containing original data in csv or excel format." r'E.g. "C:/Path/to/your/file.xlsx"', + ) + parser.add_argument( + "-n", # "--name", + default="", + help="Name of dataset. Should not be longer than 20 characters. Used in output filenames.", + ) + parser.add_argument( + "-c", # "--column", + default=None, + help='Column name in input file that should be used for analysis. E.g. "data values"', + ) + parser.add_argument( + "-o", # "--overwrite", + type=str, + default="False", + choices=["True", "true", "TRUE", "False", "false", "FALSE"], + help="If True, existing files will be overwritten.", + ) + parser.add_argument( + "-e", # "--excel_kwargs", + default="None", + help="Keyword arguments in python dictionary format to be used when opening " + "an excel file using the python pandas module. (E.g. {'sheet_name':'orig_data'}", + ) + parser.add_argument( + "-k", # "--csv_kwargs", + default=None, + help="Keyword arguments in python dictionary format to be used when opening " + "your csv file using the python pandas module. (E.g. {'delimeter':',','header'='infer'}", + ) + return parser + + +def main(argv=None): + """Entry point for the ``weighslide`` console script.""" + parser = build_parser() # obtain command-line arguments - args = parser.parse_args() - - print(args) + args = parser.parse_args(argv) # extract the boolean "overwrite" variable from the input arguments - if "o" in args: - print(args.o) - if args.o in ["True", "true", "TRUE"]: - overwrite = True - elif args.o in ["False", "false", "FALSE"]: - overwrite = False - else: - raise ValueError('Overwrite variable "{}" is not recognised. ' - 'Accepted values are "True" or "False".'.format(args.o)) - else: - overwrite = False + overwrite = args.o in ("True", "true", "TRUE") # check that the user has not input both an infile and a raw data list - if "r" in args: - if all([args.i, args.r]): - raise ValueError("Both an input file and a raw data string are entered. Please input only one data format.") + if args.i and args.r: + raise ValueError("Both an input file and a raw data string are entered. Please input only one data format.") + if not args.i and not args.r: + parser.error("No data given. Provide either an input file with -i, or a raw data list with -r.") # if the window looks like a python list (i.e., it starts with "["), convert it from stringlist to list - if args.w[0] == "[": + if args.w.startswith("["): window = ast.literal_eval(args.w) else: window = args.w @@ -456,33 +481,31 @@ def calculate_weighted_windows(data_series, window, statistic, full_output=True) statistic = args.s if args.i is not None: - # normalise the path of the infile to suit the operating system - infile = os.path.normpath(args.i) - # extract the column name from the command-line input - column = args.c - # extract the sample/data name from the command-line input - name = args.n # extract the excel_kwargs from the command-line input excel_kwargs = ast.literal_eval(args.e) # extract the csv_kwargs from the command-line input - if args.k is not None: - csv_kwargs = ast.literal_eval(args.k) - else: - csv_kwargs = None + csv_kwargs = ast.literal_eval(args.k) if args.k is not None else None # run weighslide - run_weighslide(infile=infile, window=window, statistic=statistic, column=column, - name=name, excel_kwargs=excel_kwargs, csv_kwargs=csv_kwargs) - - elif args.r is not None: - # extract the csv_kwargs from the command-line input - raw_data_user_input = args.r - # convert the stringlist to a python list - raw_data_list = ast.literal_eval(raw_data_user_input) - # convert the python list to a pandas Series - raw_data_series = pd.Series(raw_data_list) + run_weighslide( + infile=Path(args.i), + window=window, + statistic=statistic, + column=args.c, + name=args.n, + overwrite=overwrite, + excel_kwargs=excel_kwargs, + csv_kwargs=csv_kwargs, + ) + else: + # convert the stringlist to a python list, then to a pandas Series + raw_data_series = pd.Series(ast.literal_eval(args.r)) # run the calculate_weighted_windows function - window_array, df_orig_sliced, df_multiplied, output_series = calculate_weighted_windows(raw_data_series, - window, statistic) + output_series = calculate_weighted_windows(raw_data_series, window, statistic, full_output=False) print("\nWeighslide output:") # print out the values from the output series print(output_series.to_string(index=False, header=False)) + + +# if weighslide.py is run as the main python script, obtain the options from the command line. +if __name__ == "__main__": + main() From 702fa093e2db510acb89b41536049a67018d015a Mon Sep 17 00:00:00 2001 From: Mark Teese Date: Mon, 31 Aug 2026 12:22:42 +0200 Subject: [PATCH 2/6] Replace setup.py with pyproject.toml Declares pytest as an optional dev dependency instead of a runtime one. It was listed in install_requires, so installing weighslide pulled a test framework and its dependencies into the user's environment. Adds openpyxl as a runtime dependency. run_weighslide always writes an xlsx output file, which pandas delegates to openpyxl, but pandas does not install openpyxl itself. Adds a weighslide console script. The previous instruction to run `python weighslide.py` does not work for an installed package. The dependency lower bounds are the oldest versions the test suite has been run against rather than estimates. requires-python is set to 3.9, the oldest version the code runs on; declaring it lets pip offer older interpreters the previous release instead of failing on install. Version 0.2.5 to 0.3.0, for the changed dependency contract and the new console script. --- MANIFEST.in | 4 +-- pyproject.toml | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 4 --- setup.py | 33 ------------------- 4 files changed, 89 insertions(+), 40 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in index 96d3701..cb9f536 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,5 @@ -include MANIFEST.in include LICENSE.txt -include setup.cfg include README.md include examples/input.png include examples/output.png -include examples/contact.png \ No newline at end of file +include examples/contact.png diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dfb3b04 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,88 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "weighslide" +version = "0.3.0" +description = "Flexible sliding window analysis" +readme = { file = "README.md", content-type = "text/markdown" } +requires-python = ">=3.9" +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [{ name = "Mark Teese" }] +keywords = ["sliding", "data", "normalisation", "normalization", "array"] +classifiers = [ + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Bio-Informatics", +] +# Lower bounds only, deliberately. weighslide is a library that gets installed +# alongside a user's own stack, so exact pins here would make it uninstallable +# rather than reproducible. The bounds below are the oldest versions the test +# suite has actually been run against (see README), not guesses. CI pins nothing. +dependencies = [ + "pandas>=1.2", + "numpy>=1.20", + "matplotlib>=3.4", + # pandas delegates .xlsx reading and writing to openpyxl, which it does not + # install itself. run_weighslide always writes an .xlsx output file. + "openpyxl>=3.0", +] + +[project.optional-dependencies] +dev = [ + "pytest", + "pre-commit", +] + +[project.scripts] +weighslide = "weighslide.weighslide:main" + +[project.urls] +Homepage = "https://github.com/teese/weighslide" +Repository = "https://github.com/teese/weighslide" + +[tool.setuptools.packages.find] +include = ["weighslide*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +filterwarnings = [ + # Deprecations raised from weighslide's own code must fail the suite rather than + # accumulate silently, as they did over the years this package went untouched. + "error::FutureWarning:weighslide.*", + "error::DeprecationWarning:weighslide.*", +] + +[tool.black] +line-length = 120 +target-version = ["py39"] + +[tool.ruff] +target-version = "py39" +line-length = 120 +src = ["weighslide", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B"] +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +# weighslide/__init__.py re-exports the two public functions. The README documents +# `weighslide.run_weighslide(...)`, so dropping the re-export would break callers. +"weighslide/__init__.py" = ["F401"] + +[tool.ruff.lint.isort] +combine-as-imports = true +known-first-party = ["weighslide"] + +[tool.mypy] +python_version = "3.9" +ignore_missing_imports = true +allow_untyped_globals = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 35cb743..0000000 --- a/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[metadata] -description_file = README.md -# This includes the license file in the wheel. -license_file = LICENSE.txt \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 4fdad95..0000000 --- a/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -""" - Weighslide is a program for sliding window analysis of a list of numerical values, - using flexible windows determined by the user. -""" -from setuptools import setup, find_packages -from os import path - -here = path.abspath(path.dirname(__file__)) -# Get the long description from the README file -with open(path.join(here, 'README.md'), "r", encoding='utf-8') as f: - long_description = f.read() - -setup(name='weighslide', - packages=find_packages(), - version='0.2.5', - description="Flexible sliding window analysis", - author='Mark Teese', - author_email='mark.teese@SeeImageBelow.de', - long_description=long_description, - long_description_content_type='text/markdown', - url="https://github.com/teese/weighslide", - download_url='https://github.com/teese/weighslide/archive/0.2.5.tar.gz', - project_urls={'LangoschLab': 'http://cbp.wzw.tum.de/index.php?id=9', "TU_Muenchen": "https://www.tum.de"}, - license='MIT', - classifiers=[ - "Intended Audience :: Science/Research", - 'License :: OSI Approved :: MIT License', - "Programming Language :: Python :: 3.6", - "Topic :: Scientific/Engineering :: Bio-Informatics" - ], - install_requires=["pandas", "numpy", "matplotlib", "pytest"], - keywords="sliding data normalisation normalization array" - ) From a6a43f88113b8cb325f475f91828b571166f2f66 Mon Sep 17 00:00:00 2001 From: Mark Teese Date: Mon, 31 Aug 2026 12:23:05 +0200 Subject: [PATCH 3/6] Add pre-commit hooks and CI workflow Runs black, ruff and mypy, at the hook revisions already in use across the other repositories. CI runs the same hooks so that an unformatted commit is caught before merge, then runs the test suite on Linux and Windows across python 3.9, 3.12 and 3.13. That matrix spans both pandas 1.x and pandas 3.x, which behave differently for several of the calls this package makes. --- .github/workflows/ci.yml | 37 +++++++++++++++++++++++++++++++++++++ .gitignore | 6 +++++- .pre-commit-config.yaml | 25 +++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c76076a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + push: + branches: ['**'] + pull_request: + branches: ['**'] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - uses: pre-commit/action@v3.0.1 + + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ['3.9', '3.12', '3.13'] + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install package with test dependencies + run: pip install -e ".[dev]" + - name: Run tests + # Agg keeps matplotlib off a display that the runners do not have + env: + MPLBACKEND: Agg + run: pytest -q diff --git a/.gitignore b/.gitignore index 7ea786e..e702452 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,8 @@ var/ # PyInstaller /weighslide.egg-info/ -*__pycache__* \ No newline at end of file +*__pycache__* +# ruff / mypy / pytest caches +.ruff_cache/ +.mypy_cache/ +.pytest_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7cf92d0 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +# Run `pre-commit install` after cloning. CI runs the same hooks. +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + - id: check-toml + - id: check-merge-conflict + - repo: https://github.com/psf/black + rev: 26.5.1 + hooks: + - id: black + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.22 + hooks: + - id: ruff-check + args: [--fix] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v2.3.0 + hooks: + - id: mypy + args: [--ignore-missing-imports, --allow-untyped-globals] + files: ^weighslide/.*\.py$ From a9d70d89c555f0ccdde9f654299a44826084c442 Mon Sep 17 00:00:00 2001 From: Mark Teese Date: Mon, 31 Aug 2026 12:23:05 +0200 Subject: [PATCH 4/6] Replace the functional test with unit tests The previous test asserted only that four output files had been created, so nothing checked the values the package produces. The new tests cover the worked example carried in the README and the docstring, window parsing including ignored positions, every input validation path, all three statistics, index handling, csv and excel input, overwrite behaviour and the command line interface. The path handling and overwrite defects fixed earlier on this branch have regression tests, and one test asserts that a run raises no deprecation warnings, so that the drift that had accumulated here does not recur unnoticed. --- tests/test_algorithm.py | 130 +++++++++++++++++++++++++++++++++++ tests/test_cli.py | 82 ++++++++++++++++++++++ tests/test_functional.py | 37 ---------- tests/test_run_weighslide.py | 126 +++++++++++++++++++++++++++++++++ 4 files changed, 338 insertions(+), 37 deletions(-) create mode 100644 tests/test_algorithm.py create mode 100644 tests/test_cli.py delete mode 100644 tests/test_functional.py create mode 100644 tests/test_run_weighslide.py diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py new file mode 100644 index 0000000..8aaa3d8 --- /dev/null +++ b/tests/test_algorithm.py @@ -0,0 +1,130 @@ +"""Unit tests for the weighslide algorithm itself (calculate_weighted_windows).""" + +import numpy as np +import pandas as pd +import pytest + +from weighslide.weighslide import _window_to_array, calculate_weighted_windows + +# the worked example from the README and the run_weighslide docstring +DOCSTRING_DATA = [0, 0, 0, 1, 1, 2, 3, 5, 8, 13, 21] +DOCSTRING_EXPECTED = [0.0, 0.0, 0.666667, 2.333333, 3.666667, 6.0, 9.666667, 15.666667, 25.333333, 41.0, 65.5] + + +def test_matches_the_worked_example_in_the_docs(): + """The [2,5,2] mean example is quoted in the README, the docstring and the paper citation.""" + result = calculate_weighted_windows(pd.Series(DOCSTRING_DATA, dtype=float), [2, 5, 2], "mean", full_output=False) + np.testing.assert_allclose(result.to_numpy(), DOCSTRING_EXPECTED, rtol=1e-5) + + +def test_ignored_positions_are_excluded_from_the_statistic(): + """An "x" position must not contribute to the window, not even as a zero.""" + data = pd.Series([0, 0, 0, 1, 1, 2, 3, 5, 8, 13, 21], dtype=float) + result = calculate_weighted_windows(data, [1, 1, "x", 1, 1], "mean", full_output=False) + # position 5 averages the four neighbours (1, 2 upstream; 5, 8 downstream) -> 1+2+5+8 = 16/4 + assert result.iloc[6] == pytest.approx(4.0) + # a window of all-ones with an ignored centre is the mean of the 4 surrounding values + expected = [0.0, 0.333333, 0.5, 0.75, 1.5, 2.5, 4.0, 6.5, 10.5, 11.333333, 10.5] + np.testing.assert_allclose(result.to_numpy(), expected, rtol=1e-5) + + +def test_flanking_windows_are_padded_with_nan_not_zero(): + """The first and last windows are short; the missing positions must be NaN, which is skipped.""" + data = pd.Series([4.0, 4.0, 4.0]) + result = calculate_weighted_windows(data, [1, 1, 1], "mean", full_output=False) + # every window averages only the real values, so a flat input stays flat + np.testing.assert_allclose(result.to_numpy(), [4.0, 4.0, 4.0]) + + +@pytest.mark.parametrize( + ("statistic", "expected"), + [ + ("mean", 2.0), + ("sum", 6.0), + ("std", 0.0), + ], +) +def test_each_statistic_reduces_the_window(statistic, expected): + data = pd.Series([2.0] * 5) + result = calculate_weighted_windows(data, [1, 1, 1], statistic, full_output=False) + # the centre position has a full-length window of three identical values + assert result.iloc[2] == pytest.approx(expected) + + +def test_full_output_shapes(): + data = pd.Series(np.arange(10), dtype=float) + window_array, df_sliced, df_multiplied, result = calculate_weighted_windows(data, [2, 5, 2], "mean") + np.testing.assert_allclose(window_array, [2.0, 5.0, 2.0]) + # one column per input position, in both the sliced and the multiplied frames + assert df_sliced.shape[1] == len(data) + assert df_multiplied.shape[1] == len(data) + assert list(df_sliced.columns) == [f"window {i}" for i in range(len(data))] + assert len(result) == len(data) + assert result.name == "mean over window" + assert result.index.name == "position" + + +def test_output_keeps_the_index_of_the_input(): + """The window is positional, but the result must line up with the caller's own index.""" + data = pd.Series([0, 0, 0, 1, 1, 2, 3, 5, 8, 13, 21], dtype=float, index=range(100, 111)) + result = calculate_weighted_windows(data, [2, 5, 2], "mean", full_output=False) + assert list(result.index) == list(range(100, 111)) + # a shifted index must not shift the values + np.testing.assert_allclose(result.to_numpy(), DOCSTRING_EXPECTED, rtol=1e-5) + + +def test_input_series_is_not_mutated(): + data = pd.Series(DOCSTRING_DATA, dtype=float) + data.name = "my measurements" + before = data.copy() + calculate_weighted_windows(data, [2, 5, 2], "mean", full_output=False) + pd.testing.assert_series_equal(data, before) + + +class TestWindowParsing: + def test_string_window_maps_0_to_9_onto_0pt1_to_1pt0(self): + np.testing.assert_allclose(_window_to_array("094"), [0.1, 1.0, 0.5]) + + def test_x_becomes_nan_in_a_string_window(self): + np.testing.assert_allclose(_window_to_array("4x4"), [0.5, np.nan, 0.5], equal_nan=True) + + def test_x_becomes_nan_in_a_list_window(self): + np.testing.assert_allclose(_window_to_array([2, "x", 2]), [2.0, np.nan, 2.0], equal_nan=True) + + def test_list_weights_are_taken_literally(self): + np.testing.assert_allclose(_window_to_array([0.5, 1, 0.5]), [0.5, 1.0, 0.5]) + + def test_a_window_that_is_neither_string_nor_list_is_rejected(self): + with pytest.raises(TypeError, match="neither a string nor a list"): + _window_to_array(np.array([1, 1, 1])) + + +class TestInputValidation: + @pytest.fixture + def data(self): + return pd.Series(np.arange(10), dtype=float) + + def test_even_window_is_rejected(self, data): + with pytest.raises(ValueError, match="is even"): + calculate_weighted_windows(data, [1, 1], "mean") + + def test_empty_window_is_rejected(self, data): + with pytest.raises(ValueError, match="Window length is 0"): + calculate_weighted_windows(data, [], "mean") + + def test_over_long_window_is_rejected(self, data): + with pytest.raises(ValueError, match="too long"): + calculate_weighted_windows(data, [1] * 101, "mean") + + def test_unknown_statistic_is_rejected(self, data): + with pytest.raises(ValueError, match="not recognised"): + calculate_weighted_windows(data, [1, 1, 1], "median") + + def test_unknown_statistic_is_rejected_before_any_work_is_done(self): + """The check used to sit inside the loop, so it only fired after slicing the whole array.""" + with pytest.raises(ValueError, match="not recognised"): + calculate_weighted_windows(pd.Series([], dtype=float), [1, 1, 1], "median") + + def test_over_long_dataset_is_rejected(self): + with pytest.raises(ValueError, match="long input sequence"): + calculate_weighted_windows(pd.Series(np.zeros(10001)), [1, 1, 1], "mean") diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..0ca9918 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,82 @@ +"""Tests for the weighslide command-line interface.""" + +import numpy as np +import pandas as pd +import pytest + +from weighslide.weighslide import main + + +def test_raw_data_is_printed_to_stdout(capsys): + main(["[2,5,2]", "mean", "-r", "[0,0,0,1,1,2,3,5,8,13,21]"]) + printed = capsys.readouterr().out + values = [float(line) for line in printed.splitlines() if line.strip() and not line.startswith("Weighslide")] + expected = [0.0, 0.0, 0.666667, 2.333333, 3.666667, 6.0, 9.666667, 15.666667, 25.333333, 41.0, 65.5] + np.testing.assert_allclose(values, expected, rtol=1e-4) + + +def test_string_window_from_the_command_line(capsys): + main(["4x4", "mean", "-r", "[1,1,2,3,5,8,13,21,34]"]) + assert "Weighslide output:" in capsys.readouterr().out + + +def test_infile_is_analysed(tmp_path): + """`weighslide ... -i file.csv` used to fail with AttributeError before reading any data.""" + csv = tmp_path / "in.csv" + pd.DataFrame({"values": [0, 0, 0, 1, 1, 2, 3, 5, 8, 13, 21]}).to_csv(csv, index=False) + + main(["[2,5,2]", "mean", "-i", str(csv), "-n", "cli"]) + + result = pd.read_csv(tmp_path / "weighslide_output" / "cli_mean.csv", index_col=0).iloc[:, 0] + expected = [0.0, 0.0, 0.666667, 2.333333, 3.666667, 6.0, 9.666667, 15.666667, 25.333333, 41.0, 65.5] + np.testing.assert_allclose(result.to_numpy(), expected, rtol=1e-5) + + +def test_infile_with_a_named_column(tmp_path): + csv = tmp_path / "in.csv" + pd.DataFrame({"other": range(11), "values": [0, 0, 0, 1, 1, 2, 3, 5, 8, 13, 21]}).to_csv(csv, index=False) + main(["[2,5,2]", "mean", "-i", str(csv), "-n", "cli", "-c", "values"]) + assert (tmp_path / "weighslide_output" / "cli_mean.csv").is_file() + + +def test_overwrite_flag_is_honoured(tmp_path): + """The -o flag was parsed but never passed on, so a rerun always raised FileExistsError.""" + csv = tmp_path / "in.csv" + pd.DataFrame({"values": [1.0, 2, 3, 4, 5]}).to_csv(csv, index=False) + argv = ["[1,1,1]", "mean", "-i", str(csv), "-n", "cli"] + + main(argv) + with pytest.raises(FileExistsError): + main(argv) + main([*argv, "-o", "True"]) + + +def test_giving_both_an_infile_and_raw_data_is_rejected(tmp_path): + csv = tmp_path / "in.csv" + pd.DataFrame({"values": [1.0, 2, 3]}).to_csv(csv, index=False) + with pytest.raises(ValueError, match="only one data format"): + main(["[1,1,1]", "mean", "-i", str(csv), "-r", "[1,2,3]"]) + + +def test_giving_neither_an_infile_nor_raw_data_is_rejected(): + with pytest.raises(SystemExit): + main(["[1,1,1]", "mean"]) + + +def test_unknown_statistic_is_rejected_by_the_parser(): + with pytest.raises(SystemExit): + main(["[1,1,1]", "median", "-r", "[1,2,3]"]) + + +def test_importing_the_module_does_not_parse_arguments(): + """The parser used to be built at import time, which ran on every `import weighslide`.""" + import weighslide.weighslide as module + + assert not hasattr(module, "args") + assert callable(module.build_parser) + + +def test_help_is_available(capsys): + with pytest.raises(SystemExit): + main(["-h"]) + assert "weighslide" in capsys.readouterr().out diff --git a/tests/test_functional.py b/tests/test_functional.py deleted file mode 100644 index 0f17b4e..0000000 --- a/tests/test_functional.py +++ /dev/null @@ -1,37 +0,0 @@ -from shutil import rmtree - -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt - -from pathlib import Path -from weighslide import run_weighslide - - -def test_function(): - plt.rcParams["savefig.dpi"] = 120 - df = pd.DataFrame() - df['wave'] = [1, 1, 1, 3, 3, 3] * 8 - df["random"] = np.random.random_sample(df.shape[0]) - df["noisy wave"] = df.wave + df.random * 5 - fig, ax = plt.subplots() - df.plot(title="input data: noisy wave", ax=ax) - weighslide_dir = Path(__file__).parents[1] - temp_output_dir = weighslide_dir / "tests/temp_output" - if not temp_output_dir.is_dir(): - temp_output_dir.mkdir(parents=True) - data_csv = temp_output_dir / "wave.csv" - df.to_csv(data_csv) - - # run weighslide with a window that averages every 6th position - window = "9xxxxx9xxxxx9xxxxx9xxxxx9xxxxx9xxxxx9" - run_weighslide(data_csv, window, "mean", name="wavetest", column="noisy wave", overwrite=True) - - weighslide_output_dir = temp_output_dir / "weighslide_output" - assert (weighslide_output_dir / "wavetest9xxxxx9xxxxx9xxxxx9x.png").is_file() - assert (weighslide_output_dir / "wavetest9xxxxx9xxxxx9xxxxx9x.xlsx").is_file() - assert (weighslide_output_dir / "wavetest9xxxxx9xxxxx9xxxxx9x_mean.csv").is_file() - assert (weighslide_output_dir / "wavetest9xxxxx9xxxxx9xxxxx9x_sliced.csv").is_file() - - if temp_output_dir.is_dir(): - rmtree(temp_output_dir) diff --git a/tests/test_run_weighslide.py b/tests/test_run_weighslide.py new file mode 100644 index 0000000..cac2602 --- /dev/null +++ b/tests/test_run_weighslide.py @@ -0,0 +1,126 @@ +"""Tests for run_weighslide, which reads an input file and writes the output files and figure.""" + +import warnings + +import numpy as np +import pandas as pd +import pytest + +from weighslide import run_weighslide + +WINDOW = "9xxxxx9xxxxx9xxxxx9xxxxx9xxxxx9xxxxx9" +# run_weighslide truncates both the name and a string window to 20 characters +OUT_STEM = "wavetest" + WINDOW[:20] + + +@pytest.fixture +def noisy_wave_csv(tmp_path): + """A wave repeating every 6th position, buried in noise. The example from the README.""" + rng = np.random.default_rng(seed=0) + df = pd.DataFrame() + df["wave"] = [1, 1, 1, 3, 3, 3] * 8 + df["random"] = rng.random(df.shape[0]) + df["noisy wave"] = df.wave + df.random * 5 + csv = tmp_path / "wave.csv" + df.to_csv(csv) + return csv + + +def test_writes_all_four_output_files(noisy_wave_csv): + run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest", column="noisy wave") + + outdir = noisy_wave_csv.parent / "weighslide_output" + assert (outdir / f"{OUT_STEM}.png").is_file() + assert (outdir / f"{OUT_STEM}.xlsx").is_file() + assert (outdir / f"{OUT_STEM}_mean.csv").is_file() + assert (outdir / f"{OUT_STEM}_sliced.csv").is_file() + assert (outdir / f"{OUT_STEM}_mult.csv").is_file() + + +def test_accepts_a_string_path_as_documented_in_the_readme(noisy_wave_csv): + """The README passes a plain string. This used to raise AttributeError: 'str' has no attribute 'name'.""" + run_weighslide(str(noisy_wave_csv), WINDOW, "mean", name="wavetest", column="noisy wave") + assert (noisy_wave_csv.parent / "weighslide_output" / f"{OUT_STEM}_mean.csv").is_file() + + +def test_smooths_the_repeating_wave(noisy_wave_csv): + """The point of the package: a window matching the period should recover the underlying wave.""" + run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest", column="noisy wave") + + outdir = noisy_wave_csv.parent / "weighslide_output" + result = pd.read_csv(outdir / f"{OUT_STEM}_mean.csv", index_col=0).iloc[:, 0] + original = pd.read_csv(noisy_wave_csv)["noisy wave"] + assert len(result) == len(original) + # only the central positions have full-length windows; the flanks are averaged over fewer points + centre = slice(18, 30) + underlying = pd.Series([1, 1, 1, 3, 3, 3] * 8, dtype=float) + # the smoothed signal must track the underlying wave more closely than the noisy input does + assert (result[centre] - underlying[centre]).abs().mean() < (original[centre] - underlying[centre]).abs().mean() + + +def test_excel_output_has_the_three_expected_sheets(noisy_wave_csv): + run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest", column="noisy wave") + xlsx = noisy_wave_csv.parent / "weighslide_output" / f"{OUT_STEM}.xlsx" + sheets = pd.ExcelFile(xlsx).sheet_names + assert sheets == ["orig_data_sliced", "data_multipled", "window_mean"] + + +def test_reads_an_excel_input_file(tmp_path): + xlsx = tmp_path / "in.xlsx" + pd.DataFrame({"values": [0, 0, 0, 1, 1, 2, 3, 5, 8, 13, 21]}).to_excel(xlsx, index=False, sheet_name="data") + run_weighslide(xlsx, [2, 5, 2], "mean", name="xl", excel_kwargs={"sheet_name": "data"}) + + result = pd.read_csv(tmp_path / "weighslide_output" / "xl_mean.csv", index_col=0).iloc[:, 0] + expected = [0.0, 0.0, 0.666667, 2.333333, 3.666667, 6.0, 9.666667, 15.666667, 25.333333, 41.0, 65.5] + np.testing.assert_allclose(result.to_numpy(), expected, rtol=1e-5) + + +def test_single_column_input_needs_no_column_name(tmp_path): + csv = tmp_path / "single.csv" + pd.DataFrame({"values": [1.0, 2, 3, 4, 5]}).to_csv(csv, index=False) + run_weighslide(csv, [1, 1, 1], "mean", name="single") + assert (tmp_path / "weighslide_output" / "single_mean.csv").is_file() + + +def test_existing_output_is_not_overwritten_by_default(noisy_wave_csv): + run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest", column="noisy wave") + with pytest.raises(FileExistsError, match="already exist"): + run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest", column="noisy wave") + + +def test_overwrite_true_replaces_existing_output(noisy_wave_csv): + run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest", column="noisy wave") + run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest", column="noisy wave", overwrite=True) + + +def test_multi_column_input_without_a_column_name_is_rejected(noisy_wave_csv): + with pytest.raises(ValueError, match="No column name provided"): + run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest") + + +def test_a_column_name_that_is_not_in_the_file_is_rejected(noisy_wave_csv): + with pytest.raises(ValueError, match="not in the input file"): + run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest", column="no such column") + + +def test_unsupported_filetype_is_rejected(tmp_path): + txt = tmp_path / "data.txt" + txt.write_text("1\n2\n3\n", encoding="utf-8") + with pytest.raises(ValueError, match="Filetype must be excel or csv"): + run_weighslide(txt, [1, 1, 1], "mean") + + +def test_csv_kwargs_are_passed_to_pandas(tmp_path): + csv = tmp_path / "skipme.csv" + csv.write_text("# a comment line\nvalues\n1\n2\n3\n4\n5\n", encoding="utf-8") + run_weighslide(csv, [1, 1, 1], "mean", name="skip", csv_kwargs={"skiprows": 1}) + assert (tmp_path / "weighslide_output" / "skip_mean.csv").is_file() + + +def test_no_deprecation_or_future_warnings_are_raised(noisy_wave_csv): + """This package sat untouched for years; pandas and numpy deprecations must not pile up again.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + warnings.simplefilter("error", DeprecationWarning) + warnings.simplefilter("error", UserWarning) + run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest", column="noisy wave") From 14c67a1bbab8952df84ec33e27d300c10bca47da Mon Sep 17 00:00:00 2001 From: Mark Teese Date: Mon, 31 Aug 2026 12:23:25 +0200 Subject: [PATCH 5/6] Update README for current installation, usage and supported versions The installation section named python 3.5 and `python setup.py install`, and every usage example invoked `python weighslide.py`, none of which apply to an installed package. Documents the console script, the output files each run produces, the range of python and dependency versions the test suite is run against, and the development setup. Removes the contact image, which pointed at a branch of a different repository and would break if that branch were ever deleted, and points the contribute section at this repository's issue tracker instead. --- README.md | 343 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 199 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index 571c30d..5358dfa 100644 --- a/README.md +++ b/README.md @@ -1,151 +1,206 @@ -# Weighslide - -Weighslide is a python program to calculate sliding windows across of a list of numerical values. -The user sets the window size, and the exact weighting of each value in the window. - +# Weighslide + +Weighslide is a python program to calculate sliding windows across of a list of numerical values. The user sets the window size, and the exact weighting of each value in the window. + ## What is it used for? -Weighslide was developed for use in bioinformatics. Alpha-helices are common protein secondary structure, and have -a periodicity of 3.6 residues per turn. Weighslide allows numerical values to be weighted according to alpha-helical peridicity. + +Weighslide was developed for use in bioinformatics. Alpha-helices are common protein secondary structure, and have a periodicity of 3.6 residues per turn. Weighslide allows numerical values to be weighted according to alpha-helical peridicity. Note that weighslide is not currently optimised for large datasets. - -## Citation: -Please cite as follows: -"A sliding window analysis was performed using the weighslide package in python (Mark Teese, Technical University of Munich)."
- -## Keywords: -sliding window, rolling window, weighted window, data normalisation, data normalization, 1D array, numerical list
- -# How it works -Weighslide takes as an input a 1D array (list) of numerical data, and applies a user-defined weighting and algorithm in a sliding-window fashion across the data. - -``` -For example: -window = [ 2 5 2 ] -statistic = "mean" -dataset = [ 0 0 0 1 1 2 3 5 8 13 21] -The window length is 3. The array will therefore be sliced as follows: -........[ NaN 0 0 ] -.............[ 0 0 1 ] -................[ 0 1 1 ] -...................[ 1 1 2 ] and so on until the final slice [ 13 21 NaN ] -Each array slice will be "weighted" by multiplication with the window, array-style, resulting in: -........[ NaN 0 0 ] -.............[ 0 0 2 ] -................[ 0 5 2 ] -...................[ 2 5 4 ] and so on. -If the "statistic" variable is given as "mean", a mean will be calculated for each array slice. -..........[ 0 ] -.............[ 0.66 ] -................[ 2.33 ] -...................[ 3.66 ] and so on. -The "statistic" can be mean, std, or sum. -The value (in this case the mean) will replace the central position in the output 1D array. -output = [ 0.00 0.00 0.66 2.33 3.66 6.00 9.66 15.6 25.3 41.0 65.5 ] -``` - -The first and last array slices always contain flanking "not a number" (Nan) values, which are ignored in all calculations. -The first and last output values therefore do not represent results from true, full-length windows. - + +## Citation: + +Please cite as follows: +"A sliding window analysis was performed using the weighslide package in python (Mark Teese, Technical University of Munich)."
+ +## Keywords: + +sliding window, rolling window, weighted window, data normalisation, data normalization, 1D array, numerical list
+ +# How it works + +Weighslide takes as an input a 1D array (list) of numerical data, and applies a user-defined weighting and algorithm in a sliding-window fashion across the data. + +``` +For example: +window = [ 2 5 2 ] +statistic = "mean" +dataset = [ 0 0 0 1 1 2 3 5 8 13 21] +The window length is 3. The array will therefore be sliced as follows: +........[ NaN 0 0 ] +.............[ 0 0 1 ] +................[ 0 1 1 ] +...................[ 1 1 2 ] and so on until the final slice [ 13 21 NaN ] +Each array slice will be "weighted" by multiplication with the window, array-style, resulting in: +........[ NaN 0 0 ] +.............[ 0 0 2 ] +................[ 0 5 2 ] +...................[ 2 5 4 ] and so on. +If the "statistic" variable is given as "mean", a mean will be calculated for each array slice. +..........[ 0 ] +.............[ 0.66 ] +................[ 2.33 ] +...................[ 3.66 ] and so on. +The "statistic" can be mean, std, or sum. +The value (in this case the mean) will replace the central position in the output 1D array. +output = [ 0.00 0.00 0.66 2.33 3.66 6.00 9.66 15.6 25.3 41.0 65.5 ] +``` + +The first and last array slices always contain flanking "not a number" (Nan) values, which are ignored in all calculations. The first and last output values therefore do not represent results from true, full-length windows. + # Installation -`pip install weighslide` - -Weighslide depends on the following: -* python (tested for version 3.5) -* numpy -* pandas -* matplotlib - -For Windows users, we recommend Anaconda python 3.x. The Anaconda package should contain all required python packages. - -To install as a python package from GitHub, download and unzip the latest release and run `python setup.py install` in the relevant package folder. - -# Test -To test the package, open the command console and navigate to the folder -containing weighslide.py. Run the following: - -`python weighslide.py [1,1,"x",1,1] mean -r [1,1,2,3,5,8,13,21,34]` - -If successful, an output list will be printed on the screen. - -# Usage -Here is an example of how to run weighslide within python, using an excel input file. -``` -import weighslide -infile = r"D:\Path\To\Your\File\infile_name.xlsx" -# for excel files, you will need to input the sheet name containing the data -excel_kwargs = {"sheet_name":"Sheet1"} -# if it's an excel file with multiple columns, define which column contains the data -column = "your data column header" -# define the window and statistic. The following parameters are used -# if you want to calculate mean of the four surrounding values in the sequence -window = [1,1,"x",1,1] -statistic = "mean" -name = "your short sample name" -weighslide.run_weighslide(infile, window, statistic, name=name, column=column, excel_kwargs=excel_kwargs) -``` - -Here is an example of how to run weighslide from the command line, using a csv input file. -``` -python weighslide.py [1,1,'x',1,1] mean -i "D:\Path\To\Your\File\infile_name.xlsx" -c "your data column header" -``` -In both cases the output files will be created in a subfolder within the same location as the input file. - -For more help regarding the command-line options: -`python weighslide.py -h` - -Here is an example designed for jupyter notebook, showing the power of weighslide -to smoothen a repeated element in a noisy dataset. -``` -# create a noisy wave that repeats every 6th position. Save to csv. -import weighslide -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -% matplotlib inline -plt.rcParams["savefig.dpi"] = 120 -df = pd.DataFrame() -df['wave'] = [1,1,1,3,3,3]*8 -df["random"] = np.random.random_sample(df.shape[0]) -df["noisy wave"] = df.wave + df.random*5 -df.plot(title="input data: noisy wave") -df.to_csv("wave.csv") -``` -![Image of input](https://github.com/teese/weighslide/raw/master/examples/input.png) -``` -# run weighslide with a window that averages every 6th position -window = "9xxxxx9xxxxx9xxxxx9xxxxx9xxxxx9xxxxx9" -weighslide.run_weighslide("wave.csv", window, "mean", name="wavetest", column="noisy wave", overwrite=True) -``` -![Image of output](https://github.com/teese/weighslide/raw/master/examples/output.png) - - -**Examples of windows:** - -[1,1,1] -* if "statistic" is set to "mean", this window returns the average of the central position, and the two neighbouring positions -* the window size is 3 - -[1,1,"x",1,1] -* the central position "x" has no weighting at all -* the window size is 5, it consists of the central position, two upstream, and two downstream positions -* the positions upstream (-1, -2) and downstream (1, 2) of the central position are all equally weighted -* if the statistic is set to "mean", the result for each position will simply be the average of the surrounding 4 positions - -[0.5, 1, 0.5, 2, 0.5, 1, 0.5] -* the central position "2" is highly weighted (2*orig value) -* the window size is 7, it consists of the central position, three upstream, and three downstream positions -* the positions upstream (-1, -2, -3) and downstream (1, 2, 3) of the central position are unequally weighted -* if the statistic is set to "mean", the result for each position will simply be the average of the surrounding 4 positions - -# Contribute -If you encounter a bug or weighslide doesn't work for any reason, please send me an email (available in an image below, or at my TUM website) or initiate an issue in Github. -Pull requests are welcome. - -# License -Weighslide is free software distributed under the permissive MIT license. +```bash +pip install weighslide +``` + +Weighslide requires python 3.9 or later, and installs numpy, pandas, matplotlib and openpyxl. + +The dependency lower bounds are the oldest versions the test suite is actually run against, rather than guesses. As of version 0.3.0 the suite passes on both ends of that range: + +| | oldest tested | newest tested | +| --- | --- | --- | +| python | 3.9 | 3.13 | +| pandas | 1.2.5 | 3.0.5 | +| numpy | 1.20.3 | 2.5.2 | +| matplotlib | 3.4.3 | 3.11.1 | + +If you are on a python older than 3.9, `pip install weighslide` will simply give you the newest release that supports your version, rather than failing. + +To install from a clone of this repository: + +```bash +pip install . +``` + +# Usage + +## From python + +```python +import weighslide + +infile = r"D:\Path\To\Your\File\infile_name.xlsx" +# for excel files, you will need to input the sheet name containing the data +excel_kwargs = {"sheet_name": "Sheet1"} +# if it's an excel file with multiple columns, define which column contains the data +column = "your data column header" +# define the window and statistic. The following parameters are used +# if you want to calculate mean of the four surrounding values in the sequence +window = [1, 1, "x", 1, 1] +statistic = "mean" +name = "your short sample name" +weighslide.run_weighslide(infile, window, statistic, name=name, column=column, excel_kwargs=excel_kwargs) +``` + +To apply the algorithm to a pandas Series without writing any output files, use `calculate_weighted_windows` directly: + +```python +import pandas as pd +from weighslide import calculate_weighted_windows + +data = pd.Series([0, 0, 0, 1, 1, 2, 3, 5, 8, 13, 21], dtype=float) +result = calculate_weighted_windows(data, [2, 5, 2], "mean", full_output=False) +``` + +## From the command line -# Contact +Installing the package adds a `weighslide` command. -![contact details](https://github.com/teese/eccpy/raw/develop/docs/images/signac_seine_bei_samois.png) \ No newline at end of file +Analyse a csv or excel file: + +```bash +weighslide "[1,1,'x',1,1]" mean -i "D:\Path\To\Your\File\infile_name.xlsx" -c "your data column header" +``` + +Analyse a list given directly on the command line, which prints the result to the screen: + +```bash +weighslide "[1,1,'x',1,1]" mean -r "[1,1,2,3,5,8,13,21,34]" +``` + +For the full list of command-line options: + +```bash +weighslide -h +``` + +When an input file is given, the output files are created in a `weighslide_output` subfolder alongside the input file. + +## Output files + +| File | Contents | +| --- | --- | +| `_.csv` | The result of the sliding window analysis, one value per input position. | +| `_sliced.csv` | Every slice taken from the original array, for checking the slice algorithm. | +| `_mult.csv` | Every slice after multiplication by the window. | +| `.xlsx` | The three datasets above, on separate sheets. | +| `.png` | A line graph of the original data against the sliding window output. | + +## Jupyter notebook example + +This shows the power of weighslide to smoothen a repeated element in a noisy dataset. + +```python +# create a noisy wave that repeats every 6th position. Save to csv. +import weighslide +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +%matplotlib inline +plt.rcParams["savefig.dpi"] = 120 +df = pd.DataFrame() +df["wave"] = [1, 1, 1, 3, 3, 3] * 8 +df["random"] = np.random.random_sample(df.shape[0]) +df["noisy wave"] = df.wave + df.random * 5 +df.plot(title="input data: noisy wave") +df.to_csv("wave.csv") +``` + +![Image of input](https://github.com/teese/weighslide/raw/master/examples/input.png) + +```python +# run weighslide with a window that averages every 6th position +window = "9xxxxx9xxxxx9xxxxx9xxxxx9xxxxx9xxxxx9" +weighslide.run_weighslide("wave.csv", window, "mean", name="wavetest", column="noisy wave", overwrite=True) +``` + +![Image of output](https://github.com/teese/weighslide/raw/master/examples/output.png) + +## Examples of windows + +`[1,1,1]` +* if "statistic" is set to "mean", this window returns the average of the central position, and the two neighbouring positions +* the window size is 3 + +`[1,1,"x",1,1]` +* the central position "x" has no weighting at all +* the window size is 5, it consists of the central position, two upstream, and two downstream positions +* the positions upstream (-1, -2) and downstream (1, 2) of the central position are all equally weighted +* if the statistic is set to "mean", the result for each position will simply be the average of the surrounding 4 positions + +`[0.5, 1, 0.5, 2, 0.5, 1, 0.5]` +* the central position "2" is highly weighted (2*orig value) +* the window size is 7, it consists of the central position, three upstream, and three downstream positions +* the positions upstream (-1, -2, -3) and downstream (1, 2, 3) of the central position are unequally weighted +* if the statistic is set to "mean", the result for each position will simply be the average of the surrounding 4 positions + +A window can also be written as a string of digits, where `0` is the lowest weighting (0.1) and `9` the highest (1.0), and `x` marks a position to ignore. For example `"4x4"` is equivalent to `[0.5, nan, 0.5]`. Only odd-length windows are accepted, so that each result centres on a single unambiguous position. + +# Development + +```bash +pip install -e ".[dev]" +pre-commit install +pytest +``` + +`pre-commit run --all-files` runs black, ruff and mypy. CI runs the same hooks, plus the test suite on python 3.9, 3.12 and 3.13 on Linux and Windows. + +# Contribute + +If you encounter a bug, or weighslide doesn't work for any reason, please open an issue at https://github.com/teese/weighslide/issues. Pull requests are welcome. + +# License + +Weighslide is free software distributed under the permissive MIT license. From 7dd8cb3cc2c40d47f7316f681fccaba664ca0a78 Mon Sep 17 00:00:00 2001 From: Mark Teese Date: Mon, 31 Aug 2026 12:29:44 +0200 Subject: [PATCH 6/6] Build the window array without pandas replace pd.Series.replace downcasts an object column, which is deprecated in pandas 2.2 and 2.3 and raises a FutureWarning there. The list form of the window went through that call, so any caller passing a list rather than a string produced the warning on those versions. The window is now built with plain numpy, which has no equivalent deprecation and drops a pandas round trip from the conversion. The test that asserts a run raises no deprecation warnings only exercised the string form, so it did not cover the affected branch. It now runs over both forms. --- tests/test_run_weighslide.py | 11 ++++++++--- weighslide/weighslide.py | 22 ++++++++++------------ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/test_run_weighslide.py b/tests/test_run_weighslide.py index cac2602..e957683 100644 --- a/tests/test_run_weighslide.py +++ b/tests/test_run_weighslide.py @@ -117,10 +117,15 @@ def test_csv_kwargs_are_passed_to_pandas(tmp_path): assert (tmp_path / "weighslide_output" / "skip_mean.csv").is_file() -def test_no_deprecation_or_future_warnings_are_raised(noisy_wave_csv): - """This package sat untouched for years; pandas and numpy deprecations must not pile up again.""" +@pytest.mark.parametrize("window", [WINDOW, [1, 1, "x", 1, 1], [2, 5, 2]], ids=["string", "list_with_x", "list"]) +def test_no_deprecation_or_future_warnings_are_raised(noisy_wave_csv, window): + """This package sat untouched for years; pandas and numpy deprecations must not pile up again. + + Parametrised over both window forms: the string and list branches take different code + paths, and a deprecation on one of them is invisible when only the other is exercised. + """ with warnings.catch_warnings(): warnings.simplefilter("error", FutureWarning) warnings.simplefilter("error", DeprecationWarning) warnings.simplefilter("error", UserWarning) - run_weighslide(noisy_wave_csv, WINDOW, "mean", name="wavetest", column="noisy wave") + run_weighslide(noisy_wave_csv, window, "mean", name="wavetest", column="noisy wave") diff --git a/weighslide/weighslide.py b/weighslide/weighslide.py index f5508b8..74b5d66 100644 --- a/weighslide/weighslide.py +++ b/weighslide/weighslide.py @@ -242,20 +242,18 @@ def _window_to_array(window: list | str) -> np.ndarray: A string window uses a 0-9 shorthand for the weighting, which is shifted and scaled onto 0.1-1.0. A list window is taken as literal weights. """ + # Built with plain numpy rather than pd.Series.replace, whose downcasting of an + # object column is deprecated in pandas 2.2 and warns there. if isinstance(window, str): - # split into a list of single characters, e.g. "4x4" -> ["4", "x", "4"] - window_series = pd.Series(list(window)) - # replace x with np.nan, then change dtype to float - window_series = window_series.replace("x", np.nan).astype(float) + # each character is a digit 0-9, or "x" for a position to ignore + values = [np.nan if char == "x" else float(char) for char in window] # convert 0-9 scale to 1-10, divide by 10 to give a relative weighting - window_series = (window_series + 1) / 10 - elif isinstance(window, list): - # replace x with np.nan - window_series = pd.Series(window, dtype=object).replace("x", np.nan).astype(float) - else: - raise TypeError("The input variable 'window' is neither a string nor a list.") - - return np.asarray(window_series, dtype=float) + return (np.asarray(values, dtype=float) + 1) / 10 + if isinstance(window, list): + # "x" marks a position to ignore + values = [np.nan if isinstance(value, str) and value == "x" else value for value in window] + return np.asarray(values, dtype=float) + raise TypeError("The input variable 'window' is neither a string nor a list.") def calculate_weighted_windows(data_series, window, statistic, full_output=True):