diff --git a/pyproject.toml b/pyproject.toml index 3fe71a5..4ab5fc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,6 @@ dependencies = [ "pybbi>=0.4.0,<0.5", "pyjaspar>=4.0.0,<5", "scikit-learn>=1.7.2,<2", - "setuptools-scm>=8.3.0,<9", "tqdm>=4.67.1,<5", ] diff --git a/requirements.txt b/requirements.txt index fbab6bb..d66fb1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,4 @@ pandas == 2.3.3 pybbi == 0.4.2 pyjaspar == 4.0.0 scikit-learn == 1.7.2 -setuptools-scm == 8.3.1 tqdm == 4.67.1 diff --git a/src/sponge/cli.py b/src/sponge/cli.py index 122f69d..99a3464 100644 --- a/src/sponge/cli.py +++ b/src/sponge/cli.py @@ -21,8 +21,8 @@ from argparse import (ArgumentDefaultsHelpFormatter, ArgumentParser, RawDescriptionHelpFormatter) +from importlib.metadata import version from pathlib import Path -from setuptools_scm import get_version from .sponge import Sponge @@ -77,11 +77,7 @@ def cli( args = parser.parse_args() if args.show_version: - print (get_version( - Path(__file__).parents[2], - version_scheme='only-version', - local_scheme='no-local-version', - )) + print (version('netzoopy-sponge')) elif args.example_config: file_dir = Path(__file__).parents[0] shutil.copy(os.path.join(file_dir, 'user_config.yaml'), diff --git a/src/sponge/config.yaml b/src/sponge/config.yaml index 850572f..0d15715 100644 --- a/src/sponge/config.yaml +++ b/src/sponge/config.yaml @@ -2,17 +2,17 @@ url: motif: full: - - "https://mencius.uio.no/JASPAR/JASPAR_TFBSs/{year}/\ + - "https://mencius-pub.uio.no/JASPAR/JASPAR_TFBSs/{year}/\ JASPAR{year}_{genome_assembly}.bb" - "http://expdata.cmmt.ubc.ca/JASPAR/downloads/UCSC_tracks/{year}/\ JASPAR{year}_{genome_assembly}.bb" by_tf: - - "https://mencius.uio.no/JASPAR/JASPAR_TFBSs/{year}/\ + - "https://mencius-pub.uio.no/JASPAR/JASPAR_TFBSs/{year}/\ {genome_assembly}/" - "http://expdata.cmmt.ubc.ca/JASPAR/downloads/UCSC_tracks/{year}/\ {genome_assembly}/" region: - xml: "http://www.ensembl.org/biomart" + xml: "http://jun2026.archive.ensembl.org/biomart" rest: "https://rest.ensembl.org" protein: "https://rest.uniprot.org/idmapping/" ppi: "https://string-db.org/api/tsv/" diff --git a/src/sponge/config_manager.py b/src/sponge/config_manager.py index c52f738..1895fda 100644 --- a/src/sponge/config_manager.py +++ b/src/sponge/config_manager.py @@ -88,7 +88,8 @@ def __del__( if self.temp_folder is not None: log_file = os.path.join(self.temp_folder, self._log_filename) - yaml.safe_dump(self.config, open(log_file, 'w', encoding='utf-8')) + with open(log_file, 'w', encoding='utf-8') as f: + yaml.safe_dump(self.config, f) def __getitem__( diff --git a/src/sponge/modules/protein_id_mapper/protein_id_mapper.py b/src/sponge/modules/protein_id_mapper/protein_id_mapper.py index afbbcd1..3d2e793 100644 --- a/src/sponge/modules/protein_id_mapper/protein_id_mapper.py +++ b/src/sponge/modules/protein_id_mapper/protein_id_mapper.py @@ -103,7 +103,7 @@ def get_uniprot_mapping( if 'results' not in uniprot_status.json(): # Unable to retrieve the results within the given time raise ConnectionError('No results have been retrieved from UniProt' - 'in the given time') + ' in the given time') # Retrieve the results uniprot_results = requests.get(self.mapping_url + f'stream/{job_id}') diff --git a/src/sponge/modules/utils/data_retrieval.py b/src/sponge/modules/utils/data_retrieval.py index 8893b9f..839787c 100644 --- a/src/sponge/modules/utils/data_retrieval.py +++ b/src/sponge/modules/utils/data_retrieval.py @@ -26,7 +26,7 @@ from io import BytesIO from pathlib import Path from tqdm import tqdm -from typing import Iterable, List, Optional, Tuple, Union +from typing import Iterable, List, Optional, Union ### Functions ### def download_with_progress( @@ -106,6 +106,8 @@ def download_with_progress( if file_path is None: return BytesIO(stream.getvalue()) + else: + stream.close() def create_xml_query( @@ -179,10 +181,14 @@ def retrieve_ensembl_data( BytesIO Bytes retrieved from the server """ + + # Formulate the XML query xml_query = create_xml_query(dataset_name, requested_fields, filters) REQUEST_STRING = '/martservice?query=' + # Build a full link to the Ensembl BioMart service link = ensembl_url + REQUEST_STRING + xml_query MAX_ITERATIONS = 50 + # Various failure messages that can be returned by Ensembl FAILURE = ['Query ERROR', 'Service unavailable'] for _ in range(MAX_ITERATIONS): r = requests.get(link, stream=True) diff --git a/src/sponge/modules/utils/tfbs_filtering.py b/src/sponge/modules/utils/tfbs_filtering.py index 06a280e..846f640 100644 --- a/src/sponge/modules/utils/tfbs_filtering.py +++ b/src/sponge/modules/utils/tfbs_filtering.py @@ -163,6 +163,7 @@ def iterate_chromosomes( results_list += edges_chrom_list elapsed_chr = time.time() - st_chr print (f'Done in: {elapsed_chr // 60:n} m {elapsed_chr % 60:.2f} s') + p.close() return results_list @@ -254,6 +255,7 @@ def process_motif( result = p.starmap_async(process_chromosome, input_tuples, chunksize=n_processes) result_list = result.get() + p.close() # Merge the results df = pd.concat(result_list, ignore_index=True, copy=False) diff --git a/src/sponge/modules/version_logger/version_logger.py b/src/sponge/modules/version_logger/version_logger.py index eaad214..5be4fae 100644 --- a/src/sponge/modules/version_logger/version_logger.py +++ b/src/sponge/modules/version_logger/version_logger.py @@ -58,8 +58,8 @@ def __init__( self.data = defaultdict(dict) if os.path.exists(self.log_file): try: - self.data = defaultdict(dict, - yaml.safe_load(open(self.log_file))) + with open(self.log_file, 'r', encoding='utf-8') as f: + self.data = yaml.safe_load(f) except TypeError: # Most likely means an empty log file, ignore pass @@ -80,8 +80,8 @@ def __del__( """ if len(self.data) > 0: - yaml.safe_dump(dict(self.data), - open(self.log_file, 'w', encoding='utf-8')) + with open(self.log_file, 'w', encoding='utf-8') as f: + yaml.safe_dump(dict(self.data), f) def __getitem__( diff --git a/src/sponge/sponge.py b/src/sponge/sponge.py index 1dbfd27..848368f 100644 --- a/src/sponge/sponge.py +++ b/src/sponge/sponge.py @@ -20,8 +20,8 @@ from bioframe import assembly_info from functools import wraps +from importlib.metadata import version from pathlib import Path -from setuptools_scm import get_version from typing import Callable, Union from sponge.config_manager import ConfigManager @@ -132,11 +132,8 @@ def __init__( # Fill in the default values in the user config self.fill_default_values() self.version_logger = VersionLogger(temp_folder) - self.version_logger.write_retrieved('SPONGE', get_version( - Path(__file__).parents[2], - version_scheme='only-version', - local_scheme='no-local-version', - )) + self.version_logger.write_retrieved('SPONGE', + version('netzoopy-sponge')) # Retrieve necessary files if required self.retrieve_data() # Run the default workflow if selected diff --git a/tests/test_sponge.py b/tests/test_sponge.py index 0df14ee..978fc20 100644 --- a/tests/test_sponge.py +++ b/tests/test_sponge.py @@ -18,6 +18,7 @@ ### Imports ### import bioframe import datetime +import matplotlib import os import pytest import subprocess @@ -28,12 +29,16 @@ from Bio.motifs.jaspar import Motif from pathlib import Path from pyjaspar import jaspardb, JASPAR_LATEST_RELEASE +from requests.exceptions import HTTPError from typing import Any, Iterable, Tuple from sponge.config_manager import ConfigManager from sponge.modules.match_aggregator import MatchAggregator from sponge.modules.ppi_retriever import PPIRetriever +# Change plotting backend to avoid issues with headless environments +matplotlib.use('Agg') + ### Fixtures ### # Core config fixture @pytest.fixture @@ -221,9 +226,11 @@ def test_download_with_progress(input, compare_to, tmp_path): else: file_path = os.path.join(tmp_path, input[1]) data_f.download_with_progress(input[0], file_path) - data = open(file_path, 'r').read() + with open(file_path, 'r', encoding='utf-8') as f: + data = f.read() - comp_data = open(compare_to, 'r').read() + with open(compare_to, 'r', encoding='utf-8') as f: + comp_data = f.read() assert data == comp_data @@ -249,6 +256,11 @@ def test_create_xml_query(input): ('hsapiens_gene_ensembl', ['ensembl_transcript_id'], {'chromosome_name': ['chr19', 'chrX']}), ]) +@pytest.mark.xfail( + reason='Ensembl server may be down, so this test can fail', + raises=(ConnectionError, HTTPError), + strict=False, +) def test_retrieve_ensembl_data(dataset, fields, filters, core_config): ensembl_url = core_config['url']['region']['xml'] df = pd.read_table(data_f.retrieve_ensembl_data(dataset, fields, @@ -436,7 +448,8 @@ def test_config_manager(c_path, input, expected_length, tmp_path): assert os.path.exists(config_file) - data = yaml.safe_load(open(config_file, 'r', encoding='utf-8')) + with open(config_file, 'r', encoding='utf-8') as f: + data = yaml.safe_load(f) assert len(data) == expected_length @@ -467,7 +480,8 @@ def test_version_logger(input, expected_length, tmp_path): assert os.path.exists(fp_file) - data = yaml.safe_load(open(fp_file, 'r', encoding='utf-8')) + with open(fp_file, 'r', encoding='utf-8') as f: + data = yaml.safe_load(f) assert len(data) == expected_length @@ -536,6 +550,11 @@ def test_tfbs_retriever(settings, assembly, on_the_fly, default_user_config, ({'region_file': 'LICENSE'}, 'hg38'), ({'chromosomes': ['chr19']}, 'random_assembly'), ]) +@pytest.mark.xfail( + reason='Ensembl server may be down, so this test can fail', + raises=(ConnectionError, HTTPError), + strict=False, +) def test_region_retriever(settings, assembly, core_config, default_user_config, tmp_path): default_user_config.deep_update({'region': settings}) @@ -566,6 +585,11 @@ def test_region_retriever(settings, assembly, core_config, default_user_config, {'region': {'chromosomes': ['chr19']}}, {'on_the_fly_processing': True, 'region': {'chromosomes': ['chr19']}}, ]) +@pytest.mark.xfail( + reason='Ensembl server may be down, so this test can fail', + raises=(ConnectionError, HTTPError), + strict=False, +) def test_data_retriever(config_update, core_config, default_user_config, tmp_path): # The full bigbed file is way too big, just a placeholder @@ -786,7 +810,8 @@ def run_integration_test_common( motif_output = os.path.join(tmp_path, 'motif_prior.tsv') ppi_output = os.path.join(tmp_path, 'ppi_prior.tsv') - settings = yaml.safe_load(open(config_file, 'r')) + with open(config_file, 'r', encoding='utf-8') as f: + settings = yaml.safe_load(f) settings['motif_output']['file_name'] = motif_output settings['ppi_output']['file_name'] = ppi_output @@ -810,6 +835,11 @@ def run_integration_test_common( @pytest.mark.integration @pytest.mark.network @pytest.mark.slow +@pytest.mark.xfail( + reason='Ensembl server may be down, so this test can fail', + raises=(ConnectionError, HTTPError), + strict=False, +) def test_full_default_workflow(tmp_path): _,_ = run_integration_test_common( tmp_path, @@ -820,6 +850,11 @@ def test_full_default_workflow(tmp_path): @pytest.mark.integration @pytest.mark.network +@pytest.mark.xfail( + reason='Ensembl server may be down, so this test can fail', + raises=(ConnectionError, HTTPError), + strict=False, +) def test_small_workflow(tmp_path): motif_output,ppi_output = run_integration_test_common( tmp_path,