From 239ad727db43be052f0276facceac825fa672e76 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Sat, 19 Sep 2026 07:00:46 -0700 Subject: [PATCH 1/2] Use exist_ok when creating directory in stream_download --- pooch/core.py | 2 +- pooch/tests/test_core.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pooch/core.py b/pooch/core.py index d358244f..35f6814e 100644 --- a/pooch/core.py +++ b/pooch/core.py @@ -812,7 +812,7 @@ def stream_download( # Ensure the parent directory exists in case the file is in a subdirectory. # Otherwise, move will cause an error. if not fname.parent.exists(): - os.makedirs(str(fname.parent)) + os.makedirs(str(fname.parent), exist_ok=True) download_attempts = 1 + retry_if_failed max_wait = 10 for i in range(download_attempts): diff --git a/pooch/tests/test_core.py b/pooch/tests/test_core.py index 0009851e..eb038000 100644 --- a/pooch/tests/test_core.py +++ b/pooch/tests/test_core.py @@ -12,6 +12,8 @@ import hashlib import os import re +import threading +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from tempfile import TemporaryDirectory @@ -635,6 +637,40 @@ def test_stream_download(fname): check_tiny_data(str(destination)) +def test_stream_download_parallel_subdir(monkeypatch): + "Downloading into a new subdirectory in parallel shouldn't fail" + threads = 4 + barrier = threading.Barrier(nthreads, timeout=30) + makedirs = os.makedirs + + def mockmakedirs(path, exist_ok=False): + "Wait for all the threads before creating the directory" + barrier.wait() + makedirs(path, exist_ok=exist_ok) + + def downloader(url, output_file, pooch): # noqa: ARG001 + "Write some data instead of downloading it" + Path(output_file).write_text("some data", encoding="utf-8") + + monkeypatch.setattr(os, "makedirs", mockmakedirs) + + with TemporaryDirectory() as local_store: + destinations = [ + Path(local_store) / "subdir" / f"data-{i}.txt" for i in range(nthreads) + ] + with ThreadPoolExecutor(max_workers=nthreads) as executor: + futures = [ + executor.submit( + stream_download, "unused", dest, None, downloader, pooch=None + ) + for dest in destinations + ] + for future in futures: + future.result() + for dest in destinations: + assert dest.read_text(encoding="utf-8") == "some data" + + @pytest.mark.network @pytest.mark.parametrize( "url", From b546bbc352a2866d5a6901066daa380a129dbc4c Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Sat, 19 Sep 2026 14:06:39 -0700 Subject: [PATCH 2/2] nthreads --- pooch/tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pooch/tests/test_core.py b/pooch/tests/test_core.py index eb038000..c3499a6e 100644 --- a/pooch/tests/test_core.py +++ b/pooch/tests/test_core.py @@ -639,7 +639,7 @@ def test_stream_download(fname): def test_stream_download_parallel_subdir(monkeypatch): "Downloading into a new subdirectory in parallel shouldn't fail" - threads = 4 + nthreads = 4 barrier = threading.Barrier(nthreads, timeout=30) makedirs = os.makedirs