diff --git a/pooch/core.py b/pooch/core.py index d358244f..35f6814e 100644 --- a/pooch/core.py +++ b/pooch/core.py @@ -812,7 +812,7 @@ def stream_download( # Ensure the parent directory exists in case the file is in a subdirectory. # Otherwise, move will cause an error. if not fname.parent.exists(): - os.makedirs(str(fname.parent)) + os.makedirs(str(fname.parent), exist_ok=True) download_attempts = 1 + retry_if_failed max_wait = 10 for i in range(download_attempts): diff --git a/pooch/tests/test_core.py b/pooch/tests/test_core.py index 0009851e..c3499a6e 100644 --- a/pooch/tests/test_core.py +++ b/pooch/tests/test_core.py @@ -12,6 +12,8 @@ import hashlib import os import re +import threading +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from tempfile import TemporaryDirectory @@ -635,6 +637,40 @@ def test_stream_download(fname): check_tiny_data(str(destination)) +def test_stream_download_parallel_subdir(monkeypatch): + "Downloading into a new subdirectory in parallel shouldn't fail" + nthreads = 4 + barrier = threading.Barrier(nthreads, timeout=30) + makedirs = os.makedirs + + def mockmakedirs(path, exist_ok=False): + "Wait for all the threads before creating the directory" + barrier.wait() + makedirs(path, exist_ok=exist_ok) + + def downloader(url, output_file, pooch): # noqa: ARG001 + "Write some data instead of downloading it" + Path(output_file).write_text("some data", encoding="utf-8") + + monkeypatch.setattr(os, "makedirs", mockmakedirs) + + with TemporaryDirectory() as local_store: + destinations = [ + Path(local_store) / "subdir" / f"data-{i}.txt" for i in range(nthreads) + ] + with ThreadPoolExecutor(max_workers=nthreads) as executor: + futures = [ + executor.submit( + stream_download, "unused", dest, None, downloader, pooch=None + ) + for dest in destinations + ] + for future in futures: + future.result() + for dest in destinations: + assert dest.read_text(encoding="utf-8") == "some data" + + @pytest.mark.network @pytest.mark.parametrize( "url",