From 273eb47cb4f63453941dce487624c34020487383 Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Wed, 10 Jun 2026 12:23:33 +0530 Subject: [PATCH 1/8] Add repository synchronization foundation --- .../test_git_repository_client.py | 39 ++++++ .../harvester_test/test_repository_cache.py | 12 ++ .../utils/harvester/git_repository_client.py | 116 ++++++++++++++++++ .../utils/harvester/repository_cache.py | 7 ++ .../utils/harvester/repository_client.py | 24 ++++ 5 files changed, 198 insertions(+) create mode 100644 application/tests/harvester_test/test_git_repository_client.py create mode 100644 application/tests/harvester_test/test_repository_cache.py create mode 100644 application/utils/harvester/git_repository_client.py create mode 100644 application/utils/harvester/repository_cache.py create mode 100644 application/utils/harvester/repository_client.py diff --git a/application/tests/harvester_test/test_git_repository_client.py b/application/tests/harvester_test/test_git_repository_client.py new file mode 100644 index 000000000..b0aa7678d --- /dev/null +++ b/application/tests/harvester_test/test_git_repository_client.py @@ -0,0 +1,39 @@ +from application.utils.harvester.git_repository_client import ( + GitRepositoryClient, +) + + +def test_repository_url_generation(): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + assert client.repository_url == "https://github.com/OWASP/ASVS.git" + + +def test_local_repository_path(): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + assert str(client.get_local_path()) == ".harvester_cache/owasp/asvs" + + +def test_repository_exists_locally_false(): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + assert client.exists_locally() is False + + +def test_verify_repository_integrity_false(): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + assert client.verify_repository_integrity() is False diff --git a/application/tests/harvester_test/test_repository_cache.py b/application/tests/harvester_test/test_repository_cache.py new file mode 100644 index 000000000..5cf4b745f --- /dev/null +++ b/application/tests/harvester_test/test_repository_cache.py @@ -0,0 +1,12 @@ +from application.utils.harvester.repository_cache import ( + build_repository_cache_path, +) + + +def test_build_repository_cache_path(): + path = build_repository_cache_path( + "OWASP", + "ASVS", + ) + + assert str(path) == ".harvester_cache/owasp/asvs" diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py new file mode 100644 index 000000000..de97836ed --- /dev/null +++ b/application/utils/harvester/git_repository_client.py @@ -0,0 +1,116 @@ +import subprocess +from pathlib import Path + +from .repository_cache import build_repository_cache_path +from .repository_client import RepositoryClient +import logging + +logger = logging.getLogger(__name__) + + +class GitRepositoryClient(RepositoryClient): + def __init__(self, owner: str, repository: str, branch: str = "main") -> None: + self.owner = owner + self.repository = repository + self.branch = branch + + self.local_path = build_repository_cache_path( + owner, + repository, + ) + + @property + def repository_url(self) -> str: + return f"https://github.com/{self.owner}/{self.repository}.git" + + def clone(self) -> None: + logger.info( + "Cloning repository %s/%s", + self.owner, + self.repository, + ) + self.local_path.parent.mkdir(parents=True, exist_ok=True) + + subprocess.run( + [ + "git", + "clone", + "--branch", + self.branch, + self.repository_url, + str(self.local_path), + ], + check=True, + ) + + def fetch(self) -> None: + logger.info( + "Fetching repository %s/%s", + self.owner, + self.repository, + ) + + subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "fetch", + "--all", + ], + check=True, + ) + + def checkout(self, reference: str) -> None: + subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "checkout", + reference, + ], + check=True, + ) + + def get_local_path(self) -> Path: + return self.local_path + + def exists_locally(self) -> bool: + return self.local_path.exists() + + def sync(self) -> None: + logger.info( + "Synchronizing repository %s/%s", + self.owner, + self.repository, + ) + if self.exists_locally(): + self.fetch() + else: + self.clone() + + def get_current_commit_sha(self) -> str: + result = subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "rev-parse", + "HEAD", + ], + capture_output=True, + text=True, + check=True, + ) + + return result.stdout.strip() + + def verify_repository_integrity(self) -> bool: + git_directory = self.local_path / ".git" + + return ( + self.local_path.exists() + and self.local_path.is_dir() + and git_directory.exists() + ) diff --git a/application/utils/harvester/repository_cache.py b/application/utils/harvester/repository_cache.py new file mode 100644 index 000000000..f2ee3343e --- /dev/null +++ b/application/utils/harvester/repository_cache.py @@ -0,0 +1,7 @@ +from pathlib import Path + +CACHE_ROOT = Path(".harvester_cache") + + +def build_repository_cache_path(owner: str, repository: str) -> Path: + return CACHE_ROOT / owner.casefold() / repository.casefold() diff --git a/application/utils/harvester/repository_client.py b/application/utils/harvester/repository_client.py new file mode 100644 index 000000000..6e6df5439 --- /dev/null +++ b/application/utils/harvester/repository_client.py @@ -0,0 +1,24 @@ +from abc import ABC, abstractmethod +from pathlib import Path + + +class RepositoryClient(ABC): + @abstractmethod + def clone(self) -> None: + """Clone repository locally.""" + + @abstractmethod + def fetch(self) -> None: + """Fetch latest remote changes.""" + + @abstractmethod + def checkout(self, reference: str) -> None: + """Checkout repository reference.""" + + @abstractmethod + def get_local_path(self) -> Path: + """Return local repository path.""" + + @abstractmethod + def exists_locally(self) -> bool: + """Check if repository already exists locally.""" From 4a33c163c6abffe61fb873ddfd923c1a0cc5224f Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Wed, 10 Jun 2026 16:03:50 +0530 Subject: [PATCH 2/8] Addressing coderabbit and adding couple of extra gaurdrails --- .../test_git_repository_client.py | 81 ++++++++++++++++++- .../harvester_test/test_repository_cache.py | 18 ++++- application/utils/harvester/__init__.py | 7 ++ .../utils/harvester/git_repository_client.py | 37 ++++++++- .../utils/harvester/repository_cache.py | 11 ++- .../utils/harvester/repository_client.py | 19 ++++- 6 files changed, 162 insertions(+), 11 deletions(-) diff --git a/application/tests/harvester_test/test_git_repository_client.py b/application/tests/harvester_test/test_git_repository_client.py index b0aa7678d..1f622fb39 100644 --- a/application/tests/harvester_test/test_git_repository_client.py +++ b/application/tests/harvester_test/test_git_repository_client.py @@ -2,6 +2,8 @@ GitRepositoryClient, ) +from unittest.mock import patch + def test_repository_url_generation(): client = GitRepositoryClient( @@ -18,7 +20,7 @@ def test_local_repository_path(): repository="ASVS", ) - assert str(client.get_local_path()) == ".harvester_cache/owasp/asvs" + assert str(client.get_local_path()) == ".harvester_cache/owasp/asvs/main" def test_repository_exists_locally_false(): @@ -37,3 +39,80 @@ def test_verify_repository_integrity_false(): ) assert client.verify_repository_integrity() is False + + +def test_sync_clones_when_repository_missing(): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + with ( + patch.object( + client, + "verify_repository_integrity", + return_value=False, + ), + patch.object(client, "clone") as mock_clone, + ): + client.sync() + + mock_clone.assert_called_once() + + +def test_sync_fetches_when_repository_exists(): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + with ( + patch.object( + client, + "verify_repository_integrity", + return_value=True, + ), + patch.object(client, "fetch") as mock_fetch, + ): + client.sync() + + mock_fetch.assert_called_once() + + +@patch("application.utils.harvester.git_repository_client.subprocess.run") +def test_fetch_runs_git_command(mock_run): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + client.fetch() + + mock_run.assert_called_once() + + +@patch("application.utils.harvester.git_repository_client.subprocess.run") +def test_checkout_runs_git_command(mock_run): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + client.checkout("main") + + mock_run.assert_called_once() + + +@patch("application.utils.harvester.git_repository_client.subprocess.run") +def test_get_current_commit_sha_runs_git_command(mock_run): + mock_run.return_value.stdout = "abc123\n" + + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + sha = client.get_current_commit_sha() + + assert sha == "abc123" + mock_run.assert_called_once() diff --git a/application/tests/harvester_test/test_repository_cache.py b/application/tests/harvester_test/test_repository_cache.py index 5cf4b745f..4a33601d9 100644 --- a/application/tests/harvester_test/test_repository_cache.py +++ b/application/tests/harvester_test/test_repository_cache.py @@ -9,4 +9,20 @@ def test_build_repository_cache_path(): "ASVS", ) - assert str(path) == ".harvester_cache/owasp/asvs" + assert str(path) == ".harvester_cache/owasp/asvs/main" + + +def test_different_branches_have_different_cache_paths(): + main_path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="main", + ) + + dev_path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="dev", + ) + + assert main_path != dev_path diff --git a/application/utils/harvester/__init__.py b/application/utils/harvester/__init__.py index 9c74939c7..2ac608b3e 100644 --- a/application/utils/harvester/__init__.py +++ b/application/utils/harvester/__init__.py @@ -14,11 +14,18 @@ ReposFile, ) +from .git_repository_client import GitRepositoryClient +from .repository_client import RepositoryClient +from .repository_cache import build_repository_cache_path + __all__ = [ + "build_repository_cache_path", "ChunkingConfig", "ConfigLoaderError", + "GitRepositoryClient", "PathRules", "PollingConfig", + "RepositoryClient", "RepositoryConfig", "RepositoryValidationError", "ReposFile", diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py index de97836ed..9e6584efc 100644 --- a/application/utils/harvester/git_repository_client.py +++ b/application/utils/harvester/git_repository_client.py @@ -17,6 +17,7 @@ def __init__(self, owner: str, repository: str, branch: str = "main") -> None: self.local_path = build_repository_cache_path( owner, repository, + branch, ) @property @@ -24,12 +25,24 @@ def repository_url(self) -> str: return f"https://github.com/{self.owner}/{self.repository}.git" def clone(self) -> None: + if self.exists_locally(): + logger.warning( + "Repository %s/%s already exists locally", + self.owner, + self.repository, + ) + return + logger.info( "Cloning repository %s/%s", self.owner, self.repository, ) - self.local_path.parent.mkdir(parents=True, exist_ok=True) + + self.local_path.parent.mkdir( + parents=True, + exist_ok=True, + ) subprocess.run( [ @@ -41,6 +54,9 @@ def clone(self) -> None: str(self.local_path), ], check=True, + capture_output=True, + text=True, + timeout=300, ) def fetch(self) -> None: @@ -59,9 +75,19 @@ def fetch(self) -> None: "--all", ], check=True, + capture_output=True, + text=True, + timeout=300, ) def checkout(self, reference: str) -> None: + logger.info( + "Checking out %s in %s/%s", + reference, + self.owner, + self.repository, + ) + subprocess.run( [ "git", @@ -71,6 +97,9 @@ def checkout(self, reference: str) -> None: reference, ], check=True, + capture_output=True, + text=True, + timeout=300, ) def get_local_path(self) -> Path: @@ -85,7 +114,8 @@ def sync(self) -> None: self.owner, self.repository, ) - if self.exists_locally(): + + if self.verify_repository_integrity(): self.fetch() else: self.clone() @@ -99,9 +129,10 @@ def get_current_commit_sha(self) -> str: "rev-parse", "HEAD", ], + check=True, capture_output=True, text=True, - check=True, + timeout=300, ) return result.stdout.strip() diff --git a/application/utils/harvester/repository_cache.py b/application/utils/harvester/repository_cache.py index f2ee3343e..418425681 100644 --- a/application/utils/harvester/repository_cache.py +++ b/application/utils/harvester/repository_cache.py @@ -1,7 +1,12 @@ +import os from pathlib import Path -CACHE_ROOT = Path(".harvester_cache") +CACHE_ROOT = Path(os.getenv("HARVESTER_CACHE_DIR", ".harvester_cache")) -def build_repository_cache_path(owner: str, repository: str) -> Path: - return CACHE_ROOT / owner.casefold() / repository.casefold() +def build_repository_cache_path( + owner: str, + repository: str, + branch: str = "main", +) -> Path: + return CACHE_ROOT / owner.casefold() / repository.casefold() / branch.casefold() diff --git a/application/utils/harvester/repository_client.py b/application/utils/harvester/repository_client.py index 6e6df5439..22ee67700 100644 --- a/application/utils/harvester/repository_client.py +++ b/application/utils/harvester/repository_client.py @@ -9,11 +9,11 @@ def clone(self) -> None: @abstractmethod def fetch(self) -> None: - """Fetch latest remote changes.""" + """Fetch latest repository changes.""" @abstractmethod def checkout(self, reference: str) -> None: - """Checkout repository reference.""" + """Checkout a branch, tag, or commit.""" @abstractmethod def get_local_path(self) -> Path: @@ -21,4 +21,17 @@ def get_local_path(self) -> Path: @abstractmethod def exists_locally(self) -> bool: - """Check if repository already exists locally.""" + """Return whether repository exists locally.""" + + @abstractmethod + def sync(self) -> None: + """Clone if missing, otherwise fetch latest changes.""" + + @abstractmethod + def get_current_commit_sha(self) -> str: + """Return HEAD commit SHA.""" + + @abstractmethod + def verify_repository_integrity(self) -> bool: + """Verify local repository integrity.""" + From 5bba70ff7e438a573d5c5bdcfe24e14eb11a5cee Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Wed, 10 Jun 2026 16:24:25 +0530 Subject: [PATCH 3/8] Addressing coderabbit final --- .../test_git_repository_client.py | 17 +++ .../utils/harvester/git_repository_client.py | 143 +++++++++++------- .../utils/harvester/repository_client.py | 1 - 3 files changed, 107 insertions(+), 54 deletions(-) diff --git a/application/tests/harvester_test/test_git_repository_client.py b/application/tests/harvester_test/test_git_repository_client.py index 1f622fb39..151272345 100644 --- a/application/tests/harvester_test/test_git_repository_client.py +++ b/application/tests/harvester_test/test_git_repository_client.py @@ -116,3 +116,20 @@ def test_get_current_commit_sha_runs_git_command(mock_run): assert sha == "abc123" mock_run.assert_called_once() + + +@patch("application.utils.harvester.git_repository_client.subprocess.run") +def test_clone_runs_git_command(mock_run): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + with patch.object( + client, + "verify_repository_integrity", + return_value=False, + ): + client.clone() + + mock_run.assert_called_once() diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py index 9e6584efc..793fb5535 100644 --- a/application/utils/harvester/git_repository_client.py +++ b/application/utils/harvester/git_repository_client.py @@ -25,7 +25,7 @@ def repository_url(self) -> str: return f"https://github.com/{self.owner}/{self.repository}.git" def clone(self) -> None: - if self.exists_locally(): + if self.verify_repository_integrity(): logger.warning( "Repository %s/%s already exists locally", self.owner, @@ -44,20 +44,29 @@ def clone(self) -> None: exist_ok=True, ) - subprocess.run( - [ - "git", - "clone", - "--branch", - self.branch, - self.repository_url, - str(self.local_path), - ], - check=True, - capture_output=True, - text=True, - timeout=300, - ) + try: + subprocess.run( + [ + "git", + "clone", + "--branch", + self.branch, + self.repository_url, + str(self.local_path), + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.CalledProcessError as exc: + logger.error( + "Failed to clone repository %s/%s: %s", + self.owner, + self.repository, + exc.stderr, + ) + raise def fetch(self) -> None: logger.info( @@ -66,19 +75,28 @@ def fetch(self) -> None: self.repository, ) - subprocess.run( - [ - "git", - "-C", - str(self.local_path), - "fetch", - "--all", - ], - check=True, - capture_output=True, - text=True, - timeout=300, - ) + try: + subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "fetch", + "--all", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.CalledProcessError as exc: + logger.error( + "Failed to fetch repository %s/%s: %s", + self.owner, + self.repository, + exc.stderr, + ) + raise def checkout(self, reference: str) -> None: logger.info( @@ -88,19 +106,29 @@ def checkout(self, reference: str) -> None: self.repository, ) - subprocess.run( - [ - "git", - "-C", - str(self.local_path), - "checkout", + try: + subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "checkout", + reference, + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.CalledProcessError as exc: + logger.error( + "Failed to checkout %s in %s/%s: %s", reference, - ], - check=True, - capture_output=True, - text=True, - timeout=300, - ) + self.owner, + self.repository, + exc.stderr, + ) + raise def get_local_path(self) -> Path: return self.local_path @@ -121,19 +149,28 @@ def sync(self) -> None: self.clone() def get_current_commit_sha(self) -> str: - result = subprocess.run( - [ - "git", - "-C", - str(self.local_path), - "rev-parse", - "HEAD", - ], - check=True, - capture_output=True, - text=True, - timeout=300, - ) + try: + result = subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "rev-parse", + "HEAD", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.CalledProcessError as exc: + logger.error( + "Failed to retrieve commit SHA for %s/%s: %s", + self.owner, + self.repository, + exc.stderr, + ) + raise return result.stdout.strip() diff --git a/application/utils/harvester/repository_client.py b/application/utils/harvester/repository_client.py index 22ee67700..c545cce42 100644 --- a/application/utils/harvester/repository_client.py +++ b/application/utils/harvester/repository_client.py @@ -34,4 +34,3 @@ def get_current_commit_sha(self) -> str: @abstractmethod def verify_repository_integrity(self) -> bool: """Verify local repository integrity.""" - From 42089ef828f59c6371c627f39f38e053345e0764 Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Mon, 6 Jul 2026 17:17:21 +0530 Subject: [PATCH 4/8] test(harvester): align week 2 tests with project conventions --- .../git_repository_client_test.py | 138 ++++++++++++++++++ .../harvester_test/repository_cache_test.py | 37 +++++ .../test_git_repository_client.py | 135 ----------------- .../harvester_test/test_repository_cache.py | 28 ---- 4 files changed, 175 insertions(+), 163 deletions(-) create mode 100644 application/tests/harvester_test/git_repository_client_test.py create mode 100644 application/tests/harvester_test/repository_cache_test.py delete mode 100644 application/tests/harvester_test/test_git_repository_client.py delete mode 100644 application/tests/harvester_test/test_repository_cache.py diff --git a/application/tests/harvester_test/git_repository_client_test.py b/application/tests/harvester_test/git_repository_client_test.py new file mode 100644 index 000000000..4f548a8be --- /dev/null +++ b/application/tests/harvester_test/git_repository_client_test.py @@ -0,0 +1,138 @@ +import unittest +from unittest.mock import patch + +from application.utils.harvester.git_repository_client import ( + GitRepositoryClient, +) + + +class GitRepositoryClientTests(unittest.TestCase): + def test_repository_url_generation(self): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + self.assertEqual( + client.repository_url, + "https://github.com/OWASP/ASVS.git", + ) + + def test_local_repository_path(self): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + self.assertEqual( + str(client.get_local_path()), + ".harvester_cache/owasp/asvs/main", + ) + + def test_repository_exists_locally_false(self): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + self.assertFalse(client.exists_locally()) + + def test_verify_repository_integrity_false(self): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + self.assertFalse(client.verify_repository_integrity()) + + def test_sync_clones_when_repository_missing(self): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + with ( + patch.object( + client, + "verify_repository_integrity", + return_value=False, + ), + patch.object(client, "clone") as mock_clone, + ): + client.sync() + + mock_clone.assert_called_once() + + def test_sync_fetches_when_repository_exists(self): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + with ( + patch.object( + client, + "verify_repository_integrity", + return_value=True, + ), + patch.object(client, "fetch") as mock_fetch, + ): + client.sync() + + mock_fetch.assert_called_once() + + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_fetch_runs_git_command(self, mock_run): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + client.fetch() + + mock_run.assert_called_once() + + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_checkout_runs_git_command(self, mock_run): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + client.checkout("main") + + mock_run.assert_called_once() + + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_get_current_commit_sha_runs_git_command(self, mock_run): + mock_run.return_value.stdout = "abc123\n" + + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + sha = client.get_current_commit_sha() + + self.assertEqual(sha, "abc123") + mock_run.assert_called_once() + + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_clone_runs_git_command(self, mock_run): + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + ) + + with patch.object( + client, + "verify_repository_integrity", + return_value=False, + ): + client.clone() + + mock_run.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/repository_cache_test.py b/application/tests/harvester_test/repository_cache_test.py new file mode 100644 index 000000000..547a4ca23 --- /dev/null +++ b/application/tests/harvester_test/repository_cache_test.py @@ -0,0 +1,37 @@ +import unittest + +from application.utils.harvester.repository_cache import ( + build_repository_cache_path, +) + + +class RepositoryCacheTests(unittest.TestCase): + def test_build_repository_cache_path(self): + path = build_repository_cache_path( + "OWASP", + "ASVS", + ) + + self.assertEqual( + str(path), + ".harvester_cache/owasp/asvs/main", + ) + + def test_different_branches_have_different_cache_paths(self): + main_path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="main", + ) + + dev_path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="dev", + ) + + self.assertNotEqual(main_path, dev_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/test_git_repository_client.py b/application/tests/harvester_test/test_git_repository_client.py deleted file mode 100644 index 151272345..000000000 --- a/application/tests/harvester_test/test_git_repository_client.py +++ /dev/null @@ -1,135 +0,0 @@ -from application.utils.harvester.git_repository_client import ( - GitRepositoryClient, -) - -from unittest.mock import patch - - -def test_repository_url_generation(): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) - - assert client.repository_url == "https://github.com/OWASP/ASVS.git" - - -def test_local_repository_path(): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) - - assert str(client.get_local_path()) == ".harvester_cache/owasp/asvs/main" - - -def test_repository_exists_locally_false(): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) - - assert client.exists_locally() is False - - -def test_verify_repository_integrity_false(): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) - - assert client.verify_repository_integrity() is False - - -def test_sync_clones_when_repository_missing(): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) - - with ( - patch.object( - client, - "verify_repository_integrity", - return_value=False, - ), - patch.object(client, "clone") as mock_clone, - ): - client.sync() - - mock_clone.assert_called_once() - - -def test_sync_fetches_when_repository_exists(): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) - - with ( - patch.object( - client, - "verify_repository_integrity", - return_value=True, - ), - patch.object(client, "fetch") as mock_fetch, - ): - client.sync() - - mock_fetch.assert_called_once() - - -@patch("application.utils.harvester.git_repository_client.subprocess.run") -def test_fetch_runs_git_command(mock_run): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) - - client.fetch() - - mock_run.assert_called_once() - - -@patch("application.utils.harvester.git_repository_client.subprocess.run") -def test_checkout_runs_git_command(mock_run): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) - - client.checkout("main") - - mock_run.assert_called_once() - - -@patch("application.utils.harvester.git_repository_client.subprocess.run") -def test_get_current_commit_sha_runs_git_command(mock_run): - mock_run.return_value.stdout = "abc123\n" - - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) - - sha = client.get_current_commit_sha() - - assert sha == "abc123" - mock_run.assert_called_once() - - -@patch("application.utils.harvester.git_repository_client.subprocess.run") -def test_clone_runs_git_command(mock_run): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) - - with patch.object( - client, - "verify_repository_integrity", - return_value=False, - ): - client.clone() - - mock_run.assert_called_once() diff --git a/application/tests/harvester_test/test_repository_cache.py b/application/tests/harvester_test/test_repository_cache.py deleted file mode 100644 index 4a33601d9..000000000 --- a/application/tests/harvester_test/test_repository_cache.py +++ /dev/null @@ -1,28 +0,0 @@ -from application.utils.harvester.repository_cache import ( - build_repository_cache_path, -) - - -def test_build_repository_cache_path(): - path = build_repository_cache_path( - "OWASP", - "ASVS", - ) - - assert str(path) == ".harvester_cache/owasp/asvs/main" - - -def test_different_branches_have_different_cache_paths(): - main_path = build_repository_cache_path( - owner="OWASP", - repository="ASVS", - branch="main", - ) - - dev_path = build_repository_cache_path( - owner="OWASP", - repository="ASVS", - branch="dev", - ) - - assert main_path != dev_path From 1e38c438a5fa29ad8278d9bd8bcb2d195e365439 Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Sat, 18 Jul 2026 18:01:08 +0530 Subject: [PATCH 5/8] fix(harvester): address repository client review feedback --- application/tests/harvester_test/repository_cache_test.py | 5 +++-- application/utils/harvester/git_repository_client.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/application/tests/harvester_test/repository_cache_test.py b/application/tests/harvester_test/repository_cache_test.py index 547a4ca23..a308239f1 100644 --- a/application/tests/harvester_test/repository_cache_test.py +++ b/application/tests/harvester_test/repository_cache_test.py @@ -1,4 +1,5 @@ import unittest +from pathlib import Path from application.utils.harvester.repository_cache import ( build_repository_cache_path, @@ -13,8 +14,8 @@ def test_build_repository_cache_path(self): ) self.assertEqual( - str(path), - ".harvester_cache/owasp/asvs/main", + path, + Path(".harvester_cache/owasp/asvs/main"), ) def test_different_branches_have_different_cache_paths(self): diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py index 793fb5535..eff7c17d6 100644 --- a/application/utils/harvester/git_repository_client.py +++ b/application/utils/harvester/git_repository_client.py @@ -113,6 +113,7 @@ def checkout(self, reference: str) -> None: "-C", str(self.local_path), "checkout", + "--", reference, ], check=True, From 02b9140462b21775957ccb2b8f0e3f75bac95d48 Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Tue, 21 Jul 2026 18:05:12 +0530 Subject: [PATCH 6/8] Validate repository cache paths and preserve branch identity --- .../git_repository_client_test.py | 16 +++- .../harvester_test/repository_cache_test.py | 46 ++++++++++ .../utils/harvester/git_repository_client.py | 83 +++++++++++++++++-- .../utils/harvester/repository_cache.py | 22 ++++- 4 files changed, 153 insertions(+), 14 deletions(-) diff --git a/application/tests/harvester_test/git_repository_client_test.py b/application/tests/harvester_test/git_repository_client_test.py index 4f548a8be..4929f7810 100644 --- a/application/tests/harvester_test/git_repository_client_test.py +++ b/application/tests/harvester_test/git_repository_client_test.py @@ -90,7 +90,7 @@ def test_fetch_runs_git_command(self, mock_run): client.fetch() - mock_run.assert_called_once() + self.assertEqual(mock_run.call_count, 2) @patch("application.utils.harvester.git_repository_client.subprocess.run") def test_checkout_runs_git_command(self, mock_run): @@ -101,7 +101,19 @@ def test_checkout_runs_git_command(self, mock_run): client.checkout("main") - mock_run.assert_called_once() + mock_run.assert_called_once_with( + [ + "git", + "-C", + str(client.get_local_path()), + "checkout", + "main", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) @patch("application.utils.harvester.git_repository_client.subprocess.run") def test_get_current_commit_sha_runs_git_command(self, mock_run): diff --git a/application/tests/harvester_test/repository_cache_test.py b/application/tests/harvester_test/repository_cache_test.py index a308239f1..d3193d454 100644 --- a/application/tests/harvester_test/repository_cache_test.py +++ b/application/tests/harvester_test/repository_cache_test.py @@ -33,6 +33,52 @@ def test_different_branches_have_different_cache_paths(self): self.assertNotEqual(main_path, dev_path) + def test_case_sensitive_branches_have_different_cache_paths(self): + release_path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="Release", + ) + + release_lower_path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="release", + ) + + self.assertNotEqual(release_path, release_lower_path) + + def test_path_traversal_owner_rejected(self): + with self.assertRaises(ValueError): + build_repository_cache_path( + owner="../../tmp", + repository="ASVS", + ) + + def test_absolute_owner_rejected(self): + with self.assertRaises(ValueError): + build_repository_cache_path( + owner="/tmp", + repository="ASVS", + ) + + def test_invalid_repository_name_rejected(self): + with self.assertRaises(ValueError): + build_repository_cache_path( + owner="OWASP", + repository="../ASVS", + ) + + def test_branch_path_is_encoded(self): + path = build_repository_cache_path( + owner="OWASP", + repository="ASVS", + branch="feature/test", + ) + + self.assertNotIn("feature/test", str(path)) + self.assertIn("feature%2Ftest", str(path)) + if __name__ == "__main__": unittest.main() diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py index eff7c17d6..afc18348f 100644 --- a/application/utils/harvester/git_repository_client.py +++ b/application/utils/harvester/git_repository_client.py @@ -82,13 +82,30 @@ def fetch(self) -> None: "-C", str(self.local_path), "fetch", - "--all", + "origin", + self.branch, + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + + subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "reset", + "--hard", + f"origin/{self.branch}", ], check=True, capture_output=True, text=True, timeout=300, ) + except subprocess.CalledProcessError as exc: logger.error( "Failed to fetch repository %s/%s: %s", @@ -99,6 +116,9 @@ def fetch(self) -> None: raise def checkout(self, reference: str) -> None: + if reference.startswith("-"): + raise ValueError("Invalid git reference") + logger.info( "Checking out %s in %s/%s", reference, @@ -113,7 +133,6 @@ def checkout(self, reference: str) -> None: "-C", str(self.local_path), "checkout", - "--", reference, ], check=True, @@ -176,10 +195,58 @@ def get_current_commit_sha(self) -> str: return result.stdout.strip() def verify_repository_integrity(self) -> bool: - git_directory = self.local_path / ".git" + if not (self.local_path.exists() and self.local_path.is_dir()): + return False - return ( - self.local_path.exists() - and self.local_path.is_dir() - and git_directory.exists() - ) + try: + subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "rev-parse", + "--is-inside-work-tree", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + + remote = subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "config", + "--get", + "remote.origin.url", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ).stdout.strip() + + if remote.rstrip("/") != self.repository_url.rstrip("/"): + return False + + subprocess.run( + [ + "git", + "-C", + str(self.local_path), + "show-ref", + "--verify", + f"refs/remotes/origin/{self.branch}", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + + return True + + except subprocess.CalledProcessError: + return False diff --git a/application/utils/harvester/repository_cache.py b/application/utils/harvester/repository_cache.py index 418425681..94b721a60 100644 --- a/application/utils/harvester/repository_cache.py +++ b/application/utils/harvester/repository_cache.py @@ -1,12 +1,26 @@ import os +import re + from pathlib import Path +from urllib.parse import quote CACHE_ROOT = Path(os.getenv("HARVESTER_CACHE_DIR", ".harvester_cache")) +_VALID_COMPONENT = re.compile(r"^[A-Za-z0-9_.-]+$") + def build_repository_cache_path( - owner: str, - repository: str, - branch: str = "main", + owner: str, repository: str, branch: str = "main" ) -> Path: - return CACHE_ROOT / owner.casefold() / repository.casefold() / branch.casefold() + if not _VALID_COMPONENT.fullmatch(owner): + raise ValueError(f"Invalid repository owner: {owner}") + + if not _VALID_COMPONENT.fullmatch(repository): + raise ValueError(f"Invalid repository name: {repository}") + + encoded_branch = quote(branch, safe="") + candidate = CACHE_ROOT / owner.casefold() / repository.casefold() / encoded_branch + + candidate.resolve().relative_to(CACHE_ROOT.resolve()) + + return candidate From 4604b5b56794cdbc5b52b292071ec133fa57263f Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Tue, 21 Jul 2026 19:20:35 +0530 Subject: [PATCH 7/8] Add integration tests for Git repository client --- .../git_repository_client_integration_test.py | 170 ++++++++++++++++++ .../utils/harvester/git_repository_client.py | 16 +- 2 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 application/tests/harvester_test/git_repository_client_integration_test.py diff --git a/application/tests/harvester_test/git_repository_client_integration_test.py b/application/tests/harvester_test/git_repository_client_integration_test.py new file mode 100644 index 000000000..c11401bc1 --- /dev/null +++ b/application/tests/harvester_test/git_repository_client_integration_test.py @@ -0,0 +1,170 @@ +import subprocess +import tempfile +import unittest +from pathlib import Path + +from application.utils.harvester.git_repository_client import ( + GitRepositoryClient, +) + + +class IntegrationGitRepositoryClient(GitRepositoryClient): + def __init__(self, *args, repository_url: str, **kwargs): + super().__init__(*args, **kwargs) + self._repository_url = repository_url + + @property + def repository_url(self) -> str: + return self._repository_url + + +def git(*args, cwd=None): + subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + + +def git_output(*args, cwd=None): + return subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +class GitRepositoryClientIntegrationTests(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + self.root = Path(self.tempdir.name) + + self.remote = self.root / "remote.git" + self.work = self.root / "work" + self.cache = self.root / "cache" + + git("init", "--bare", self.remote) + + git("clone", self.remote, self.work) + + git("config", "user.name", "Test User", cwd=self.work) + git("config", "user.email", "test@example.com", cwd=self.work) + git("checkout", "-b", "main", cwd=self.work) + + (self.work / "test.txt").write_text("v1") + + git("add", ".", cwd=self.work) + git("commit", "-m", "initial", cwd=self.work) + git("push", "origin", "main", cwd=self.work) + + def tearDown(self): + self.tempdir.cleanup() + + def create_client(self): + return IntegrationGitRepositoryClient( + owner="OWASP", + repository="ASVS", + local_path=self.cache, + repository_url=str(self.remote), + ) + + def test_fetch_updates_worktree_and_commit(self): + client = self.create_client() + + client.clone() + + sha1 = client.get_current_commit_sha() + + self.assertEqual( + (client.get_local_path() / "test.txt").read_text(), + "v1", + ) + + (self.work / "test.txt").write_text("v2") + + git("add", ".", cwd=self.work) + git("commit", "-m", "update", cwd=self.work) + git("push", "origin", "main", cwd=self.work) + + expected_sha = git_output( + "rev-parse", + "HEAD", + cwd=self.work, + ) + + client.fetch() + + self.assertEqual( + client.get_current_commit_sha(), + expected_sha, + ) + + self.assertNotEqual( + sha1, + expected_sha, + ) + + self.assertEqual( + (client.get_local_path() / "test.txt").read_text(), + "v2", + ) + + def test_verify_repository_integrity_rejects_fake_git_directory(self): + fake = self.root / "fake" + + fake.mkdir() + (fake / ".git").mkdir() + + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + local_path=fake, + ) + + self.assertFalse( + client.verify_repository_integrity(), + ) + + def test_verify_repository_integrity_rejects_wrong_origin(self): + other_remote = self.root / "other.git" + + git("init", "--bare", other_remote) + + client = self.create_client() + + client.clone() + + git( + "remote", + "set-url", + "origin", + other_remote, + cwd=client.get_local_path(), + ) + + self.assertFalse( + client.verify_repository_integrity(), + ) + + def test_verify_repository_integrity_rejects_missing_branch(self): + client = IntegrationGitRepositoryClient( + owner="OWASP", + repository="ASVS", + branch="dev", + local_path=self.cache, + repository_url=str(self.remote), + ) + + git("clone", self.remote, self.cache) + + self.assertFalse( + client.verify_repository_integrity(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py index afc18348f..05af12987 100644 --- a/application/utils/harvester/git_repository_client.py +++ b/application/utils/harvester/git_repository_client.py @@ -9,15 +9,21 @@ class GitRepositoryClient(RepositoryClient): - def __init__(self, owner: str, repository: str, branch: str = "main") -> None: + def __init__( + self, + owner: str, + repository: str, + branch: str = "main", + local_path: Path | None = None, + ) -> None: self.owner = owner self.repository = repository self.branch = branch - self.local_path = build_repository_cache_path( - owner, - repository, - branch, + self.local_path = ( + local_path + if local_path is not None + else build_repository_cache_path(owner, repository, branch) ) @property From adf606565fababc7815529ca8a519c1fad3245c0 Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Tue, 21 Jul 2026 22:05:43 +0530 Subject: [PATCH 8/8] Implement atomic repository cloning and locking --- .../git_repository_client_integration_test.py | 36 +++++++++++ .../git_repository_client_test.py | 36 ++++++----- .../utils/harvester/git_repository_client.py | 59 +++++++++++++------ .../utils/harvester/repository_lock.py | 32 ++++++++++ 4 files changed, 131 insertions(+), 32 deletions(-) create mode 100644 application/utils/harvester/repository_lock.py diff --git a/application/tests/harvester_test/git_repository_client_integration_test.py b/application/tests/harvester_test/git_repository_client_integration_test.py index c11401bc1..4803e1d67 100644 --- a/application/tests/harvester_test/git_repository_client_integration_test.py +++ b/application/tests/harvester_test/git_repository_client_integration_test.py @@ -2,6 +2,7 @@ import tempfile import unittest from pathlib import Path +import threading from application.utils.harvester.git_repository_client import ( GitRepositoryClient, @@ -165,6 +166,41 @@ def test_verify_repository_integrity_rejects_missing_branch(self): client.verify_repository_integrity(), ) + def test_sync_serializes_clone_operations(self): + client1 = self.create_client() + client2 = self.create_client() + + exceptions = [] + + def run_sync(client): + try: + client.sync() + except Exception as exc: + exceptions.append(exc) + + t1 = threading.Thread(target=run_sync, args=(client1,)) + t2 = threading.Thread(target=run_sync, args=(client2,)) + + t1.start() + t2.start() + + t1.join() + t2.join() + + self.assertFalse(exceptions, f"Unexpected exceptions: {exceptions}") + + self.assertTrue(client1.verify_repository_integrity()) + self.assertTrue(client2.verify_repository_integrity()) + + self.assertTrue((self.cache / ".git").exists()) + + self.assertEqual( + client1.get_current_commit_sha(), + client2.get_current_commit_sha(), + ) + + self.assertEqual((self.cache / "test.txt").read_text(), "v1") + if __name__ == "__main__": unittest.main() diff --git a/application/tests/harvester_test/git_repository_client_test.py b/application/tests/harvester_test/git_repository_client_test.py index 4929f7810..774715617 100644 --- a/application/tests/harvester_test/git_repository_client_test.py +++ b/application/tests/harvester_test/git_repository_client_test.py @@ -1,6 +1,9 @@ import unittest from unittest.mock import patch +import tempfile +from pathlib import Path + from application.utils.harvester.git_repository_client import ( GitRepositoryClient, ) @@ -30,20 +33,24 @@ def test_local_repository_path(self): ) def test_repository_exists_locally_false(self): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) + with tempfile.TemporaryDirectory() as tmpdir: + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + local_path=Path(tmpdir) / "repo", + ) - self.assertFalse(client.exists_locally()) + self.assertFalse(client.exists_locally()) def test_verify_repository_integrity_false(self): - client = GitRepositoryClient( - owner="OWASP", - repository="ASVS", - ) + with tempfile.TemporaryDirectory() as tmpdir: + client = GitRepositoryClient( + owner="OWASP", + repository="ASVS", + local_path=Path(tmpdir) / "repo", + ) - self.assertFalse(client.verify_repository_integrity()) + self.assertFalse(client.verify_repository_integrity()) def test_sync_clones_when_repository_missing(self): client = GitRepositoryClient( @@ -136,14 +143,13 @@ def test_clone_runs_git_command(self, mock_run): repository="ASVS", ) - with patch.object( - client, - "verify_repository_integrity", - return_value=False, + with ( + patch.object(client, "verify_repository_integrity", return_value=False), + patch.object(client, "is_valid_repository", return_value=True), ): client.clone() - mock_run.assert_called_once() + mock_run.assert_called() if __name__ == "__main__": diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py index 05af12987..1390e6606 100644 --- a/application/utils/harvester/git_repository_client.py +++ b/application/utils/harvester/git_repository_client.py @@ -4,6 +4,10 @@ from .repository_cache import build_repository_cache_path from .repository_client import RepositoryClient import logging +from .repository_lock import repository_lock +import os +import shutil +import tempfile logger = logging.getLogger(__name__) @@ -31,13 +35,6 @@ def repository_url(self) -> str: return f"https://github.com/{self.owner}/{self.repository}.git" def clone(self) -> None: - if self.verify_repository_integrity(): - logger.warning( - "Repository %s/%s already exists locally", - self.owner, - self.repository, - ) - return logger.info( "Cloning repository %s/%s", @@ -50,6 +47,16 @@ def clone(self) -> None: exist_ok=True, ) + self._clone_atomically() + + def _clone_atomically(self) -> None: + temp_path = Path( + tempfile.mkdtemp( + prefix=f"{self.repository}-", + dir=self.local_path.parent, + ) + ) + try: subprocess.run( [ @@ -58,13 +65,23 @@ def clone(self) -> None: "--branch", self.branch, self.repository_url, - str(self.local_path), + str(temp_path), ], check=True, capture_output=True, text=True, timeout=300, ) + + if not self.is_valid_repository(temp_path): + raise RuntimeError("Temporary clone failed integrity verification") + + if self.local_path.exists(): + shutil.rmtree(temp_path) + return + + os.replace(temp_path, self.local_path) + except subprocess.CalledProcessError as exc: logger.error( "Failed to clone repository %s/%s: %s", @@ -74,6 +91,10 @@ def clone(self) -> None: ) raise + finally: + if temp_path.exists(): + shutil.rmtree(temp_path, ignore_errors=True) + def fetch(self) -> None: logger.info( "Fetching repository %s/%s", @@ -169,10 +190,11 @@ def sync(self) -> None: self.repository, ) - if self.verify_repository_integrity(): - self.fetch() - else: - self.clone() + with repository_lock(self.local_path): + if self.verify_repository_integrity(): + self.fetch() + else: + self.clone() def get_current_commit_sha(self) -> str: try: @@ -200,8 +222,8 @@ def get_current_commit_sha(self) -> str: return result.stdout.strip() - def verify_repository_integrity(self) -> bool: - if not (self.local_path.exists() and self.local_path.is_dir()): + def is_valid_repository(self, repository_path: Path) -> bool: + if not (repository_path.exists() and repository_path.is_dir()): return False try: @@ -209,7 +231,7 @@ def verify_repository_integrity(self) -> bool: [ "git", "-C", - str(self.local_path), + str(repository_path), "rev-parse", "--is-inside-work-tree", ], @@ -223,7 +245,7 @@ def verify_repository_integrity(self) -> bool: [ "git", "-C", - str(self.local_path), + str(repository_path), "config", "--get", "remote.origin.url", @@ -241,7 +263,7 @@ def verify_repository_integrity(self) -> bool: [ "git", "-C", - str(self.local_path), + str(repository_path), "show-ref", "--verify", f"refs/remotes/origin/{self.branch}", @@ -256,3 +278,6 @@ def verify_repository_integrity(self) -> bool: except subprocess.CalledProcessError: return False + + def verify_repository_integrity(self) -> bool: + return self.is_valid_repository(self.local_path) diff --git a/application/utils/harvester/repository_lock.py b/application/utils/harvester/repository_lock.py new file mode 100644 index 000000000..9890e6e2e --- /dev/null +++ b/application/utils/harvester/repository_lock.py @@ -0,0 +1,32 @@ +import contextlib +import os +from pathlib import Path + +if os.name == "nt": + import msvcrt +else: + import fcntl + + +@contextlib.contextmanager +def repository_lock(repository_path: Path): + """ + Acquire an exclusive inter-process lock for a repository cache path. + """ + + lock_path = repository_path.with_suffix(".lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + + with lock_path.open("w") as lock_file: + if os.name == "nt": + msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1) + else: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + + try: + yield + finally: + if os.name == "nt": + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + else: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)