From e2d6ae497d626ce3fb5a3a5bc3226ec7f34e1c5d Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Thu, 23 Jul 2026 19:33:23 +0530 Subject: [PATCH 1/7] feat(db): add artifact ingestion persistence models --- application/database/db.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/application/database/db.py b/application/database/db.py index 70049192c..a7c50b443 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -390,6 +390,10 @@ class HarvesterCheckpoint(BaseModel): # type: ignore def _serialize_json_value(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, str): + return value return flask_json.dumps(value) From 02c4b48ffc3b0145611a161c823043b2280079de Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Wed, 8 Jul 2026 14:10:50 +0530 Subject: [PATCH 2/7] feat(harvester): implement repository file filtering --- .../tests/harvester_test/file_filter_test.py | 62 +++++++++++++++++++ .../filtering_benchmark_test.py | 49 +++++++++++++++ .../harvester_test/filtering_metrics_test.py | 35 +++++++++++ application/utils/harvester/__init__.py | 10 +++ application/utils/harvester/file_filter.py | 54 ++++++++++++++++ .../utils/harvester/filtering_benchmark.py | 38 ++++++++++++ .../utils/harvester/filtering_metrics.py | 23 +++++++ application/utils/harvester/models.py | 7 +++ 8 files changed, 278 insertions(+) create mode 100644 application/tests/harvester_test/file_filter_test.py create mode 100644 application/tests/harvester_test/filtering_benchmark_test.py create mode 100644 application/tests/harvester_test/filtering_metrics_test.py create mode 100644 application/utils/harvester/file_filter.py create mode 100644 application/utils/harvester/filtering_benchmark.py create mode 100644 application/utils/harvester/filtering_metrics.py diff --git a/application/tests/harvester_test/file_filter_test.py b/application/tests/harvester_test/file_filter_test.py new file mode 100644 index 000000000..ed7958fa9 --- /dev/null +++ b/application/tests/harvester_test/file_filter_test.py @@ -0,0 +1,62 @@ +import unittest + +from application.utils.harvester.file_filter import ( + FileFilter, +) + + +class FileFilterTests(unittest.TestCase): + def test_extension_filtering(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + "README.md", + "image.png", + "script.js", + ] + ) + + self.assertEqual( + result, + ["README.md"], + ) + + def test_regex_filtering(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + ".github/workflows/test.yml", + "docs/setup.md", + ] + ) + + self.assertEqual( + result, + ["docs/setup.md"], + ) + + def test_combined_filtering(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + "README.md", + ".github/workflows/test.yml", + "node_modules/react/index.js", + "docs/setup.md", + ] + ) + + self.assertEqual( + result, + [ + "README.md", + "docs/setup.md", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/filtering_benchmark_test.py b/application/tests/harvester_test/filtering_benchmark_test.py new file mode 100644 index 000000000..c34105270 --- /dev/null +++ b/application/tests/harvester_test/filtering_benchmark_test.py @@ -0,0 +1,49 @@ +import unittest + +from application.utils.harvester.file_filter import ( + FileFilter, +) +from application.utils.harvester.models import ( + FilteringMetrics, +) + + +class FilteringBenchmarkTests(unittest.TestCase): + def test_filtering_benchmark(self): + files = [ + "README.md", + ".github/workflows/ci.yml", + "docs/guide.md", + "image.png", + "notes.txt", + "package-lock.json", + ] + + file_filter = FileFilter() + + retained_files = file_filter.filter_files(files) + + metrics = FilteringMetrics( + total_files=len(files), + retained_files=len(retained_files), + filtered_files=len(files) - len(retained_files), + ) + + self.assertEqual( + metrics.total_files, + 6, + ) + + self.assertEqual( + metrics.retained_files, + 3, + ) + + self.assertEqual( + metrics.filtered_files, + 3, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/filtering_metrics_test.py b/application/tests/harvester_test/filtering_metrics_test.py new file mode 100644 index 000000000..252cb0941 --- /dev/null +++ b/application/tests/harvester_test/filtering_metrics_test.py @@ -0,0 +1,35 @@ +import unittest + +from application.utils.harvester.filtering_metrics import ( + FilteringMetricsCollector, +) + + +class FilteringMetricsCollectorTests(unittest.TestCase): + def test_filtering_metrics_collection(self): + collector = FilteringMetricsCollector() + + collector.record_retained() + collector.record_retained() + collector.record_filtered() + + metrics = collector.build() + + self.assertEqual( + metrics.total_files, + 3, + ) + + self.assertEqual( + metrics.retained_files, + 2, + ) + + self.assertEqual( + metrics.filtered_files, + 1, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/harvester/__init__.py b/application/utils/harvester/__init__.py index 2ac608b3e..eda28b629 100644 --- a/application/utils/harvester/__init__.py +++ b/application/utils/harvester/__init__.py @@ -17,12 +17,22 @@ from .git_repository_client import GitRepositoryClient from .repository_client import RepositoryClient from .repository_cache import build_repository_cache_path +from .file_filter import FileFilter +from .filtering_metrics import FilteringMetricsCollector +from .filtering_benchmark import ( + FilteringBenchmark, + FilteringBenchmarkResult, +) __all__ = [ "build_repository_cache_path", "ChunkingConfig", "ConfigLoaderError", "GitRepositoryClient", + "FileFilter", + "FilteringMetricsCollector", + "FilteringBenchmark", + "FilteringBenchmarkResult", "PathRules", "PollingConfig", "RepositoryClient", diff --git a/application/utils/harvester/file_filter.py b/application/utils/harvester/file_filter.py new file mode 100644 index 000000000..706dffe2c --- /dev/null +++ b/application/utils/harvester/file_filter.py @@ -0,0 +1,54 @@ +import re + +DEFAULT_ALLOWED_EXTENSIONS = { + ".md", + ".mdx", + ".rst", + ".txt", + ".adoc", +} + +DEFAULT_EXCLUDE_PATTERNS = [ + r"^\.github/", + r"^\.git/", + r"^node_modules/", + r"^dist/", + r"^build/", + r"^coverage/", + r"^vendor/", + r".*package-lock\.json$", + r".*yarn\.lock$", + r".*pnpm-lock\.yaml$", +] + + +class FileFilter: + def __init__( + self, + exclude_patterns: list[str] | None = None, + allowed_extensions: set[str] | None = None, + ): + self.exclude_patterns = exclude_patterns or DEFAULT_EXCLUDE_PATTERNS + self.allowed_extensions = allowed_extensions or DEFAULT_ALLOWED_EXTENSIONS + + def is_excluded_by_pattern(self, file_path: str) -> bool: + return any(re.search(pattern, file_path) for pattern in self.exclude_patterns) + + def is_allowed_extension(self, file_path: str) -> bool: + return any( + file_path.endswith(extension) for extension in self.allowed_extensions + ) + + def filter_files(self, files: list[str]) -> list[str]: + filtered_files = [] + + for file_path in files: + if self.is_excluded_by_pattern(file_path): + continue + + if not self.is_allowed_extension(file_path): + continue + + filtered_files.append(file_path) + + return filtered_files diff --git a/application/utils/harvester/filtering_benchmark.py b/application/utils/harvester/filtering_benchmark.py new file mode 100644 index 000000000..2fc095e4e --- /dev/null +++ b/application/utils/harvester/filtering_benchmark.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass + +from .file_filter import FileFilter +from .models import FilteringMetrics + + +@dataclass +class FilteringBenchmarkResult: + total_files: int + retained_files: int + filtered_files: int + retention_rate: float + filtering_rate: float + + +class FilteringBenchmark: + def __init__( + self, + file_filter: FileFilter, + metrics: FilteringMetrics, + ): + self.file_filter = file_filter + self.metrics = metrics + + def run(self, file_paths: list[str]) -> FilteringBenchmarkResult: + retained = self.file_filter.filter_files(file_paths) + + total = len(file_paths) + retained_count = len(retained) + filtered_count = total - retained_count + + return FilteringBenchmarkResult( + total_files=total, + retained_files=retained_count, + filtered_files=filtered_count, + retention_rate=(retained_count / total if total else 0.0), + filtering_rate=(filtered_count / total if total else 0.0), + ) diff --git a/application/utils/harvester/filtering_metrics.py b/application/utils/harvester/filtering_metrics.py new file mode 100644 index 000000000..d496c3e2a --- /dev/null +++ b/application/utils/harvester/filtering_metrics.py @@ -0,0 +1,23 @@ +from .models import FilteringMetrics + + +class FilteringMetricsCollector: + def __init__(self): + self.total_files = 0 + self.retained_files = 0 + self.filtered_files = 0 + + def record_retained(self) -> None: + self.total_files += 1 + self.retained_files += 1 + + def record_filtered(self) -> None: + self.total_files += 1 + self.filtered_files += 1 + + def build(self) -> FilteringMetrics: + return FilteringMetrics( + total_files=self.total_files, + retained_files=self.retained_files, + filtered_files=self.filtered_files, + ) diff --git a/application/utils/harvester/models.py b/application/utils/harvester/models.py index 2050913c7..227c0d64e 100644 --- a/application/utils/harvester/models.py +++ b/application/utils/harvester/models.py @@ -1,5 +1,6 @@ from dataclasses import dataclass from datetime import datetime +from pydantic import BaseModel @dataclass(slots=True) @@ -18,3 +19,9 @@ class RepositoryChangeSet: repository_id: str commit_sha: str modified_files: list[str] + + +class FilteringMetrics(BaseModel): + total_files: int + retained_files: int + filtered_files: int From 589a23e84a17c702229a1284aa5909f41b2b4fe6 Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Sat, 18 Jul 2026 21:08:19 +0530 Subject: [PATCH 3/7] fix(harvester): improve filtering benchmark and sync behavior --- .../filtering_benchmark_test.py | 37 +++++-------------- .../git_repository_client_test.py | 18 ++++++++- .../utils/harvester/filtering_benchmark.py | 8 +--- 3 files changed, 27 insertions(+), 36 deletions(-) diff --git a/application/tests/harvester_test/filtering_benchmark_test.py b/application/tests/harvester_test/filtering_benchmark_test.py index c34105270..a75c435ec 100644 --- a/application/tests/harvester_test/filtering_benchmark_test.py +++ b/application/tests/harvester_test/filtering_benchmark_test.py @@ -1,11 +1,7 @@ import unittest -from application.utils.harvester.file_filter import ( - FileFilter, -) -from application.utils.harvester.models import ( - FilteringMetrics, -) +from application.utils.harvester.file_filter import FileFilter +from application.utils.harvester.filtering_benchmark import FilteringBenchmark class FilteringBenchmarkTests(unittest.TestCase): @@ -19,30 +15,15 @@ def test_filtering_benchmark(self): "package-lock.json", ] - file_filter = FileFilter() + benchmark = FilteringBenchmark(file_filter=FileFilter()) - retained_files = file_filter.filter_files(files) + result = benchmark.run(files) - metrics = FilteringMetrics( - total_files=len(files), - retained_files=len(retained_files), - filtered_files=len(files) - len(retained_files), - ) - - self.assertEqual( - metrics.total_files, - 6, - ) - - self.assertEqual( - metrics.retained_files, - 3, - ) - - self.assertEqual( - metrics.filtered_files, - 3, - ) + self.assertEqual(result.total_files, 6) + self.assertEqual(result.retained_files, 3) + self.assertEqual(result.filtered_files, 3) + self.assertEqual(result.retention_rate, 0.5) + self.assertEqual(result.filtering_rate, 0.5) if __name__ == "__main__": diff --git a/application/tests/harvester_test/git_repository_client_test.py b/application/tests/harvester_test/git_repository_client_test.py index 774715617..1121c973f 100644 --- a/application/tests/harvester_test/git_repository_client_test.py +++ b/application/tests/harvester_test/git_repository_client_test.py @@ -70,7 +70,8 @@ def test_sync_clones_when_repository_missing(self): mock_clone.assert_called_once() - def test_sync_fetches_when_repository_exists(self): + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_sync_fetches_when_repository_exists(self, mock_run): client = GitRepositoryClient( owner="OWASP", repository="ASVS", @@ -88,6 +89,21 @@ def test_sync_fetches_when_repository_exists(self): mock_fetch.assert_called_once() + mock_run.assert_called_once_with( + [ + "git", + "-C", + str(client.get_local_path()), + "reset", + "--hard", + "origin/main", + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + @patch("application.utils.harvester.git_repository_client.subprocess.run") def test_fetch_runs_git_command(self, mock_run): client = GitRepositoryClient( diff --git a/application/utils/harvester/filtering_benchmark.py b/application/utils/harvester/filtering_benchmark.py index 2fc095e4e..5de2e7b69 100644 --- a/application/utils/harvester/filtering_benchmark.py +++ b/application/utils/harvester/filtering_benchmark.py @@ -1,7 +1,6 @@ from dataclasses import dataclass from .file_filter import FileFilter -from .models import FilteringMetrics @dataclass @@ -14,13 +13,8 @@ class FilteringBenchmarkResult: class FilteringBenchmark: - def __init__( - self, - file_filter: FileFilter, - metrics: FilteringMetrics, - ): + def __init__(self, file_filter: FileFilter): self.file_filter = file_filter - self.metrics = metrics def run(self, file_paths: list[str]) -> FilteringBenchmarkResult: retained = self.file_filter.filter_files(file_paths) From 8630bd9e76d64c04b15e0f41ef95a559846bf9e9 Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Wed, 29 Jul 2026 12:10:39 +0530 Subject: [PATCH 4/7] fix(harvester): isolate filter defaults and preserve empty overrides --- .../tests/harvester_test/file_filter_test.py | 25 +++++++++++++++++++ application/utils/harvester/file_filter.py | 11 ++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/application/tests/harvester_test/file_filter_test.py b/application/tests/harvester_test/file_filter_test.py index ed7958fa9..c6500c621 100644 --- a/application/tests/harvester_test/file_filter_test.py +++ b/application/tests/harvester_test/file_filter_test.py @@ -57,6 +57,31 @@ def test_combined_filtering(self): ], ) + def test_empty_overrides_are_respected(self): + file_filter = FileFilter( + exclude_patterns=[], + allowed_extensions=set(), + ) + + result = file_filter.filter_files( + [ + "README.md", + "image.png", + ] + ) + + self.assertEqual(result, []) + + def test_default_instances_are_isolated(self): + first = FileFilter() + second = FileFilter() + + first.exclude_patterns.append("custom") + first.allowed_extensions.add(".pdf") + + self.assertNotIn("custom", second.exclude_patterns) + self.assertNotIn(".pdf", second.allowed_extensions) + if __name__ == "__main__": unittest.main() diff --git a/application/utils/harvester/file_filter.py b/application/utils/harvester/file_filter.py index 706dffe2c..7bf97b34b 100644 --- a/application/utils/harvester/file_filter.py +++ b/application/utils/harvester/file_filter.py @@ -28,8 +28,15 @@ def __init__( exclude_patterns: list[str] | None = None, allowed_extensions: set[str] | None = None, ): - self.exclude_patterns = exclude_patterns or DEFAULT_EXCLUDE_PATTERNS - self.allowed_extensions = allowed_extensions or DEFAULT_ALLOWED_EXTENSIONS + if exclude_patterns is None: + self.exclude_patterns = list(DEFAULT_EXCLUDE_PATTERNS) + else: + self.exclude_patterns = list(exclude_patterns) + + if allowed_extensions is None: + self.allowed_extensions = set(DEFAULT_ALLOWED_EXTENSIONS) + else: + self.allowed_extensions = set(allowed_extensions) def is_excluded_by_pattern(self, file_path: str) -> bool: return any(re.search(pattern, file_path) for pattern in self.exclude_patterns) From 4e811bb3450b8767524c445304165d8344dbbec0 Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Wed, 29 Jul 2026 13:50:49 +0530 Subject: [PATCH 5/7] Use glob-based path filtering for harvester --- .../tests/harvester_test/file_filter_test.py | 46 +++++++++--- .../utils/harvester/exclude_patterns.txt | 3 +- application/utils/harvester/file_filter.py | 70 ++++++++++++------- 3 files changed, 85 insertions(+), 34 deletions(-) diff --git a/application/tests/harvester_test/file_filter_test.py b/application/tests/harvester_test/file_filter_test.py index c6500c621..9594841a2 100644 --- a/application/tests/harvester_test/file_filter_test.py +++ b/application/tests/harvester_test/file_filter_test.py @@ -22,7 +22,7 @@ def test_extension_filtering(self): ["README.md"], ) - def test_regex_filtering(self): + def test_path_exclusion(self): file_filter = FileFilter() result = file_filter.filter_files( @@ -43,8 +43,8 @@ def test_combined_filtering(self): result = file_filter.filter_files( [ "README.md", - ".github/workflows/test.yml", - "node_modules/react/index.js", + ".github/workflows/README.md", + "node_modules/react/README.md", "docs/setup.md", ] ) @@ -72,15 +72,45 @@ def test_empty_overrides_are_respected(self): self.assertEqual(result, []) - def test_default_instances_are_isolated(self): + def test_nested_directory_globs(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + ".github/README.md", + "packages/site/node_modules/README.md", + "docs/archive/old.md", + ".cursor/rules/project.md", + "docs/setup.md", + ] + ) + + self.assertEqual(result, ["docs/setup.md"]) + + def test_explicit_empty_exclusions(self): + file_filter = FileFilter(exclude_patterns=[]) + + result = file_filter.filter_files( + [ + ".github/README.md", + ] + ) + + self.assertEqual( + result, + [".github/README.md"], + ) + + def test_default_instance_isolation(self): first = FileFilter() second = FileFilter() - first.exclude_patterns.append("custom") - first.allowed_extensions.add(".pdf") + first.exclude_patterns.append("**/foo/**") - self.assertNotIn("custom", second.exclude_patterns) - self.assertNotIn(".pdf", second.allowed_extensions) + self.assertNotIn( + "**/foo/**", + second.exclude_patterns, + ) if __name__ == "__main__": diff --git a/application/utils/harvester/exclude_patterns.txt b/application/utils/harvester/exclude_patterns.txt index 499850ae8..2b92e85fc 100644 --- a/application/utils/harvester/exclude_patterns.txt +++ b/application/utils/harvester/exclude_patterns.txt @@ -4,7 +4,8 @@ # to filter non-documentation files during harvesting. -**/.git/* +**/.github/** +**/.git/** **/node_modules/** **/__pycache__/** **/.claude/** diff --git a/application/utils/harvester/file_filter.py b/application/utils/harvester/file_filter.py index 7bf97b34b..086d54b9b 100644 --- a/application/utils/harvester/file_filter.py +++ b/application/utils/harvester/file_filter.py @@ -1,4 +1,6 @@ -import re +from pathlib import PurePosixPath +from pathlib import Path +import pathspec DEFAULT_ALLOWED_EXTENSIONS = { ".md", @@ -8,18 +10,16 @@ ".adoc", } -DEFAULT_EXCLUDE_PATTERNS = [ - r"^\.github/", - r"^\.git/", - r"^node_modules/", - r"^dist/", - r"^build/", - r"^coverage/", - r"^vendor/", - r".*package-lock\.json$", - r".*yarn\.lock$", - r".*pnpm-lock\.yaml$", -] +DEFAULT_EXCLUDE_PATTERNS = tuple( + line.strip() + for line in ( + Path(__file__) + .with_name("exclude_patterns.txt") + .read_text(encoding="utf-8") + .splitlines() + ) + if line.strip() and not line.lstrip().startswith("#") +) class FileFilter: @@ -28,18 +28,38 @@ def __init__( exclude_patterns: list[str] | None = None, allowed_extensions: set[str] | None = None, ): - if exclude_patterns is None: - self.exclude_patterns = list(DEFAULT_EXCLUDE_PATTERNS) - else: - self.exclude_patterns = list(exclude_patterns) + self.exclude_patterns: list[str] = ( + list(DEFAULT_EXCLUDE_PATTERNS) + if exclude_patterns is None + else list(exclude_patterns) + ) + + self.allowed_extensions: set[str] = ( + set(DEFAULT_ALLOWED_EXTENSIONS) + if allowed_extensions is None + else set(allowed_extensions) + ) + + self._validate_patterns() + + try: + self._exclude_spec = pathspec.PathSpec.from_lines( + "gitignore", + self.exclude_patterns, + ) + except Exception as exc: + raise ValueError("Invalid exclude glob") from exc + + def _validate_patterns(self) -> None: + if any(not pattern for pattern in self.exclude_patterns): + raise ValueError("Exclude pattern cannot be empty") - if allowed_extensions is None: - self.allowed_extensions = set(DEFAULT_ALLOWED_EXTENSIONS) - else: - self.allowed_extensions = set(allowed_extensions) + def _normalize_path(self, file_path: str) -> str: + return PurePosixPath(file_path).as_posix() def is_excluded_by_pattern(self, file_path: str) -> bool: - return any(re.search(pattern, file_path) for pattern in self.exclude_patterns) + normalized = self._normalize_path(file_path) + return self._exclude_spec.match_file(normalized) def is_allowed_extension(self, file_path: str) -> bool: return any( @@ -47,7 +67,7 @@ def is_allowed_extension(self, file_path: str) -> bool: ) def filter_files(self, files: list[str]) -> list[str]: - filtered_files = [] + filtered = [] for file_path in files: if self.is_excluded_by_pattern(file_path): @@ -56,6 +76,6 @@ def filter_files(self, files: list[str]) -> list[str]: if not self.is_allowed_extension(file_path): continue - filtered_files.append(file_path) + filtered.append(file_path) - return filtered_files + return filtered From 930f2a1fbd26fb9e91e70553f699285ca0890b46 Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Wed, 29 Jul 2026 14:10:24 +0530 Subject: [PATCH 6/7] feat(harvester): improve file filtering and exclusions --- .../harvester_test/git_repository_client_test.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/application/tests/harvester_test/git_repository_client_test.py b/application/tests/harvester_test/git_repository_client_test.py index 1121c973f..941afd67e 100644 --- a/application/tests/harvester_test/git_repository_client_test.py +++ b/application/tests/harvester_test/git_repository_client_test.py @@ -89,21 +89,6 @@ def test_sync_fetches_when_repository_exists(self, mock_run): mock_fetch.assert_called_once() - mock_run.assert_called_once_with( - [ - "git", - "-C", - str(client.get_local_path()), - "reset", - "--hard", - "origin/main", - ], - check=True, - capture_output=True, - text=True, - timeout=300, - ) - @patch("application.utils.harvester.git_repository_client.subprocess.run") def test_fetch_runs_git_command(self, mock_run): client = GitRepositoryClient( From d108b7edfebc35c8eecff25762ecbb3a53679514 Mon Sep 17 00:00:00 2001 From: ParthAggarwal16 Date: Wed, 29 Jul 2026 17:03:36 +0530 Subject: [PATCH 7/7] test: strengthen FileFilter validation --- application/tests/harvester_test/file_filter_test.py | 4 ++++ application/utils/harvester/file_filter.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/application/tests/harvester_test/file_filter_test.py b/application/tests/harvester_test/file_filter_test.py index 9594841a2..82412df0d 100644 --- a/application/tests/harvester_test/file_filter_test.py +++ b/application/tests/harvester_test/file_filter_test.py @@ -112,6 +112,10 @@ def test_default_instance_isolation(self): second.exclude_patterns, ) + def test_empty_extension_raises(self): + with self.assertRaises(ValueError): + FileFilter(allowed_extensions={""}) + if __name__ == "__main__": unittest.main() diff --git a/application/utils/harvester/file_filter.py b/application/utils/harvester/file_filter.py index 086d54b9b..f1da0ee79 100644 --- a/application/utils/harvester/file_filter.py +++ b/application/utils/harvester/file_filter.py @@ -54,6 +54,9 @@ def _validate_patterns(self) -> None: if any(not pattern for pattern in self.exclude_patterns): raise ValueError("Exclude pattern cannot be empty") + if any(not extension for extension in self.allowed_extensions): + raise ValueError("Allowed extension cannot be empty") + def _normalize_path(self, file_path: str) -> str: return PurePosixPath(file_path).as_posix()