diff --git a/application/database/db.py b/application/database/db.py index 70049192c..a7c50b443 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -390,6 +390,10 @@ class HarvesterCheckpoint(BaseModel): # type: ignore def _serialize_json_value(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, str): + return value return flask_json.dumps(value) diff --git a/application/tests/harvester_test/file_filter_test.py b/application/tests/harvester_test/file_filter_test.py new file mode 100644 index 000000000..82412df0d --- /dev/null +++ b/application/tests/harvester_test/file_filter_test.py @@ -0,0 +1,121 @@ +import unittest + +from application.utils.harvester.file_filter import ( + FileFilter, +) + + +class FileFilterTests(unittest.TestCase): + def test_extension_filtering(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + "README.md", + "image.png", + "script.js", + ] + ) + + self.assertEqual( + result, + ["README.md"], + ) + + def test_path_exclusion(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + ".github/workflows/test.yml", + "docs/setup.md", + ] + ) + + self.assertEqual( + result, + ["docs/setup.md"], + ) + + def test_combined_filtering(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + "README.md", + ".github/workflows/README.md", + "node_modules/react/README.md", + "docs/setup.md", + ] + ) + + self.assertEqual( + result, + [ + "README.md", + "docs/setup.md", + ], + ) + + def test_empty_overrides_are_respected(self): + file_filter = FileFilter( + exclude_patterns=[], + allowed_extensions=set(), + ) + + result = file_filter.filter_files( + [ + "README.md", + "image.png", + ] + ) + + self.assertEqual(result, []) + + def test_nested_directory_globs(self): + file_filter = FileFilter() + + result = file_filter.filter_files( + [ + ".github/README.md", + "packages/site/node_modules/README.md", + "docs/archive/old.md", + ".cursor/rules/project.md", + "docs/setup.md", + ] + ) + + self.assertEqual(result, ["docs/setup.md"]) + + def test_explicit_empty_exclusions(self): + file_filter = FileFilter(exclude_patterns=[]) + + result = file_filter.filter_files( + [ + ".github/README.md", + ] + ) + + self.assertEqual( + result, + [".github/README.md"], + ) + + def test_default_instance_isolation(self): + first = FileFilter() + second = FileFilter() + + first.exclude_patterns.append("**/foo/**") + + self.assertNotIn( + "**/foo/**", + second.exclude_patterns, + ) + + def test_empty_extension_raises(self): + with self.assertRaises(ValueError): + FileFilter(allowed_extensions={""}) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/filtering_benchmark_test.py b/application/tests/harvester_test/filtering_benchmark_test.py new file mode 100644 index 000000000..a75c435ec --- /dev/null +++ b/application/tests/harvester_test/filtering_benchmark_test.py @@ -0,0 +1,30 @@ +import unittest + +from application.utils.harvester.file_filter import FileFilter +from application.utils.harvester.filtering_benchmark import FilteringBenchmark + + +class FilteringBenchmarkTests(unittest.TestCase): + def test_filtering_benchmark(self): + files = [ + "README.md", + ".github/workflows/ci.yml", + "docs/guide.md", + "image.png", + "notes.txt", + "package-lock.json", + ] + + benchmark = FilteringBenchmark(file_filter=FileFilter()) + + result = benchmark.run(files) + + self.assertEqual(result.total_files, 6) + self.assertEqual(result.retained_files, 3) + self.assertEqual(result.filtered_files, 3) + self.assertEqual(result.retention_rate, 0.5) + self.assertEqual(result.filtering_rate, 0.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/filtering_metrics_test.py b/application/tests/harvester_test/filtering_metrics_test.py new file mode 100644 index 000000000..252cb0941 --- /dev/null +++ b/application/tests/harvester_test/filtering_metrics_test.py @@ -0,0 +1,35 @@ +import unittest + +from application.utils.harvester.filtering_metrics import ( + FilteringMetricsCollector, +) + + +class FilteringMetricsCollectorTests(unittest.TestCase): + def test_filtering_metrics_collection(self): + collector = FilteringMetricsCollector() + + collector.record_retained() + collector.record_retained() + collector.record_filtered() + + metrics = collector.build() + + self.assertEqual( + metrics.total_files, + 3, + ) + + self.assertEqual( + metrics.retained_files, + 2, + ) + + self.assertEqual( + metrics.filtered_files, + 1, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/git_repository_client_test.py b/application/tests/harvester_test/git_repository_client_test.py index 774715617..941afd67e 100644 --- a/application/tests/harvester_test/git_repository_client_test.py +++ b/application/tests/harvester_test/git_repository_client_test.py @@ -70,7 +70,8 @@ def test_sync_clones_when_repository_missing(self): mock_clone.assert_called_once() - def test_sync_fetches_when_repository_exists(self): + @patch("application.utils.harvester.git_repository_client.subprocess.run") + def test_sync_fetches_when_repository_exists(self, mock_run): client = GitRepositoryClient( owner="OWASP", repository="ASVS", diff --git a/application/utils/harvester/__init__.py b/application/utils/harvester/__init__.py index 2ac608b3e..eda28b629 100644 --- a/application/utils/harvester/__init__.py +++ b/application/utils/harvester/__init__.py @@ -17,12 +17,22 @@ from .git_repository_client import GitRepositoryClient from .repository_client import RepositoryClient from .repository_cache import build_repository_cache_path +from .file_filter import FileFilter +from .filtering_metrics import FilteringMetricsCollector +from .filtering_benchmark import ( + FilteringBenchmark, + FilteringBenchmarkResult, +) __all__ = [ "build_repository_cache_path", "ChunkingConfig", "ConfigLoaderError", "GitRepositoryClient", + "FileFilter", + "FilteringMetricsCollector", + "FilteringBenchmark", + "FilteringBenchmarkResult", "PathRules", "PollingConfig", "RepositoryClient", diff --git a/application/utils/harvester/exclude_patterns.txt b/application/utils/harvester/exclude_patterns.txt index 499850ae8..2b92e85fc 100644 --- a/application/utils/harvester/exclude_patterns.txt +++ b/application/utils/harvester/exclude_patterns.txt @@ -4,7 +4,8 @@ # to filter non-documentation files during harvesting. -**/.git/* +**/.github/** +**/.git/** **/node_modules/** **/__pycache__/** **/.claude/** diff --git a/application/utils/harvester/file_filter.py b/application/utils/harvester/file_filter.py new file mode 100644 index 000000000..f1da0ee79 --- /dev/null +++ b/application/utils/harvester/file_filter.py @@ -0,0 +1,84 @@ +from pathlib import PurePosixPath +from pathlib import Path +import pathspec + +DEFAULT_ALLOWED_EXTENSIONS = { + ".md", + ".mdx", + ".rst", + ".txt", + ".adoc", +} + +DEFAULT_EXCLUDE_PATTERNS = tuple( + line.strip() + for line in ( + Path(__file__) + .with_name("exclude_patterns.txt") + .read_text(encoding="utf-8") + .splitlines() + ) + if line.strip() and not line.lstrip().startswith("#") +) + + +class FileFilter: + def __init__( + self, + exclude_patterns: list[str] | None = None, + allowed_extensions: set[str] | None = None, + ): + self.exclude_patterns: list[str] = ( + list(DEFAULT_EXCLUDE_PATTERNS) + if exclude_patterns is None + else list(exclude_patterns) + ) + + self.allowed_extensions: set[str] = ( + set(DEFAULT_ALLOWED_EXTENSIONS) + if allowed_extensions is None + else set(allowed_extensions) + ) + + self._validate_patterns() + + try: + self._exclude_spec = pathspec.PathSpec.from_lines( + "gitignore", + self.exclude_patterns, + ) + except Exception as exc: + raise ValueError("Invalid exclude glob") from exc + + def _validate_patterns(self) -> None: + if any(not pattern for pattern in self.exclude_patterns): + raise ValueError("Exclude pattern cannot be empty") + + if any(not extension for extension in self.allowed_extensions): + raise ValueError("Allowed extension cannot be empty") + + def _normalize_path(self, file_path: str) -> str: + return PurePosixPath(file_path).as_posix() + + def is_excluded_by_pattern(self, file_path: str) -> bool: + normalized = self._normalize_path(file_path) + return self._exclude_spec.match_file(normalized) + + def is_allowed_extension(self, file_path: str) -> bool: + return any( + file_path.endswith(extension) for extension in self.allowed_extensions + ) + + def filter_files(self, files: list[str]) -> list[str]: + filtered = [] + + for file_path in files: + if self.is_excluded_by_pattern(file_path): + continue + + if not self.is_allowed_extension(file_path): + continue + + filtered.append(file_path) + + return filtered diff --git a/application/utils/harvester/filtering_benchmark.py b/application/utils/harvester/filtering_benchmark.py new file mode 100644 index 000000000..5de2e7b69 --- /dev/null +++ b/application/utils/harvester/filtering_benchmark.py @@ -0,0 +1,32 @@ +from dataclasses import dataclass + +from .file_filter import FileFilter + + +@dataclass +class FilteringBenchmarkResult: + total_files: int + retained_files: int + filtered_files: int + retention_rate: float + filtering_rate: float + + +class FilteringBenchmark: + def __init__(self, file_filter: FileFilter): + self.file_filter = file_filter + + def run(self, file_paths: list[str]) -> FilteringBenchmarkResult: + retained = self.file_filter.filter_files(file_paths) + + total = len(file_paths) + retained_count = len(retained) + filtered_count = total - retained_count + + return FilteringBenchmarkResult( + total_files=total, + retained_files=retained_count, + filtered_files=filtered_count, + retention_rate=(retained_count / total if total else 0.0), + filtering_rate=(filtered_count / total if total else 0.0), + ) diff --git a/application/utils/harvester/filtering_metrics.py b/application/utils/harvester/filtering_metrics.py new file mode 100644 index 000000000..d496c3e2a --- /dev/null +++ b/application/utils/harvester/filtering_metrics.py @@ -0,0 +1,23 @@ +from .models import FilteringMetrics + + +class FilteringMetricsCollector: + def __init__(self): + self.total_files = 0 + self.retained_files = 0 + self.filtered_files = 0 + + def record_retained(self) -> None: + self.total_files += 1 + self.retained_files += 1 + + def record_filtered(self) -> None: + self.total_files += 1 + self.filtered_files += 1 + + def build(self) -> FilteringMetrics: + return FilteringMetrics( + total_files=self.total_files, + retained_files=self.retained_files, + filtered_files=self.filtered_files, + ) diff --git a/application/utils/harvester/models.py b/application/utils/harvester/models.py index 2050913c7..227c0d64e 100644 --- a/application/utils/harvester/models.py +++ b/application/utils/harvester/models.py @@ -1,5 +1,6 @@ from dataclasses import dataclass from datetime import datetime +from pydantic import BaseModel @dataclass(slots=True) @@ -18,3 +19,9 @@ class RepositoryChangeSet: repository_id: str commit_sha: str modified_files: list[str] + + +class FilteringMetrics(BaseModel): + total_files: int + retained_files: int + filtered_files: int