GSoC Module A : week 4 : feat(harvester): implement repository file filtering (stacked on top of #985)

application/database/db.py

@@ -390,6 +390,10 @@ class HarvesterCheckpoint(BaseModel):  # type: ignore
 
 
 def _serialize_json_value(value: Any) -> str:
+    if value is None:
+        return "null"
+    if isinstance(value, str):
+        return value
     return flask_json.dumps(value)
 
 

application/tests/harvester_test/file_filter_test.py

@@ -0,0 +1,121 @@
+import unittest
+
+from application.utils.harvester.file_filter import (
+    FileFilter,
+)
+
+
+class FileFilterTests(unittest.TestCase):
+    def test_extension_filtering(self):
+        file_filter = FileFilter()
+
+        result = file_filter.filter_files(
+            [
+                "README.md",
+                "image.png",
+                "script.js",
+            ]
+        )
+
+        self.assertEqual(
+            result,
+            ["README.md"],
+        )
+
+    def test_path_exclusion(self):
+        file_filter = FileFilter()
+
+        result = file_filter.filter_files(
+            [
+                ".github/workflows/test.yml",
+                "docs/setup.md",
+            ]
+        )
+
+        self.assertEqual(
+            result,
+            ["docs/setup.md"],
+        )
+
+    def test_combined_filtering(self):
+        file_filter = FileFilter()
+
+        result = file_filter.filter_files(
+            [
+                "README.md",
+                ".github/workflows/README.md",
+                "node_modules/react/README.md",
+                "docs/setup.md",
+            ]
+        )
+
+        self.assertEqual(
+            result,
+            [
+                "README.md",
+                "docs/setup.md",
+            ],
+        )
+
+    def test_empty_overrides_are_respected(self):
+        file_filter = FileFilter(
+            exclude_patterns=[],
+            allowed_extensions=set(),
+        )
+
+        result = file_filter.filter_files(
+            [
+                "README.md",
+                "image.png",
+            ]
+        )
+
+        self.assertEqual(result, [])
+
+    def test_nested_directory_globs(self):
+        file_filter = FileFilter()
+
+        result = file_filter.filter_files(
+            [
+                ".github/README.md",
+                "packages/site/node_modules/README.md",
+                "docs/archive/old.md",
+                ".cursor/rules/project.md",
+                "docs/setup.md",
+            ]
+        )
+
+        self.assertEqual(result, ["docs/setup.md"])
+
+    def test_explicit_empty_exclusions(self):
+        file_filter = FileFilter(exclude_patterns=[])
+
+        result = file_filter.filter_files(
+            [
+                ".github/README.md",
+            ]
+        )
+
+        self.assertEqual(
+            result,
+            [".github/README.md"],
+        )
+
+    def test_default_instance_isolation(self):
+        first = FileFilter()
+        second = FileFilter()
+
+        first.exclude_patterns.append("**/foo/**")
+
+        self.assertNotIn(
+            "**/foo/**",
+            second.exclude_patterns,
+        )
+
+    def test_empty_extension_raises(self):
+        with self.assertRaises(ValueError):
+            FileFilter(allowed_extensions={""})
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/filtering_benchmark_test.py

@@ -0,0 +1,30 @@
+import unittest
+
+from application.utils.harvester.file_filter import FileFilter
+from application.utils.harvester.filtering_benchmark import FilteringBenchmark
+
+
+class FilteringBenchmarkTests(unittest.TestCase):
+    def test_filtering_benchmark(self):
+        files = [
+            "README.md",
+            ".github/workflows/ci.yml",
+            "docs/guide.md",
+            "image.png",
+            "notes.txt",
+            "package-lock.json",
+        ]
+
+        benchmark = FilteringBenchmark(file_filter=FileFilter())
+
+        result = benchmark.run(files)
+
+        self.assertEqual(result.total_files, 6)
+        self.assertEqual(result.retained_files, 3)
+        self.assertEqual(result.filtered_files, 3)
+        self.assertEqual(result.retention_rate, 0.5)
+        self.assertEqual(result.filtering_rate, 0.5)
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/filtering_metrics_test.py

@@ -0,0 +1,35 @@
+import unittest
+
+from application.utils.harvester.filtering_metrics import (
+    FilteringMetricsCollector,
+)
+
+
+class FilteringMetricsCollectorTests(unittest.TestCase):
+    def test_filtering_metrics_collection(self):
+        collector = FilteringMetricsCollector()
+
+        collector.record_retained()
+        collector.record_retained()
+        collector.record_filtered()
+
+        metrics = collector.build()
+
+        self.assertEqual(
+            metrics.total_files,
+            3,
+        )
+
+        self.assertEqual(
+            metrics.retained_files,
+            2,
+        )
+
+        self.assertEqual(
+            metrics.filtered_files,
+            1,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/git_repository_client_test.py

@@ -70,7 +70,8 @@ def test_sync_clones_when_repository_missing(self):
 
         mock_clone.assert_called_once()
 
-    def test_sync_fetches_when_repository_exists(self):
+    @patch("application.utils.harvester.git_repository_client.subprocess.run")
+    def test_sync_fetches_when_repository_exists(self, mock_run):
         client = GitRepositoryClient(
             owner="OWASP",
             repository="ASVS",

application/utils/harvester/__init__.py

@@ -17,12 +17,22 @@
 from .git_repository_client import GitRepositoryClient
 from .repository_client import RepositoryClient
 from .repository_cache import build_repository_cache_path
+from .file_filter import FileFilter
+from .filtering_metrics import FilteringMetricsCollector
+from .filtering_benchmark import (
+    FilteringBenchmark,
+    FilteringBenchmarkResult,
+)
 
 __all__ = [
     "build_repository_cache_path",
     "ChunkingConfig",
     "ConfigLoaderError",
     "GitRepositoryClient",
+    "FileFilter",
+    "FilteringMetricsCollector",
+    "FilteringBenchmark",
+    "FilteringBenchmarkResult",
     "PathRules",
     "PollingConfig",
     "RepositoryClient",

application/utils/harvester/exclude_patterns.txt

@@ -4,7 +4,8 @@
 
 # to filter non-documentation files during harvesting.
 
-**/.git/*
+**/.github/**
+**/.git/**
 **/node_modules/**
 **/__pycache__/**
 **/.claude/**

application/utils/harvester/file_filter.py

@@ -0,0 +1,84 @@
+from pathlib import PurePosixPath
+from pathlib import Path
+import pathspec
+
+DEFAULT_ALLOWED_EXTENSIONS = {
+    ".md",
+    ".mdx",
+    ".rst",
+    ".txt",
+    ".adoc",
+}
+
+DEFAULT_EXCLUDE_PATTERNS = tuple(
+    line.strip()
+    for line in (
+        Path(__file__)
+        .with_name("exclude_patterns.txt")
+        .read_text(encoding="utf-8")
+        .splitlines()
+    )
+    if line.strip() and not line.lstrip().startswith("#")
+)
+
+
+class FileFilter:
+    def __init__(
+        self,
+        exclude_patterns: list[str] | None = None,
+        allowed_extensions: set[str] | None = None,
+    ):
+        self.exclude_patterns: list[str] = (
+            list(DEFAULT_EXCLUDE_PATTERNS)
+            if exclude_patterns is None
+            else list(exclude_patterns)
+        )
+
+        self.allowed_extensions: set[str] = (
+            set(DEFAULT_ALLOWED_EXTENSIONS)
+            if allowed_extensions is None
+            else set(allowed_extensions)
+        )
+
+        self._validate_patterns()
+
+        try:
+            self._exclude_spec = pathspec.PathSpec.from_lines(
+                "gitignore",
+                self.exclude_patterns,
+            )
+        except Exception as exc:
+            raise ValueError("Invalid exclude glob") from exc
+
+    def _validate_patterns(self) -> None:
+        if any(not pattern for pattern in self.exclude_patterns):
+            raise ValueError("Exclude pattern cannot be empty")
+
+        if any(not extension for extension in self.allowed_extensions):
+            raise ValueError("Allowed extension cannot be empty")
+
+    def _normalize_path(self, file_path: str) -> str:
+        return PurePosixPath(file_path).as_posix()
+
+    def is_excluded_by_pattern(self, file_path: str) -> bool:
+        normalized = self._normalize_path(file_path)
+        return self._exclude_spec.match_file(normalized)
+
+    def is_allowed_extension(self, file_path: str) -> bool:
+        return any(
+            file_path.endswith(extension) for extension in self.allowed_extensions
+        )
+
+    def filter_files(self, files: list[str]) -> list[str]:
+        filtered = []
+
+        for file_path in files:
+            if self.is_excluded_by_pattern(file_path):
+                continue
+
+            if not self.is_allowed_extension(file_path):
+                continue
+
+            filtered.append(file_path)
+
+        return filtered

application/utils/harvester/filtering_benchmark.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass
+
+from .file_filter import FileFilter
+
+
+@dataclass
+class FilteringBenchmarkResult:
+    total_files: int
+    retained_files: int
+    filtered_files: int
+    retention_rate: float
+    filtering_rate: float
+
+
+class FilteringBenchmark:
+    def __init__(self, file_filter: FileFilter):
+        self.file_filter = file_filter
+
+    def run(self, file_paths: list[str]) -> FilteringBenchmarkResult:
+        retained = self.file_filter.filter_files(file_paths)
+
+        total = len(file_paths)
+        retained_count = len(retained)
+        filtered_count = total - retained_count
+
+        return FilteringBenchmarkResult(
+            total_files=total,
+            retained_files=retained_count,
+            filtered_files=filtered_count,
+            retention_rate=(retained_count / total if total else 0.0),
+            filtering_rate=(filtered_count / total if total else 0.0),
+        )

application/utils/harvester/filtering_metrics.py

@@ -0,0 +1,23 @@
+from .models import FilteringMetrics
+
+
+class FilteringMetricsCollector:
+    def __init__(self):
+        self.total_files = 0
+        self.retained_files = 0
+        self.filtered_files = 0
+
+    def record_retained(self) -> None:
+        self.total_files += 1
+        self.retained_files += 1
+
+    def record_filtered(self) -> None:
+        self.total_files += 1
+        self.filtered_files += 1
+
+    def build(self) -> FilteringMetrics:
+        return FilteringMetrics(
+            total_files=self.total_files,
+            retained_files=self.retained_files,
+            filtered_files=self.filtered_files,
+        )