diff --git a/README.md b/README.md index 0017533..9831a30 100644 --- a/README.md +++ b/README.md @@ -594,11 +594,13 @@ To use cloud-hosted serverless PostgreSQL with Neon: ## 🛡️ Privacy, Security, and Responsible Use - **Local-First Boundary**: Secrets (`.env`, encryption keys, SQLite database) are never committed to version control and are ignored by `.gitignore`. -- **SSRF Defenses**: The audit crawler validates every target domain against DNS rebinding, private IP subnets (`127.0.0.0/8`, `10.0.0.0/8`, `192.168.0.0/16`, `172.16.0.0/12`), and cloud metadata endpoints (`169.254.169.254`). +- **Connection-Level SSRF Protection & IP Pinning**: Outbound audit probes enforce destination IP pinning at the socket connection level via custom `PinnedAsyncNetworkBackend` (`httpcore`) and `SSRFProtectedTransport` (`httpx`). Target hostnames are resolved and checked against comprehensive prohibited CIDR ranges (RFC 1918 private subnets, RFC 3927 link-local, loopback `127.0.0.0/8`, IPv6 link-local `fe80::/10`, multicast, IPv4-mapped IPv6 `::ffff:0:0/96`, and cloud metadata `169.254.169.254`). The actual TCP connection is pinned to the validated IP, eliminating time-of-check-to-time-of-use (TOCTOU) DNS rebinding attacks while strictly preserving standard TLS Server Name Indication (SNI), TLS certificate hostname verification, and the HTTP `Host` header. +- **Hop-by-Hop Redirect Revalidation**: Redirects are followed individually (`max_redirects=5`) with each hop's destination re-resolved and security-checked before connecting. Cross-origin sensitive headers (`Authorization`, `Cookie`) are automatically stripped across origins. +- **Resource and Memory Protection**: Streaming responses are strictly bounded to 5MB (`max_bytes=5242880`) to prevent memory exhaustion from massive bodies or decompression bombs. Transport connections disable keepalive reuse across audit targets (`max_keepalive_connections=0`) to eliminate connection-pool reuse poisoning. - **Prompt-Injection Defense**: Model prompts enclose untrusted crawled pages and business names in strict `` XML tags. -- **Strictly Single-Message Dispatch**: Leddro contains zero mass-mail sequences or automated dispatch triggers. Approving an email draft updates review status in SQLite only; dispatch requires explicit confirmation. +- **Strictly Single-Message Dispatch**: Leddro contains zero mass-mail sequences or automated dispatch triggers. Approving an email draft updates review status in SQLite only; dispatch requires explicit human confirmation. - **Suppression Registry**: Recipient emails and domains are verified against `suppression_records` to prevent messaging unsubscribed contacts. -- **Single-User Development Disclaimer**: Leddro is built for single-operator local use and does not contain multi-tenant user authentication. Do **not** expose the development server to untrusted public networks without reverse proxy authentication (e.g., HTTP Basic Auth or VPN). +- **Known Limitations & Operational Boundaries**: Leddro is built for single-operator local use and does not contain multi-tenant user authentication. Probes restrict requests to allowed ports (`80`, `443`, `8080`, `8443`). Do **not** expose the development server to untrusted public networks without reverse proxy authentication (e.g., HTTP Basic Auth or VPN). - **Vulnerability Reporting**: See [SECURITY.md](SECURITY.md) for our coordinated vulnerability disclosure policy. --- @@ -670,9 +672,12 @@ python -m pytest backend/tests/test_email_scraper.py -v # Run Google OAuth & SMTP dispatch test suite python -m pytest backend/tests/test_google_oauth_and_email.py -v -# Run SSRF security validation test suite +# Run SSRF security validation unit test suite (26 tests) python -m pytest backend/tests/test_ssrf_validator.py -v +# Run real-socket SSRF connection-pinning integration test suite (8 tests) +python -m pytest backend/tests/test_ssrf_real_network_integration.py -v + # Run SQLite backup and recovery test suite python -m pytest backend/tests/test_backup_and_recovery.py -v diff --git a/backend/app/ai/service.py b/backend/app/ai/service.py index a08d548..54810c0 100644 --- a/backend/app/ai/service.py +++ b/backend/app/ai/service.py @@ -176,13 +176,15 @@ def list_providers(self) -> List[AIProviderInfo]: # 4. Compatible (Local Ollama / vLLM) comp_p = self._providers["compatible"] + raw_comp_models = [settings.COMPATIBLE_MODEL, "llama3", "mistral", "qwen2.5"] + comp_models = [m.strip() for m in dict.fromkeys(raw_comp_models) if m and m.strip()] info_list.append( AIProviderInfo( provider_id="compatible", name=comp_p.name, is_configured=comp_p.is_configured(), current_model=settings.COMPATIBLE_MODEL, - available_models=[settings.COMPATIBLE_MODEL, "llama3", "mistral", "qwen2.5"], + available_models=comp_models, status_message=comp_p.get_status_message(), is_active=(active_id == "compatible"), is_local=True, diff --git a/backend/app/api/audits.py b/backend/app/api/audits.py index 4ef99d0..cd2624b 100644 --- a/backend/app/api/audits.py +++ b/backend/app/api/audits.py @@ -18,7 +18,7 @@ from backend.app.services.ssrf_validator import validate_target_url from backend.app.services.audit_engine import execute_audit from backend.app.services.audit_comparison import compare_audits -from sqlalchemy.orm import joinedload +from sqlalchemy.orm import joinedload, selectinload router = APIRouter(prefix="/audits", tags=["Audits"]) @@ -123,39 +123,48 @@ async def run_audit(payload: AuditRunRequest, db: Session = Depends(get_db)): def list_audits(db: Session = Depends(get_db)): """ List all executed audit reports. + Uses selectinload to eagerly load findings in a single batched query, eliminating N+1 queries. """ - reports = db.query(AuditReportModel).order_by(AuditReportModel.created_at.desc()).all() - results = [] - for r in reports: - findings = db.query(AuditFindingModel).filter(AuditFindingModel.audit_report_id == r.id).all() - results.append(db_audit_to_schema(r, findings)) - return results + reports = ( + db.query(AuditReportModel) + .options(selectinload(AuditReportModel.findings)) + .order_by(AuditReportModel.created_at.desc()) + .all() + ) + return [db_audit_to_schema(r) for r in reports] @router.get("/{report_id}") def get_audit(report_id: str, db: Session = Depends(get_db)): """ Get audit report details by report ID. """ - report = db.query(AuditReportModel).filter(AuditReportModel.id == report_id).first() + report = ( + db.query(AuditReportModel) + .options(selectinload(AuditReportModel.findings)) + .filter(AuditReportModel.id == report_id) + .first() + ) if not report: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Audit report with ID '{report_id}' not found" ) - findings = db.query(AuditFindingModel).filter(AuditFindingModel.audit_report_id == report.id).all() - return db_audit_to_schema(report, findings) + return db_audit_to_schema(report) @router.get("/prospect/{prospect_id}") def get_prospect_audits(prospect_id: str, db: Session = Depends(get_db)): """ Get all audit reports for a specific prospect. + Uses selectinload to eagerly load findings in a single batched query. """ - reports = db.query(AuditReportModel).filter(AuditReportModel.prospect_id == prospect_id).order_by(AuditReportModel.created_at.desc()).all() - results = [] - for r in reports: - findings = db.query(AuditFindingModel).filter(AuditFindingModel.audit_report_id == r.id).all() - results.append(db_audit_to_schema(r, findings)) - return results + reports = ( + db.query(AuditReportModel) + .options(selectinload(AuditReportModel.findings)) + .filter(AuditReportModel.prospect_id == prospect_id) + .order_by(AuditReportModel.created_at.desc()) + .all() + ) + return [db_audit_to_schema(r) for r in reports] @router.post("/compare", response_model=AuditComparisonResponse) def compare_two_audits(payload: AuditCompareRequest, db: Session = Depends(get_db)): diff --git a/backend/app/discovery/deduplication.py b/backend/app/discovery/deduplication.py index ee3b7de..feb3b3a 100644 --- a/backend/app/discovery/deduplication.py +++ b/backend/app/discovery/deduplication.py @@ -55,9 +55,10 @@ def check_single_duplicate( # Avoid matching generic public hosts generic_hosts = {"gmail.com", "yahoo.com", "outlook.com", "facebook.com", "instagram.com", "linkedin.com", "twitter.com", "x.com"} if candidate_domain not in generic_hosts: - # Query prospects with website_url + safe_domain = candidate_domain.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + # Query only prospects whose website_url contains the candidate domain (eliminates full-table scan) prospects_with_url = db.query(ProspectModel).filter( - ProspectModel.website_url.isnot(None) + ProspectModel.website_url.ilike(f"%{safe_domain}%", escape="\\") ).all() for p in prospects_with_url: p_domain = extract_registered_domain(p.website_url) @@ -73,8 +74,14 @@ def check_single_duplicate( # 3. Check by Business Name and Location norm_candidate_name = normalize_business_name(name) if norm_candidate_name and len(norm_candidate_name) >= 3: - # Search candidate names - potential_matches = db.query(ProspectModel).all() + # Efficiently pre-filter candidates by significant name token instead of full-table scan + tokens = [t for t in norm_candidate_name.split() if len(t) >= 2] + best_token = max(tokens, key=len) if tokens else norm_candidate_name[:3] + safe_token = best_token.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + potential_matches = db.query(ProspectModel).filter( + ProspectModel.name.ilike(f"%{safe_token}%", escape="\\") + ).all() candidate_city = (city or "").strip().lower() for p in potential_matches: diff --git a/backend/app/services/audit_engine.py b/backend/app/services/audit_engine.py index d177214..f90a705 100644 --- a/backend/app/services/audit_engine.py +++ b/backend/app/services/audit_engine.py @@ -9,7 +9,12 @@ from typing import Dict, Any, List, Optional, Tuple from backend.app.core.config import settings -from backend.app.services.ssrf_validator import validate_target_url +from backend.app.services.ssrf_validator import ( + validate_target_url, + SSRFProtectedTransport, + connect_safe_tls_socket, + SSRFSecurityError +) from backend.app.services.email_scraper import extract_contacts_from_html, verify_email_mx class AuditEngine: @@ -184,6 +189,38 @@ def _inspect_dns(self, domain: str) -> Dict[str, Any]: except Exception: pass + # Subdomain fallback: If SPF or DMARC not on subdomain (e.g. www.domain.com), + # query organizational / apex domain per RFC 7489 Section 6.6.3 + parts = domain.split(".") + base_domain = None + if len(parts) > 2 and parts[0].lower() == "www": + base_domain = ".".join(parts[1:]) + elif len(parts) > 2: + base_domain = ".".join(parts[-2:]) + + if base_domain and base_domain != domain: + if not spf_found: + try: + base_txt = resolver.resolve(base_domain, "TXT") + for rdata in base_txt: + txt_str = "".join([part.decode("utf-8", errors="ignore") for part in rdata.strings]) + if "v=spf1" in txt_str.lower(): + spf_found = True + break + except Exception: + pass + + if not dmarc_found: + try: + base_dmarc = resolver.resolve(f"_dmarc.{base_domain}", "TXT") + for rdata in base_dmarc: + dmarc_str = "".join([part.decode("utf-8", errors="ignore") for part in rdata.strings]) + if "v=dmarc1" in dmarc_str.lower(): + dmarc_found = True + break + except Exception: + pass + query_ms = int((time.time() - dns_start) * 1000) # Evidence evaluation @@ -275,76 +312,75 @@ def _inspect_tls(self, domain: str, port: int = 443) -> Dict[str, Any]: """Connects via SSL socket to evaluate certificate validity, cipher, and expiration.""" findings = [] try: - ctx = ssl.create_default_context() - with socket.create_connection((domain, port), timeout=settings.AUDIT_CONNECT_TIMEOUT) as sock: - with ctx.wrap_socket(sock, server_hostname=domain) as ssock: - cert = ssock.getpeercert() - tls_version = ssock.version() - cipher_tuple = ssock.cipher() - cipher_name = cipher_tuple[0] if cipher_tuple else "Unknown" - - # Parse expiration date - not_after_str = cert.get("notAfter", "") - # Format: 'Apr 18 23:59:59 2026 GMT' - not_after_dt = datetime.datetime.strptime(not_after_str, "%b %d %H:%M:%S %Y %Z").replace(tzinfo=datetime.timezone.utc) - now_dt = datetime.datetime.now(datetime.timezone.utc) - days_remaining = (not_after_dt - now_dt).days - - # Extract Issuer - issuer_dict = dict(x[0] for x in cert.get("issuer", ())) - issuer_name = issuer_dict.get("organizationName") or issuer_dict.get("commonName") or "Unknown CA" - - if days_remaining < 0: - findings.append({ - "id": f"f-ssl-exp-{int(time.time()*1000)}", - "title": "SSL/TLS certificate has expired", - "severity": "high", - "check_status": "FAIL", - "observed": f"The TLS certificate expired {abs(days_remaining)} days ago on {not_after_str}.", - "evidence": f"Certificate Expiry: {not_after_str}\nCurrent Audit Time: {now_dt.strftime('%b %d %Y')}\nDays Remaining: {days_remaining}\nIssuer: {issuer_name}", - "why_it_matters": "Browsers reject the connection with intimidating warning screens, preventing customers from accessing the site.", - "recommended_action": "Renew the SSL/TLS certificate immediately via your hosting control panel or ACME bot.", - "confidence": 1.0, - "category": "security", - "checked_url": f"https://{domain}" - }) - elif days_remaining <= 14: - findings.append({ - "id": f"f-ssl-warn-{int(time.time()*1000)}", - "title": "SSL/TLS certificate expiring soon", - "severity": "medium", - "check_status": "WARNING", - "observed": f"The TLS certificate expires in {days_remaining} days on {not_after_str}.", - "evidence": f"Certificate Expiry: {not_after_str}\nDays Remaining: {days_remaining}", - "why_it_matters": "Risk of unexpected outage if automated renewal triggers fail.", - "recommended_action": "Verify automated renewal automation status before certificate reaches expiration.", - "confidence": 0.98, - "category": "security", - "checked_url": f"https://{domain}" - }) - else: - findings.append({ - "id": f"f-ssl-pass-{int(time.time()*1000)}", - "title": "Valid SSL/TLS certificate verified", - "severity": "info", - "check_status": "PASS", - "observed": f"Valid TLS certificate issued by {issuer_name} with {days_remaining} days remaining.", - "evidence": f"Issuer: {issuer_name}\nTLS Version: {tls_version}\nCipher: {cipher_name}\nDays Remaining: {days_remaining}", - "why_it_matters": "Encryption protects customer data in transit and satisfies browser security requirements.", - "recommended_action": "Certificate is healthy. No action required.", - "confidence": 1.0, - "category": "security", - "checked_url": f"https://{domain}" - }) + ssock, pinned_ip = connect_safe_tls_socket(domain, port=port, timeout=settings.AUDIT_CONNECT_TIMEOUT) + with ssock: + cert = ssock.getpeercert() + tls_version = ssock.version() + cipher_tuple = ssock.cipher() + cipher_name = cipher_tuple[0] if cipher_tuple else "Unknown" + + # Parse expiration date + not_after_str = cert.get("notAfter", "") + # Format: 'Apr 18 23:59:59 2026 GMT' + not_after_dt = datetime.datetime.strptime(not_after_str, "%b %d %H:%M:%S %Y %Z").replace(tzinfo=datetime.timezone.utc) + now_dt = datetime.datetime.now(datetime.timezone.utc) + days_remaining = (not_after_dt - now_dt).days + + # Extract Issuer + issuer_dict = dict(x[0] for x in cert.get("issuer", ())) + issuer_name = issuer_dict.get("organizationName") or issuer_dict.get("commonName") or "Unknown CA" + + if days_remaining < 0: + findings.append({ + "id": f"f-ssl-exp-{int(time.time()*1000)}", + "title": "SSL/TLS certificate has expired", + "severity": "high", + "check_status": "FAIL", + "observed": f"The TLS certificate expired {abs(days_remaining)} days ago on {not_after_str}.", + "evidence": f"Certificate Expiry: {not_after_str}\nCurrent Audit Time: {now_dt.strftime('%b %d %Y')}\nDays Remaining: {days_remaining}\nIssuer: {issuer_name}", + "why_it_matters": "Browsers reject the connection with intimidating warning screens, preventing customers from accessing the site.", + "recommended_action": "Renew the SSL/TLS certificate immediately via your hosting control panel or ACME bot.", + "confidence": 1.0, + "category": "security", + "checked_url": f"https://{domain}" + }) + elif days_remaining <= 14: + findings.append({ + "id": f"f-ssl-warn-{int(time.time()*1000)}", + "title": "SSL/TLS certificate expiring soon", + "severity": "medium", + "check_status": "WARNING", + "observed": f"The TLS certificate expires in {days_remaining} days on {not_after_str}.", + "evidence": f"Certificate Expiry: {not_after_str}\nDays Remaining: {days_remaining}", + "why_it_matters": "Risk of unexpected outage if automated renewal triggers fail.", + "recommended_action": "Verify automated renewal automation status before certificate reaches expiration.", + "confidence": 0.98, + "category": "security", + "checked_url": f"https://{domain}" + }) + else: + findings.append({ + "id": f"f-ssl-pass-{int(time.time()*1000)}", + "title": "Valid SSL/TLS certificate verified", + "severity": "info", + "check_status": "PASS", + "observed": f"Valid TLS certificate issued by {issuer_name} with {days_remaining} days remaining.", + "evidence": f"Issuer: {issuer_name}\nTLS Version: {tls_version}\nCipher: {cipher_name}\nDays Remaining: {days_remaining}", + "why_it_matters": "Encryption protects customer data in transit and satisfies browser security requirements.", + "recommended_action": "Certificate is healthy. No action required.", + "confidence": 1.0, + "category": "security", + "checked_url": f"https://{domain}" + }) - return { - "valid": days_remaining >= 0, - "issuer": issuer_name, - "expiresDays": days_remaining, - "tlsVersion": tls_version, - "cipher": cipher_name, - "findings": findings - } + return { + "valid": days_remaining >= 0, + "issuer": issuer_name, + "expiresDays": days_remaining, + "tlsVersion": tls_version, + "cipher": cipher_name, + "findings": findings + } except Exception as e: err_msg = str(e) findings.append({ @@ -381,7 +417,7 @@ async def _fetch_http_safely(self, initial_url: str) -> Dict[str, Any]: final_response = None body_content = "" - transport = httpx.AsyncHTTPTransport(retries=1) + transport = SSRFProtectedTransport() headers = { "User-Agent": settings.AUDIT_USER_AGENT, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", @@ -391,7 +427,7 @@ async def _fetch_http_safely(self, initial_url: str) -> Dict[str, Any]: async with httpx.AsyncClient(transport=transport, follow_redirects=False, timeout=settings.AUDIT_READ_TIMEOUT) as client: while redirect_hops <= settings.AUDIT_MAX_REDIRECTS: # SSRF validation before every hop - is_safe, error_msg, parsed_hop, _ = validate_target_url(current_url) + is_safe, error_msg, parsed_hop, resolved_ips = validate_target_url(current_url) if not is_safe: findings.append({ "id": f"f-ssrf-hop-{int(time.time()*1000)}", @@ -408,6 +444,10 @@ async def _fetch_http_safely(self, initial_url: str) -> Dict[str, Any]: }) break + hop_port = parsed_hop.port or (443 if parsed_hop.scheme == "https" else 80) + if resolved_ips: + transport.pinned_backend.pin_destination(parsed_hop.hostname, hop_port, resolved_ips[0]) + req_start = time.time() try: # Stream response to enforce size limits @@ -441,8 +481,37 @@ async def _fetch_http_safely(self, initial_url: str) -> Dict[str, Any]: # Check for redirect status if resp.is_redirect and "Location" in resp.headers: redirect_hops += 1 - target_loc = resp.headers["Location"] - current_url = urljoin(current_url, target_loc) + target_loc = resp.headers["Location"].strip() + if not target_loc: + break + target_url = urljoin(current_url, target_loc) + parsed_next = urlparse(target_url) + if parsed_next.scheme.lower() not in ("http", "https"): + findings.append({ + "id": f"f-ssrf-scheme-{int(time.time()*1000)}", + "title": "Redirect hop targeted prohibited URI scheme", + "severity": "high", + "check_status": "FAIL", + "observed": f"Redirect to prohibited scheme '{parsed_next.scheme}' was blocked: {target_url}", + "evidence": f"Redirect Target: {target_url}\nBlocked By: SSRF Scheme Validator", + "why_it_matters": "Web servers must not redirect crawlers to non-HTTP protocols.", + "recommended_action": "Fix redirect rules to only route to public HTTP/HTTPS URLs.", + "confidence": 1.0, + "category": "security", + "checked_url": current_url + }) + break + + # Cross-origin sensitive header stripping + next_port = parsed_next.port or (443 if parsed_next.scheme == "https" else 80) + if ( + parsed_hop.hostname.lower() != parsed_next.hostname.lower() or + parsed_hop.scheme.lower() != parsed_next.scheme.lower() or + hop_port != next_port + ): + headers = {k: v for k, v in headers.items() if k.lower() not in ("authorization", "cookie")} + + current_url = target_url continue else: # Not a redirect, finished @@ -462,20 +531,37 @@ async def _fetch_http_safely(self, initial_url: str) -> Dict[str, Any]: "checked_url": current_url }) break - except httpx.ConnectError as e: - findings.append({ - "id": f"f-conn-err-{int(time.time()*1000)}", - "title": "TCP Connection refused or unreachable", - "severity": "high", - "check_status": "ERROR", - "observed": f"Network error connecting to {current_url}: {str(e)}", - "evidence": f"URL: {current_url}\nSocket Error: {str(e)}", - "why_it_matters": "The target web server is offline or rejecting incoming HTTP traffic.", - "recommended_action": "Verify web server process status (nginx/apache) and network connectivity.", - "confidence": 1.0, - "category": "reachability", - "checked_url": current_url - }) + except (SSRFSecurityError, httpx.ConnectError) as e: + err_str = str(e) + is_ssrf_block = isinstance(e, SSRFSecurityError) or any(term in err_str.lower() for term in ["prohibited", "ssrf", "loopback", "security"]) + if is_ssrf_block: + findings.append({ + "id": f"f-ssrf-block-{int(time.time()*1000)}", + "title": "Outbound connection blocked by connection-level SSRF protection", + "severity": "high", + "check_status": "FAIL", + "observed": f"SSRF transport blocked connection to {current_url}: {err_str}", + "evidence": f"URL: {current_url}\nSSRF Defense: {err_str}", + "why_it_matters": "Outbound traffic to private, internal, or prohibited destinations is strictly denied.", + "recommended_action": "Ensure target host resolves to publicly routable IP addresses.", + "confidence": 1.0, + "category": "security", + "checked_url": current_url + }) + else: + findings.append({ + "id": f"f-conn-err-{int(time.time()*1000)}", + "title": "TCP Connection refused or unreachable", + "severity": "high", + "check_status": "ERROR", + "observed": f"Network error connecting to {current_url}: {err_str}", + "evidence": f"URL: {current_url}\nSocket Error: {err_str}", + "why_it_matters": "The target web server is offline or rejecting incoming HTTP traffic.", + "recommended_action": "Verify web server process status (nginx/apache) and network connectivity.", + "confidence": 1.0, + "category": "reachability", + "checked_url": current_url + }) break except Exception as e: findings.append({ diff --git a/backend/app/services/deep_search_enrichment.py b/backend/app/services/deep_search_enrichment.py index 303b4fa..839cf22 100644 --- a/backend/app/services/deep_search_enrichment.py +++ b/backend/app/services/deep_search_enrichment.py @@ -25,6 +25,8 @@ PHONE_REGEX, ) +from backend.app.services.ssrf_validator import safe_http_get, validate_target_url + logger = logging.getLogger("leddro.services.deep_search_enrichment") # Aggregator and platform domains to exclude from official website detection @@ -311,30 +313,29 @@ async def deep_search_prospect( elif "indiamart.com" in domain and not social_profiles["indiamart"]: social_profiles["indiamart"] = url - # If Justdial page found, extract structured Schema.org data + # If Justdial page found, extract structured Schema.org data safely if social_profiles["justdial"]: try: headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", } - async with httpx.AsyncClient(timeout=6.0, follow_redirects=True) as client: - res_jd = await client.get(social_profiles["justdial"], headers=headers) - if res_jd.status_code == 200: - soup_jd = BeautifulSoup(res_jd.text, "html.parser") - import json - for script in soup_jd.find_all("script", type="application/ld+json"): - try: - jd_data = json.loads(script.string) - if jd_data.get("@type") in ["Restaurant", "LocalBusiness"]: - addr = jd_data.get("address", {}) - if isinstance(addr, dict) and addr.get("streetAddress"): - directory_metadata["address"] = f"{addr.get('streetAddress', '')}, {addr.get('addressLocality', '')}" - rating = jd_data.get("aggregateRating", {}) - if isinstance(rating, dict) and rating.get("ratingValue"): - directory_metadata["rating"] = f"{rating.get('ratingValue')}/5 ({rating.get('ratingCount', 0)} reviews)" - break - except Exception: - continue + jd_success, res_jd, jd_err = await safe_http_get(social_profiles["justdial"], headers=headers, timeout=6.0) + if jd_success and res_jd and res_jd.status_code == 200: + soup_jd = BeautifulSoup(res_jd.text, "html.parser") + import json + for script in soup_jd.find_all("script", type="application/ld+json"): + try: + jd_data = json.loads(script.string) + if jd_data.get("@type") in ["Restaurant", "LocalBusiness"]: + addr = jd_data.get("address", {}) + if isinstance(addr, dict) and addr.get("streetAddress"): + directory_metadata["address"] = f"{addr.get('streetAddress', '')}, {addr.get('addressLocality', '')}" + rating = jd_data.get("aggregateRating", {}) + if isinstance(rating, dict) and rating.get("ratingValue"): + directory_metadata["rating"] = f"{rating.get('ratingValue')}/5 ({rating.get('ratingCount', 0)} reviews)" + break + except Exception: + continue except Exception as e: logger.debug(f"Justdial extraction error: {e}") diff --git a/backend/app/services/email_scraper.py b/backend/app/services/email_scraper.py index 94ea6f4..10fc314 100644 --- a/backend/app/services/email_scraper.py +++ b/backend/app/services/email_scraper.py @@ -8,7 +8,7 @@ import dns.resolver from backend.app.core.config import settings -from backend.app.services.ssrf_validator import validate_target_url +from backend.app.services.ssrf_validator import validate_target_url, safe_http_get logger = logging.getLogger("leddro.services.email_scraper") @@ -21,7 +21,7 @@ JUNK_DOMAINS = { "example.com", "example.org", "domain.com", "yourcompany.com", - "placeholder.com", "email.com", "sentry.io", "wixpress.com", + "company.com", "placeholder.com", "email.com", "sentry.io", "wixpress.com", "w3.org", "schema.org", "gravatar.com", "wordpress.org" } @@ -47,7 +47,7 @@ def is_valid_email_candidate(email: str) -> bool: if domain in JUNK_DOMAINS or domain.startswith("sentry-") or "sentry" in domain: return False - if user in {"name", "user", "username", "yourname", "test", "admin@localhost"}: + if user in {"name", "user", "username", "yourname", "you", "test", "admin@localhost"}: return False return True @@ -161,45 +161,42 @@ async def scrape_website_contacts(url: str, max_pages: int = 3) -> Dict[str, Any visited_urls: List[str] = [] contact_subpage_links: Set[str] = set() - transport = httpx.AsyncHTTPTransport(retries=1) - async with httpx.AsyncClient(transport=transport, timeout=7.0, follow_redirects=True) as client: - # 1. Fetch homepage + # 1. Fetch homepage safely with manual redirect SSRF checks + success, resp, err_msg = await safe_http_get(url, headers=headers, timeout=7.0) + if not success or not resp: + logger.warning(f"Error fetching homepage {url}: {err_msg}") + else: + visited_urls.append(str(resp.url)) + if resp.status_code == 200: + contacts = extract_contacts_from_html(resp.text, str(resp.url)) + all_emails.update(contacts["emails"]) + all_phones.update(contacts["phones"]) + + # Look for contact / about subpage links + soup = BeautifulSoup(resp.text, "html.parser") + for a in soup.find_all("a", href=True): + href = a["href"].strip() + href_lower = href.lower() + if any(slug in href_lower for slug in ["contact", "about", "reach-us", "touch", "location"]): + sub_url = urljoin(str(resp.url), href) + parsed_sub = urlparse(sub_url) + if parsed_sub.hostname == domain and sub_url not in visited_urls: + contact_subpage_links.add(sub_url) + + # 2. Fetch contact subpages (up to max_pages) safely + subpage_list = list(contact_subpage_links)[:max_pages - 1] + for sub_url in subpage_list: + if sub_url in visited_urls: + continue try: - resp = await client.get(url, headers=headers) - visited_urls.append(str(resp.url)) - if resp.status_code == 200: - contacts = extract_contacts_from_html(resp.text, str(resp.url)) + visited_urls.append(sub_url) + sub_success, sub_resp, sub_err = await safe_http_get(sub_url, headers=headers, timeout=7.0) + if sub_success and sub_resp and sub_resp.status_code == 200: + contacts = extract_contacts_from_html(sub_resp.text, str(sub_resp.url)) all_emails.update(contacts["emails"]) all_phones.update(contacts["phones"]) - - # Look for contact / about subpage links - soup = BeautifulSoup(resp.text, "html.parser") - for a in soup.find_all("a", href=True): - href = a["href"].strip() - href_lower = href.lower() - if any(slug in href_lower for slug in ["contact", "about", "reach-us", "touch", "location"]): - sub_url = urljoin(str(resp.url), href) - parsed_sub = urlparse(sub_url) - if parsed_sub.hostname == domain and sub_url not in visited_urls: - contact_subpage_links.add(sub_url) except Exception as e: - logger.warning(f"Error fetching homepage {url}: {e}") - - # 2. Fetch contact subpages (up to max_pages) - subpage_list = list(contact_subpage_links)[:max_pages - 1] - for sub_url in subpage_list: - is_sub_safe, _, _, _ = validate_target_url(sub_url) - if not is_sub_safe or sub_url in visited_urls: - continue - try: - visited_urls.append(sub_url) - sub_resp = await client.get(sub_url, headers=headers) - if sub_resp.status_code == 200: - contacts = extract_contacts_from_html(sub_resp.text, str(sub_resp.url)) - all_emails.update(contacts["emails"]) - all_phones.update(contacts["phones"]) - except Exception as e: - logger.warning(f"Error fetching subpage {sub_url}: {e}") + logger.warning(f"Error fetching subpage {sub_url}: {e}") # 3. Validate MX records for candidate emails verified_emails = [] diff --git a/backend/app/services/ssrf_validator.py b/backend/app/services/ssrf_validator.py index f0b0b95..60bf943 100644 --- a/backend/app/services/ssrf_validator.py +++ b/backend/app/services/ssrf_validator.py @@ -1,7 +1,10 @@ import ipaddress import socket +import ssl import urllib.parse -from typing import Tuple, List, Optional +from typing import Tuple, List, Optional, Dict, Any, Set +import httpcore +import httpx from backend.app.core.config import settings # Explicit IPv4 and IPv6 networks prohibited from access @@ -41,13 +44,22 @@ "metadata", } -def is_ip_allowed(ip_str: str) -> Tuple[bool, Optional[str]]: +def is_ip_allowed(ip_str: str, allow_loopback: bool = False) -> Tuple[bool, Optional[str]]: """Check if an IP address is publicly routable and not in private/reserved/cloud-metadata ranges.""" try: ip = ipaddress.ip_address(ip_str) except ValueError: return False, f"Invalid IP address representation: {ip_str}" + if allow_loopback and ip.is_loopback: + return True, None + + # Strict handling for IPv4-mapped IPv6 addresses (e.g. ::ffff:127.0.0.1) + if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped: + v4_allowed, v4_reason = is_ip_allowed(str(ip.ipv4_mapped), allow_loopback=allow_loopback) + if not v4_allowed: + return False, f"IPv4-mapped IPv6 address {ip_str} resolves to prohibited IPv4: {v4_reason}" + if ip.is_loopback: return False, f"Loopback address prohibited: {ip_str}" if ip.is_private: @@ -62,25 +74,25 @@ def is_ip_allowed(ip_str: str) -> Tuple[bool, Optional[str]]: return False, f"Reserved address prohibited: {ip_str}" for net in BLOCKED_NETWORKS: - if ip in net: + if ip.version == net.version and ip in net: return False, f"Address {ip_str} falls within prohibited network {net}" return True, None -def validate_host_and_resolve(hostname: str) -> Tuple[bool, List[str], Optional[str]]: +def validate_host_and_resolve(hostname: str, allow_loopback: bool = False) -> Tuple[bool, List[str], Optional[str]]: """ Validate a hostname against blocked names and resolve it to ensure all resolved IPs are safe. Mitigates DNS rebinding by performing resolution check upfront. """ host_clean = hostname.strip().lower() - if host_clean in BLOCKED_HOSTNAMES or host_clean.endswith(".internal") or host_clean.endswith(".local"): + if not allow_loopback and (host_clean in BLOCKED_HOSTNAMES or host_clean.endswith(".internal") or host_clean.endswith(".local")): return False, [], f"Prohibited or internal hostname: {hostname}" # Check if host is direct IP try: ip_obj = ipaddress.ip_address(host_clean) - allowed, reason = is_ip_allowed(str(ip_obj)) + allowed, reason = is_ip_allowed(str(ip_obj), allow_loopback=allow_loopback) if not allowed: return False, [], reason return True, [str(ip_obj)], None @@ -105,13 +117,13 @@ def validate_host_and_resolve(hostname: str) -> Tuple[bool, List[str], Optional[ # Validate EVERY resolved IP for ip in resolved_ips: - allowed, reason = is_ip_allowed(ip) + allowed, reason = is_ip_allowed(ip, allow_loopback=allow_loopback) if not allowed: return False, resolved_ips, f"Security violation: {hostname} resolves to prohibited address {ip} ({reason})" return True, resolved_ips, None -def validate_target_url(url: str) -> Tuple[bool, Optional[str], Optional[urllib.parse.ParseResult], List[str]]: +def validate_target_url(url: str, allow_loopback: bool = False) -> Tuple[bool, Optional[str], Optional[urllib.parse.ParseResult], List[str]]: """ Validates a URL syntax, scheme, port, and hostname/IP safety. Returns: (is_valid, error_reason, parsed_url_object, resolved_ips) @@ -139,12 +151,374 @@ def validate_target_url(url: str) -> Tuple[bool, Optional[str], Optional[urllib. return False, "URL is missing a valid hostname", parsed, [] port = parsed.port or (443 if parsed.scheme == "https" else 80) - if port not in settings.AUDIT_ALLOWED_PORTS: + if not allow_loopback and port not in settings.AUDIT_ALLOWED_PORTS: return False, f"Prohibited port {port}. Allowed ports are: {settings.AUDIT_ALLOWED_PORTS}", parsed, [] # Validate hostname and DNS resolution against SSRF blocklist - is_safe, resolved_ips, error_msg = validate_host_and_resolve(parsed.hostname) + is_safe, resolved_ips, error_msg = validate_host_and_resolve(parsed.hostname, allow_loopback=allow_loopback) if not is_safe: return False, error_msg, parsed, resolved_ips return True, None, parsed, resolved_ips + +class SSRFSecurityError(httpcore.ConnectError): + """Raised when an outbound connection targets a prohibited IP, scheme, or SSRF vector.""" + pass + +def resolve_and_pin_host(host: str, port: int, allow_loopback: bool = False) -> Tuple[str, List[str]]: + """ + Resolves hostname and strictly validates every returned IP address. + If ANY returned IP address is prohibited, raises SSRFSecurityError. + Prevents DNS rebinding and TOCTOU vulnerabilities. + Returns (selected_pinned_ip, all_resolved_ips). + """ + host_clean = host.strip().lower() + + if not allow_loopback and (host_clean in BLOCKED_HOSTNAMES or host_clean.endswith(".internal") or host_clean.endswith(".local")): + raise SSRFSecurityError(f"Security Policy: Host '{host}' is in prohibited internal hostname list") + + # 1. Handle IP Literals directly + try: + ip_obj = ipaddress.ip_address(host_clean) + allowed, reason = is_ip_allowed(str(ip_obj)) + if not allowed and not (allow_loopback and ip_obj.is_loopback): + raise SSRFSecurityError(f"Security Policy: Target IP literal '{host}' is prohibited: {reason}") + return str(ip_obj), [str(ip_obj)] + except ValueError: + pass # It's a domain name, proceed to DNS resolution + + # 2. Resolve via socket.getaddrinfo (all A and AAAA records) + try: + addr_infos = socket.getaddrinfo(host_clean, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except socket.gaierror as e: + raise SSRFSecurityError(f"DNS resolution failed for '{host}': {e}") + except Exception as e: + raise SSRFSecurityError(f"DNS lookup error for '{host}': {e}") + + if not addr_infos: + raise SSRFSecurityError(f"No DNS records found for host '{host}'") + + resolved_ips: List[str] = [] + for info in addr_infos: + ip_str = info[4][0] + try: + ip_obj = ipaddress.ip_address(ip_str) + except ValueError: + raise SSRFSecurityError(f"DNS for '{host}' returned invalid IP address representation '{ip_str}'") + + allowed, reason = is_ip_allowed(ip_str) + if not allowed and not (allow_loopback and ip_obj.is_loopback): + raise SSRFSecurityError(f"Security Policy: DNS for '{host}' returned prohibited IP '{ip_str}': {reason}") + + if ip_str not in resolved_ips: + resolved_ips.append(ip_str) + + if not resolved_ips: + raise SSRFSecurityError(f"No valid publicly routable IP addresses resolved for host '{host}'") + + # Policy: Deterministically pin the first valid IPv4 address if available, otherwise first valid IPv6 + v4_ips = [ip for ip in resolved_ips if ":" not in ip] + pinned_ip = v4_ips[0] if v4_ips else resolved_ips[0] + return pinned_ip, resolved_ips + +class PinnedAsyncNetworkBackend(httpcore.AsyncNetworkBackend): + """ + Custom httpcore AsyncNetworkBackend enforcing connection-level IP pinning. + Guarantees the socket is connected exclusively to the pre-validated IP address, + preventing DNS rebinding / TOCTOU attacks while preserving: + - TLS certificate verification + - TLS SNI (Server Name Indication) + - HTTP Host header + """ + def __init__( + self, + pinned_map: Optional[Dict[Tuple[str, int], str]] = None, + allow_loopback_for_testing: bool = False, + real_backend: Optional[httpcore.AsyncNetworkBackend] = None + ): + self._pinned_map: Dict[Tuple[str, int], str] = dict(pinned_map or {}) + self.allow_loopback_for_testing = allow_loopback_for_testing + self._backend = real_backend or httpcore.AnyIOBackend() + self.last_connected_addr: Optional[Tuple[str, int]] = None + + def pin_destination(self, host: str, port: int, ip: str) -> None: + """Explicitly pins a destination (host, port) to a validated IP.""" + self._pinned_map[(host.strip().lower(), port)] = ip + + def get_pinned_destination(self, host: str, port: int) -> Optional[str]: + return self._pinned_map.get((host.strip().lower(), port)) + + async def connect_tcp( + self, + host: str, + port: int, + timeout: Optional[float] = None, + local_address: Optional[str] = None, + socket_options: Optional[Any] = None, + ) -> httpcore.AsyncNetworkStream: + host_key = host.strip().lower() + + # Retrieve pinned IP or resolve and validate on the fly + if (host_key, port) in self._pinned_map: + target_ip = self._pinned_map[(host_key, port)] + # Defense-in-depth: Re-verify pinned IP is safe before socket connect + ip_obj = ipaddress.ip_address(target_ip) + allowed, reason = is_ip_allowed(target_ip) + if not allowed and not (self.allow_loopback_for_testing and ip_obj.is_loopback): + raise SSRFSecurityError(f"Security Policy: Pinned IP '{target_ip}' is prohibited: {reason}") + else: + target_ip, _ = resolve_and_pin_host(host, port, allow_loopback=self.allow_loopback_for_testing) + self._pinned_map[(host_key, port)] = target_ip + + # Establish TCP socket connection directly to the pinned IP + stream = await self._backend.connect_tcp( + target_ip, + port, + timeout=timeout, + local_address=local_address, + socket_options=socket_options + ) + + # Confirm actual connected socket peer matches the pinned destination + peer = stream.get_extra_info("server_addr") + if peer: + connected_ip = str(peer[0]) + self.last_connected_addr = (connected_ip, peer[1]) + if connected_ip != target_ip: + await stream.aclose() + raise SSRFSecurityError( + f"Security Policy: Connected peer IP '{connected_ip}' does not match pinned IP '{target_ip}'" + ) + + return stream + + async def connect_unix_socket(self, path: str, timeout=None, socket_options=None): + raise SSRFSecurityError("Unix domain sockets are prohibited for outbound SSRF requests") + + async def sleep(self, seconds: float) -> None: + await self._backend.sleep(seconds) + +class SSRFProtectedTransport(httpx.AsyncHTTPTransport): + """ + HTTPX Async Transport with connection-level IP pinning and SSRF defense. + - Connects TCP sockets only to security-validated destination IPs + - Preserves TLS SNI, certificate hostname verification, and Host header + - Prevents DNS rebinding and connection-pool poisoning (max_keepalive_connections=0) + """ + def __init__( + self, + pinned_map: Optional[Dict[Tuple[str, int], str]] = None, + allow_loopback_for_testing: bool = False, + verify: Any = True, + **kwargs + ): + self.pinned_backend = PinnedAsyncNetworkBackend( + pinned_map=pinned_map, + allow_loopback_for_testing=allow_loopback_for_testing + ) + kwargs.setdefault("retries", 0) + super().__init__(verify=verify, **kwargs) + self._pool = httpcore.AsyncConnectionPool( + ssl_context=self._pool._ssl_context, + network_backend=self.pinned_backend, + max_connections=10, + max_keepalive_connections=0, # Zero keepalive eliminates cross-request connection reuse poisoning + keepalive_expiry=0.0, + retries=0, + http1=True, + http2=False + ) + +def connect_safe_tls_socket( + domain: str, + port: int = 443, + timeout: float = 5.0, + ssl_context: Optional[ssl.SSLContext] = None, + allow_loopback_for_testing: bool = False +) -> Tuple[ssl.SSLSocket, str]: + """ + Connects a raw TLS socket to a pre-validated, pinned destination IP while + strictly preserving TLS SNI and certificate verification against the logical domain. + Used for direct SSL inspection in audit engine. + """ + pinned_ip, _ = resolve_and_pin_host(domain, port, allow_loopback=allow_loopback_for_testing) + + ctx = ssl_context or ssl.create_default_context() + raw_sock = socket.create_connection((pinned_ip, port), timeout=timeout) + try: + ssock = ctx.wrap_socket(raw_sock, server_hostname=domain) + return ssock, pinned_ip + except Exception: + raw_sock.close() + raise + +class SafeHttpResponse: + """Wrapper around HTTP response for SSRF-safe requests.""" + def __init__( + self, + status_code: int, + text: str, + url: str, + headers: Dict[str, str], + raw_response: Optional[httpx.Response] = None, + was_truncated: bool = False, + pinned_ip: Optional[str] = None + ): + self.status_code = status_code + self.text = text + self.url = url + self.headers = headers + self.raw_response = raw_response + self.was_truncated = was_truncated + self.pinned_ip = pinned_ip + +async def _do_safe_get( + client: httpx.AsyncClient, + initial_url: str, + headers: Optional[Dict[str, str]], + timeout: float, + max_redirects: int, + max_bytes: int, + transport: Optional[SSRFProtectedTransport] = None, + allow_loopback_for_testing: bool = False +) -> Tuple[bool, Optional[SafeHttpResponse], Optional[str]]: + current_url = initial_url + redirect_count = 0 + visited_urls: Set[str] = set() + + req_headers = dict(headers or {}) + + while True: + # 1. Pre-flight SSRF validation on current URL before connecting + is_safe, error_msg, parsed, resolved_ips = validate_target_url( + current_url, + allow_loopback=allow_loopback_for_testing + ) + if not is_safe: + return False, None, f"SSRF Protection blocked destination '{current_url}': {error_msg}" + + if current_url in visited_urls: + return False, None, f"SSRF Protection: Redirect loop detected at '{current_url}'" + visited_urls.add(current_url) + + # Pin validated destination IP in transport if available + current_port = parsed.port or (443 if parsed.scheme == "https" else 80) + pinned_ip = resolved_ips[0] if resolved_ips else None + if transport and pinned_ip: + transport.pinned_backend.pin_destination(parsed.hostname, current_port, pinned_ip) + + try: + async with client.stream("GET", current_url, headers=req_headers, follow_redirects=False) as resp: + if resp.status_code in (301, 302, 303, 307, 308) and "Location" in resp.headers: + redirect_count += 1 + if redirect_count > max_redirects: + return False, None, f"SSRF Protection: Maximum redirect limit ({max_redirects}) exceeded" + + raw_location = resp.headers.get("Location", "") + if not raw_location or not raw_location.strip(): + return False, None, "SSRF Protection: Malformed empty redirect Location header" + + try: + next_url = urllib.parse.urljoin(current_url, raw_location.strip()) + except Exception as e: + return False, None, f"SSRF Protection: Malformed redirect Location header '{raw_location}': {e}" + + # Explicitly reject non-http(s) schemes on redirect + parsed_next = urllib.parse.urlparse(next_url) + if parsed_next.scheme.lower() not in ("http", "https"): + return False, None, f"SSRF Protection: Redirect to prohibited scheme '{parsed_next.scheme}'" + + # Cross-origin authorization header stripping + next_port = parsed_next.port or (443 if parsed_next.scheme == "https" else 80) + if ( + parsed.hostname.lower() != parsed_next.hostname.lower() or + parsed.scheme.lower() != parsed_next.scheme.lower() or + current_port != next_port + ): + req_headers = {k: v for k, v in req_headers.items() if k.lower() not in ("authorization", "cookie")} + + current_url = next_url + continue + + # Stream response body up to max_bytes to prevent memory exhaustion / compression bombs + downloaded_bytes = bytearray() + was_truncated = False + async for chunk in resp.aiter_bytes(): + downloaded_bytes.extend(chunk) + if len(downloaded_bytes) > max_bytes: + downloaded_bytes = downloaded_bytes[:max_bytes] + was_truncated = True + break + + text = downloaded_bytes.decode("utf-8", errors="ignore") + return True, SafeHttpResponse( + status_code=resp.status_code, + text=text, + url=str(resp.url), + headers=dict(resp.headers), + raw_response=resp, + was_truncated=was_truncated, + pinned_ip=pinned_ip + ), None + + except (SSRFSecurityError, httpx.ConnectError) as e: + return False, None, f"Connection blocked by SSRF protection for '{current_url}': {str(e)}" + except httpx.RequestError as e: + return False, None, f"Network request failed for '{current_url}': {str(e)}" + except Exception as e: + return False, None, f"Request error for '{current_url}': {str(e)}" + +async def safe_http_get( + url: str, + headers: Optional[Dict[str, str]] = None, + timeout: float = 7.0, + max_redirects: int = 5, + max_bytes: int = 5 * 1024 * 1024, + client: Optional[httpx.AsyncClient] = None, + allow_loopback_for_testing: bool = False, + verify: Any = True +) -> Tuple[bool, Optional[SafeHttpResponse], Optional[str]]: + """ + Executes an SSRF-safe HTTP GET request with: + - Connection-level IP pinning and DNS rebinding protection + - Preserved TLS SNI and hostname certificate validation + - Manual per-hop redirect validation (follow_redirects=False) + - Cross-origin Authorization header stripping + - Loop detection (visited URLs tracking) + - Scheme whitelist validation (http/https only) + - Safe relative and absolute URL resolution with urljoin + - Malformed Location header protection + - Response body streaming with byte caps to prevent memory exhaustion and compression bombs + + Returns: (is_success, safe_http_response, error_message) + """ + if client is not None: + return await _do_safe_get( + client, + url, + headers, + timeout, + max_redirects, + max_bytes, + allow_loopback_for_testing=allow_loopback_for_testing + ) + else: + transport = SSRFProtectedTransport( + allow_loopback_for_testing=allow_loopback_for_testing, + verify=verify + ) + async with httpx.AsyncClient( + transport=transport, + timeout=timeout, + follow_redirects=False + ) as own_client: + return await _do_safe_get( + own_client, + url, + headers, + timeout, + max_redirects, + max_bytes, + transport=transport, + allow_loopback_for_testing=allow_loopback_for_testing + ) diff --git a/backend/tests/test_ai_drafting.py b/backend/tests/test_ai_drafting.py index 0914f73..c237546 100644 --- a/backend/tests/test_ai_drafting.py +++ b/backend/tests/test_ai_drafting.py @@ -546,3 +546,24 @@ def test_draft_deletion(client, db_session, sample_prospect): # Subsequent get returns 404 get_res = client.get(f"/api/drafts/{draft.id}") assert get_res.status_code == 404 + +def test_compatible_provider_models_deduplication(monkeypatch): + """ + Finding 5 Regression Test: + Verify that available_models for the compatible provider does not contain duplicate + entries when settings.COMPATIBLE_MODEL is set to a default like 'llama3' or a custom model. + """ + monkeypatch.setattr(settings, "COMPATIBLE_MODEL", "llama3") + providers = ai_service.list_providers() + comp = next((p for p in providers if p.provider_id == "compatible"), None) + assert comp is not None + assert comp.available_models.count("llama3") == 1 + assert len(comp.available_models) == len(set(comp.available_models)) + + # Test with custom model name + monkeypatch.setattr(settings, "COMPATIBLE_MODEL", "qwen2.5-coder") + providers_custom = ai_service.list_providers() + comp_custom = next((p for p in providers_custom if p.provider_id == "compatible"), None) + assert comp_custom is not None + assert comp_custom.available_models[0] == "qwen2.5-coder" + assert len(comp_custom.available_models) == len(set(comp_custom.available_models)) diff --git a/backend/tests/test_audit_engine.py b/backend/tests/test_audit_engine.py index 45b6bd9..8cc9933 100644 --- a/backend/tests/test_audit_engine.py +++ b/backend/tests/test_audit_engine.py @@ -333,3 +333,55 @@ async def mock_aiter_bytes(): size_findings = [f for f in findings if "exceeded" in f["title"].lower() or "limit" in f["title"].lower()] assert len(size_findings) > 0 assert size_findings[0]["check_status"] == "WARNING" + +def test_dns_spf_and_dmarc_apex_fallback(engine): + """ + Test that when a subdomain (e.g. www.brand.com) lacks direct SPF and DMARC records, + _inspect_dns falls back to querying the apex organizational domain (brand.com). + """ + mock_resolver = MagicMock() + + # Simulate: A record exists on www.brand.com + a_answer = MagicMock() + a_answer.__str__ = MagicMock(return_value="1.2.3.4") + + # Simulate: TXT on www.brand.com has no SPF; TXT on brand.com has SPF + apex_spf = MagicMock() + apex_spf.strings = [b"v=spf1 include:_spf.google.com ~all"] + + # Simulate: _dmarc.www.brand.com raises NXDOMAIN; _dmarc.brand.com returns DMARC + apex_dmarc = MagicMock() + apex_dmarc.strings = [b"v=DMARC1; p=reject;"] + + def mock_resolve(qname, rdtype, **kwargs): + if rdtype == "A": + return [a_answer] + if rdtype == "TXT": + if qname == "www.brand.com": + return [] + if qname == "brand.com": + return [apex_spf] + if qname == "_dmarc.www.brand.com": + raise Exception("NXDOMAIN") + if qname == "_dmarc.brand.com": + return [apex_dmarc] + raise Exception("NXDOMAIN") + + mock_resolver.resolve.side_effect = mock_resolve + + with patch("dns.resolver.Resolver", return_value=mock_resolver): + res = engine._inspect_dns("www.brand.com") + findings = res["findings"] + titles = [f["title"] for f in findings] + + # Should NOT flag missing SPF or missing DMARC because apex domain published them + assert not any("Missing Sender Policy Framework" in t for t in titles) + assert not any("Missing DMARC" in t for t in titles) + assert any("DNS A/IPv4 resolution verified" in t for t in titles) + +def test_dummy_template_email_rejected(): + """Verify that template placeholder emails like you@company.com are rejected.""" + from backend.app.services.email_scraper import is_valid_email_candidate + assert is_valid_email_candidate("you@company.com") is False + assert is_valid_email_candidate("user@company.com") is False + assert is_valid_email_candidate("info@realbusiness.in") is True diff --git a/backend/tests/test_csv_and_discovery_api.py b/backend/tests/test_csv_and_discovery_api.py index 2e120e0..4921ae4 100644 --- a/backend/tests/test_csv_and_discovery_api.py +++ b/backend/tests/test_csv_and_discovery_api.py @@ -218,3 +218,190 @@ def test_discovery_search_and_save_workflow(client, db_session): run_2 = search_res_2.json() assert run_2["results"][0]["is_saved"] is True assert run_2["results"][0]["matched_existing_id"] == in_db.id + +def test_deduplication_matching_semantics_and_edge_cases(db_session): + """ + Regression test for Finding 3: + Verifies that the optimized deduplication queries preserve 100% of matching semantics: + - Provider record ID + - Registered domain normalization + - Generic host avoidance + - Business name normalization (suffix removal, whitespace, punctuation) + - Locality separation (different cities not merged) + - Blank city edge-case matching + """ + # 1. Seed base prospects + p1 = ProspectModel( + id="pros_seed_1", + name="Apex Industrial Logistics Ltd.", + city="Atlanta", + state="Georgia", + country="USA", + website_url="https://www.apex-logistics.com/about", + source_provider="osm", + source_record_id="node/1001", + opportunity_score=75 + ) + p2 = ProspectModel( + id="pros_seed_2", + name="Generic Freelancer", + city="Seattle", + website_url="https://gmail.com", + opportunity_score=50 + ) + p3 = ProspectModel( + id="pros_seed_3", + name="Omni National Corp", + city="", + opportunity_score=60 + ) + db_session.add_all([p1, p2, p3]) + db_session.commit() + + # Case A: Provider ID match + match_prov = check_single_duplicate( + db=db_session, + name="Completely Different Name", + source_provider="osm", + source_record_id="node/1001" + ) + assert match_prov is not None + assert match_prov.match_type == "record_id" + assert match_prov.existing_id == "pros_seed_1" + + # Case B: Registered domain match (with different scheme & www) + match_domain = check_single_duplicate( + db=db_session, + name="Apex Log", + website_url="http://apex-logistics.com/contact-us" + ) + assert match_domain is not None + assert match_domain.match_type == "domain" + assert match_domain.existing_id == "pros_seed_1" + + # Case C: Generic domain avoidance (gmail.com should NOT trigger domain match) + match_generic = check_single_duplicate( + db=db_session, + name="Another Freelancer", + website_url="https://gmail.com" + ) + assert match_generic is None + + # Case D: Business name match with same city (suffix stripping: Inc. vs Ltd.) + match_name_city = check_single_duplicate( + db=db_session, + name="Apex Industrial Logistics Inc.", + city="Atlanta" + ) + assert match_name_city is not None + assert match_name_city.match_type == "name_location" + assert match_name_city.existing_id == "pros_seed_1" + + # Case E: Business name match with DIFFERENT city (must NOT merge branches) + match_diff_city = check_single_duplicate( + db=db_session, + name="Apex Industrial Logistics Inc.", + city="Chicago" + ) + assert match_diff_city is None + + # Case F: Both cities blank matches + match_blank_cities = check_single_duplicate( + db=db_session, + name="Omni National LLC", + city="" + ) + assert match_blank_cities is not None + assert match_blank_cities.match_type == "name_location" + assert match_blank_cities.existing_id == "pros_seed_3" + + # Case G: Candidate city provided but database city blank does NOT match + match_city_blank_db = check_single_duplicate( + db=db_session, + name="Omni National LLC", + city="Denver" + ) + assert match_city_blank_db is None + +def test_deduplication_performance_benchmark(db_session): + """ + Performance benchmark for Finding 3: + Seeds 500 prospects in SQLite and runs 50 duplicate checks + (20 expected duplicates, 30 unique). + Measures duration and asserts 100% accuracy and high throughput. + """ + import time + + # Seed 500 prospects + bulk_prospects = [] + for i in range(500): + bulk_prospects.append( + ProspectModel( + id=f"pros_perf_{i}", + name=f"Enterprise Solutions {i} Inc", + city=f"City_{i % 20}", + state="State", + country="Country", + website_url=f"https://www.enterprise-solutions-{i}.io", + source_provider="bulk_seed", + source_record_id=f"seed_{i}", + opportunity_score=60 + ) + ) + db_session.bulk_save_objects(bulk_prospects) + db_session.commit() + + # Prepare 50 candidates: 20 duplicates, 30 non-duplicates + candidates = [] + for i in range(20): + # 10 duplicate by domain, 10 duplicate by name + city + if i < 10: + candidates.append({ + "name": f"Alt Name {i}", + "website_url": f"http://enterprise-solutions-{i}.io/contact", + "city": "Unknown", + "expected_dupe": True, + "expected_id": f"pros_perf_{i}" + }) + else: + candidates.append({ + "name": f"Enterprise Solutions {i} LLC", + "website_url": "https://different-domain.org", + "city": f"City_{i % 20}", + "expected_dupe": True, + "expected_id": f"pros_perf_{i}" + }) + + for i in range(30): + candidates.append({ + "name": f"Completely Unique Company {i} Corp", + "website_url": f"https://unique-firm-{i}.com", + "city": f"NewCity_{i}", + "expected_dupe": False, + "expected_id": None + }) + + # Benchmark execution time + start_time = time.perf_counter() + dupe_count = 0 + for cand in candidates: + match = check_single_duplicate( + db=db_session, + name=cand["name"], + website_url=cand["website_url"], + city=cand["city"] + ) + if cand["expected_dupe"]: + assert match is not None, f"Failed to detect duplicate for {cand['name']}" + assert match.existing_id == cand["expected_id"] + dupe_count += 1 + else: + assert match is None, f"False duplicate detected for {cand['name']}" + + elapsed_ms = (time.perf_counter() - start_time) * 1000 + ops_per_sec = len(candidates) / (elapsed_ms / 1000) + + assert dupe_count == 20 + # On modern CPU, 50 queries with indexed/targeted LIKE should execute in under 500ms + assert elapsed_ms < 1000, f"Deduplication benchmark took too long: {elapsed_ms:.2f}ms" + print(f"\n[DEDUPLICATION BENCHMARK] 50 candidates against 500 DB prospects took {elapsed_ms:.2f}ms ({ops_per_sec:.1f} checks/sec). 100% accuracy verified.") diff --git a/backend/tests/test_health_and_audits_api.py b/backend/tests/test_health_and_audits_api.py index 9e45514..678adac 100644 --- a/backend/tests/test_health_and_audits_api.py +++ b/backend/tests/test_health_and_audits_api.py @@ -106,3 +106,90 @@ def test_audit_run_and_persistence(client): assert prospect_check["latestAuditId"] == report_id assert prospect_check["opportunityScore"] == 88 assert prospect_check["websiteStatus"] == "active" + +def test_list_audits_eager_loading_no_n_plus_one(client, db_session): + """ + Regression test for Finding 4: + Verify that GET /api/audits and GET /api/audits/prospect/{id} do NOT produce + N+1 queries when fetching multiple audit reports with findings. + With selectinload, finding retrieval is batched in a single IN query (O(1) queries). + """ + from sqlalchemy import event + from backend.tests.conftest import test_engine + from backend.app.models.audit import AuditReportModel, AuditFindingModel + from backend.app.models.prospect import ProspectModel + + # Seed parent prospect first to satisfy FK constraint + pros = ProspectModel( + id="pros_test_batch", + name="Batch Test Prospect", + website_url="https://example0.com", + opportunity_score=50 + ) + db_session.add(pros) + db_session.flush() + + # Seed 5 audits, each with 3 findings + for i in range(5): + rep = AuditReportModel( + id=f"aud_batch_{i}", + url=f"https://example{i}.com", + timestamp="2026-10-21 12:00 UTC", + status="completed", + overall_score=80 + i, + prospect_id="pros_test_batch" + ) + db_session.add(rep) + for j in range(3): + finding = AuditFindingModel( + id=f"f_batch_{i}_{j}", + audit_report_id=rep.id, + title=f"Finding {j} for audit {i}", + severity="medium", + check_status="WARNING", + observed="Observed detail", + evidence="Evidence detail", + confidence=1.0, + category="reachability" + ) + db_session.add(finding) + db_session.commit() + + queries = [] + def capture_queries(conn, cursor, statement, parameters, context, executemany): + queries.append(statement) + + event.listen(test_engine, "before_cursor_execute", capture_queries) + try: + res = client.get("/api/audits") + assert res.status_code == 200 + data = res.json() + assert len(data) >= 5 + # Ensure all 3 findings per audit report are returned in the response + batch_reports = [r for r in data if r["id"].startswith("aud_batch_")] + assert len(batch_reports) == 5 + for rep in batch_reports: + assert len(rep["findings"]) == 3 + + # Exactly 2 SELECT queries should be executed: + # 1. SELECT audit_reports ... + # 2. SELECT audit_findings ... WHERE audit_findings.audit_report_id IN (...) + select_queries = [q for q in queries if q.strip().upper().startswith("SELECT")] + assert len(select_queries) == 2, f"Expected 2 queries with selectinload, got {len(select_queries)}: {select_queries}" + finally: + event.remove(test_engine, "before_cursor_execute", capture_queries) + + # Test prospect-filtered audits also eagerly loads without N+1 + queries.clear() + event.listen(test_engine, "before_cursor_execute", capture_queries) + try: + res = client.get("/api/audits/prospect/pros_test_batch") + assert res.status_code == 200 + data = res.json() + assert len(data) == 5 + for rep in data: + assert len(rep["findings"]) == 3 + select_queries = [q for q in queries if q.strip().upper().startswith("SELECT")] + assert len(select_queries) == 2, f"Expected 2 queries with selectinload, got {len(select_queries)}: {select_queries}" + finally: + event.remove(test_engine, "before_cursor_execute", capture_queries) diff --git a/backend/tests/test_ssrf_real_network_integration.py b/backend/tests/test_ssrf_real_network_integration.py new file mode 100644 index 0000000..0bd76cf --- /dev/null +++ b/backend/tests/test_ssrf_real_network_integration.py @@ -0,0 +1,432 @@ +import asyncio +import datetime +import http.server +import ipaddress +import os +import socket +import ssl +import tempfile +import threading +import time +from typing import Tuple, Dict, Any, Optional + +import pytest +import httpx +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + +from backend.app.services.ssrf_validator import ( + SSRFProtectedTransport, + safe_http_get, + SSRFSecurityError, + connect_safe_tls_socket +) + +# ----------------------------------------------------------------------------- +# Certificate Helper for Local HTTPS Integration Tests +# ----------------------------------------------------------------------------- + +def generate_test_pki(hostname: str = "secure.localtest.me") -> Tuple[str, str, str]: + """Generates temporary CA, server certificate, and private key for local testing.""" + ca_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + ca_name = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, "Leddro Test Root CA"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Leddro Security Testing"), + ]) + ca_cert = ( + x509.CertificateBuilder() + .subject_name(ca_name) + .issuer_name(ca_name) + .public_key(ca_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1)) + .not_valid_after(datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=30)) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .add_extension(x509.SubjectKeyIdentifier.from_public_key(ca_key.public_key()), critical=False) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=True, + crl_sign=True, + encipher_only=False, + decipher_only=False + ), + critical=True + ) + .sign(ca_key, hashes.SHA256()) + ) + + server_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + server_name = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, hostname), + ]) + server_cert = ( + x509.CertificateBuilder() + .subject_name(server_name) + .issuer_name(ca_name) + .public_key(server_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1)) + .not_valid_after(datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=30)) + .add_extension( + x509.SubjectAlternativeName([ + x509.DNSName(hostname), + x509.IPAddress(ipaddress.IPv4Address("127.0.0.1")) + ]), + critical=False + ) + .add_extension(x509.SubjectKeyIdentifier.from_public_key(server_key.public_key()), critical=False) + .add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(ca_key.public_key()), critical=False) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=True, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False + ), + critical=True + ) + .add_extension( + x509.ExtendedKeyUsage([x509.ExtendedKeyUsageOID.SERVER_AUTH]), + critical=False + ) + .sign(ca_key, hashes.SHA256()) + ) + + ca_file = tempfile.NamedTemporaryFile("wb", delete=False, suffix=".pem") + ca_file.write(ca_cert.public_bytes(serialization.Encoding.PEM)) + ca_file.close() + + cert_file = tempfile.NamedTemporaryFile("wb", delete=False, suffix=".pem") + cert_file.write(server_cert.public_bytes(serialization.Encoding.PEM)) + cert_file.close() + + key_file = tempfile.NamedTemporaryFile("wb", delete=False, suffix=".pem") + key_file.write(server_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + )) + key_file.close() + + return ca_file.name, cert_file.name, key_file.name + +# ----------------------------------------------------------------------------- +# Local Real Network Test Servers +# ----------------------------------------------------------------------------- + +class RealTestServerHandler(http.server.BaseHTTPRequestHandler): + """Test HTTP handler that inspects real incoming socket headers, SNI, and paths.""" + server_stats: Dict[str, Any] = {} + + def log_message(self, format, *args): + # Suppress noisy standard HTTP logs during tests + pass + + def do_GET(self): + host_header = self.headers.get("Host", "") + self.server.received_host_headers.append(host_header) + self.server.received_paths.append(self.path) + + if self.path == "/hello": + self.send_response(200) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.send_header("Server", "LeddroTestServer/1.0") + self.end_headers() + self.wfile.write(b"Hello from real local test server!") + + elif self.path == "/redirect-to-metadata": + self.send_response(302) + self.send_header("Location", "http://169.254.169.254/latest/meta-data") + self.end_headers() + + elif self.path == "/redirect-to-private": + self.send_response(302) + self.send_header("Location", "http://10.0.0.1/admin-secret") + self.end_headers() + + elif self.path == "/large-payload": + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.end_headers() + # Send 500 KB of data in chunks + chunk = b"X" * 1024 + for _ in range(500): + self.wfile.write(chunk) + + elif self.path == "/slow": + time.sleep(1.5) + self.send_response(200) + self.end_headers() + self.wfile.write(b"Slow response") + + else: + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"OK") + +class RealLocalHttpServer: + def __init__(self, is_https: bool = False, cert_file: Optional[str] = None, key_file: Optional[str] = None): + self.is_https = is_https + self.cert_file = cert_file + self.key_file = key_file + self.received_host_headers = [] + self.received_paths = [] + self.received_sni = None + + self.httpd = http.server.HTTPServer(("127.0.0.1", 0), RealTestServerHandler) + self.httpd.received_host_headers = self.received_host_headers + self.httpd.received_paths = self.received_paths + self.port = self.httpd.server_address[1] + + if is_https: + ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_ctx.load_cert_chain(certfile=self.cert_file, keyfile=self.key_file) + + # Capture SNI via sni_callback if supported + def sni_callback(sock, req_server_name, cb_ctx): + self.received_sni = req_server_name + ssl_ctx.sni_callback = sni_callback + + self.httpd.socket = ssl_ctx.wrap_socket(self.httpd.socket, server_side=True) + + self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True) + self.thread.start() + + def shutdown(self): + try: + self.httpd.shutdown() + self.httpd.server_close() + except Exception: + pass + +@pytest.fixture(scope="module") +def real_http_server(): + server = RealLocalHttpServer(is_https=False) + yield server + server.shutdown() + +@pytest.fixture(scope="module") +def real_https_server(): + ca_path, cert_path, key_path = generate_test_pki(hostname="secure.localtest.me") + server = RealLocalHttpServer(is_https=True, cert_file=cert_path, key_file=key_path) + server.ca_path = ca_path + server.test_ssl_ctx = ssl.create_default_context(cafile=ca_path) + yield server + server.shutdown() + for p in (ca_path, cert_path, key_path): + try: + os.unlink(p) + except OSError: + pass + +# ============================================================================= +# LEVEL 2: REAL LOCAL NETWORK INTEGRATION TESTS (REAL SOCKETS) +# ============================================================================= + +@pytest.mark.asyncio +async def test_real_socket_connection_pins_destination_ip(real_http_server): + """ + Real socket integration test: + Verifies that the transport connects the actual TCP socket to the pinned IP (127.0.0.1) + and sends the original logical hostname in the HTTP Host header. + """ + port = real_http_server.port + hostname = "pinned.localtest.me" + + transport = SSRFProtectedTransport(allow_loopback_for_testing=True) + transport.pinned_backend.pin_destination(hostname, port, "127.0.0.1") + + async with httpx.AsyncClient(transport=transport, follow_redirects=False) as client: + resp = await client.get(f"http://{hostname}:{port}/hello") + + assert resp.status_code == 200 + assert resp.text == "Hello from real local test server!" + + # Verify physical socket destination directly from the backend stream + last_peer = transport.pinned_backend.last_connected_addr + assert last_peer is not None + assert last_peer[0] == "127.0.0.1" + assert last_peer[1] == port + + # Verify original logical hostname was sent in Host header + assert any(hostname in h for h in real_http_server.received_host_headers) + +@pytest.mark.asyncio +async def test_real_socket_dns_answer_change_cannot_redirect(real_http_server): + """ + DNS Rebinding Prevention: + Simulates DNS resolving to a public address during validation, but changing + to a prohibited address immediately before connection. + Because the transport is pinned to 127.0.0.1 (the validated destination), + the socket connects strictly to 127.0.0.1 and cannot be hijacked. + """ + from unittest.mock import patch + + port = real_http_server.port + hostname = "rebind.localtest.me" + + transport = SSRFProtectedTransport(allow_loopback_for_testing=True) + # The validated destination is pinned + transport.pinned_backend.pin_destination(hostname, port, "127.0.0.1") + + # Now simulate DNS answering with a prohibited internal address (10.99.88.77) + with patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("10.99.88.77", port))]): + async with httpx.AsyncClient(transport=transport, follow_redirects=False) as client: + resp = await client.get(f"http://{hostname}:{port}/hello") + + # Connection succeeded because socket connected to pinned 127.0.0.1 + assert resp.status_code == 200 + assert transport.pinned_backend.last_connected_addr[0] == "127.0.0.1" + +@pytest.mark.asyncio +async def test_real_https_sni_and_certificate_hostname_verification(real_https_server): + """ + Real HTTPS socket integration test: + Verifies that for an encrypted HTTPS connection: + - The TCP socket connects to the pinned IP (127.0.0.1). + - TLS Client Hello contains the original logical hostname in SNI. + - TLS certificate verification verifies against the logical hostname. + - The HTTP request header contains the logical Host. + """ + port = real_https_server.port + hostname = "secure.localtest.me" + + transport = SSRFProtectedTransport( + allow_loopback_for_testing=True, + verify=real_https_server.test_ssl_ctx + ) + transport.pinned_backend.pin_destination(hostname, port, "127.0.0.1") + + async with httpx.AsyncClient(transport=transport, follow_redirects=False) as client: + resp = await client.get(f"https://{hostname}:{port}/hello") + + assert resp.status_code == 200 + assert resp.text == "Hello from real local test server!" + + # Verify physical socket connected to pinned IP + assert transport.pinned_backend.last_connected_addr[0] == "127.0.0.1" + + # Verify SNI received by the real TLS server + if real_https_server.received_sni is not None: + assert real_https_server.received_sni == hostname + + # Verify Host header received by server + assert any(hostname in h for h in real_https_server.received_host_headers) + +@pytest.mark.asyncio +async def test_real_https_certificate_mismatch_rejected(real_https_server): + """ + Verifies that if the server certificate does not match the logical request hostname, + TLS certificate verification strictly fails and does NOT connect. + """ + port = real_https_server.port + wrong_hostname = "wrong-host.localtest.me" + + transport = SSRFProtectedTransport( + allow_loopback_for_testing=True, + verify=real_https_server.test_ssl_ctx + ) + # Pin wrong_hostname to 127.0.0.1 (where cert is for secure.localtest.me) + transport.pinned_backend.pin_destination(wrong_hostname, port, "127.0.0.1") + + with pytest.raises(httpx.ConnectError) as exc_info: + async with httpx.AsyncClient(transport=transport, follow_redirects=False) as client: + await client.get(f"https://{wrong_hostname}:{port}/hello") + + err_text = str(exc_info.value).lower() + assert "certificate verify failed" in err_text or "hostname mismatch" in err_text or "ssl" in err_text + +@pytest.mark.asyncio +async def test_real_http_unsafe_redirect_blocked_before_prohibited_connection(real_http_server): + """ + Redirect security test with real server: + Real HTTP server returns a 302 redirect targeting cloud metadata (169.254.169.254). + SSRF safe_http_get must revalidate the redirect target and block it BEFORE + any network connection to the prohibited destination occurs. + """ + port = real_http_server.port + initial_url = f"http://127.0.0.1:{port}/redirect-to-metadata" + + success, resp, error = await safe_http_get( + initial_url, + allow_loopback_for_testing=True, + max_redirects=5 + ) + + assert not success + assert resp is None + assert "SSRF Protection blocked destination" in error + assert "169.254.169.254" in error + +@pytest.mark.asyncio +async def test_real_server_response_size_limit(real_http_server): + """ + Resource limit test with real server: + Real HTTP server streams a 500 KB response body. + safe_http_get with max_bytes=8192 (8 KB) truncates the body and flags was_truncated=True. + """ + port = real_http_server.port + url = f"http://127.0.0.1:{port}/large-payload" + + success, resp, error = await safe_http_get( + url, + allow_loopback_for_testing=True, + max_bytes=8192 + ) + + assert success + assert resp is not None + assert resp.was_truncated is True + assert len(resp.text) == 8192 + +@pytest.mark.asyncio +async def test_real_slow_server_timeout(real_http_server): + """ + Timeout test with real server: + Real server delays response by 1.5 seconds. + safe_http_get with timeout=0.25 seconds enforces the deadline cleanly. + """ + port = real_http_server.port + url = f"http://127.0.0.1:{port}/slow" + + success, resp, error = await safe_http_get( + url, + allow_loopback_for_testing=True, + timeout=0.25 + ) + + assert not success + assert resp is None + assert "failed" in error.lower() or "timeout" in error.lower() + +def test_ipv6_real_socket_support(): + """ + IPv6 environment check: + Tests real socket binding on IPv6 loopback (::1). + If IPv6 is not available or disabled by the OS, explicitly reports NOT RUN. + """ + try: + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock.bind(("::1", 0)) + port = sock.getsockname()[1] + sock.listen(1) + sock.close() + except Exception as e: + pytest.skip(f"IPv6 real socket test NOT RUN on this system: {e}") + + # If socket binding succeeded, IPv6 is supported + assert port > 0 diff --git a/backend/tests/test_ssrf_validator.py b/backend/tests/test_ssrf_validator.py index f720688..1aa6bd9 100644 --- a/backend/tests/test_ssrf_validator.py +++ b/backend/tests/test_ssrf_validator.py @@ -98,3 +98,304 @@ def test_unresolvable_public_domain_is_safe_from_ssrf(): assert parsed.hostname == "nonexistent-prospect-domain.xyz" assert ips == [] +@pytest.mark.asyncio +async def test_safe_http_get_redirect_to_loopback_blocked(): + """ + Regression test for Finding 2: + Public URL redirects to 127.0.0.1. + Must block the redirect and NEVER issue an HTTP request to loopback. + """ + import httpx + from backend.app.services.ssrf_validator import safe_http_get + + requested_urls = [] + def handler(request: httpx.Request): + url_str = str(request.url) + requested_urls.append(url_str) + if "example.com" in url_str: + return httpx.Response(302, headers={"Location": "http://127.0.0.1:8000/internal-secret"}) + return httpx.Response(200, text="SECRET_EXPOSED") + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + with patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 80))]): + success, resp, error = await safe_http_get("http://example.com/test", client=client) + + assert not success + assert resp is None + assert "SSRF Protection blocked destination" in error + assert "127.0.0.1" in error + assert "http://127.0.0.1:8000/internal-secret" not in requested_urls + +@pytest.mark.asyncio +async def test_safe_http_get_redirect_to_private_ipv4_blocked(): + """Public URL redirects to RFC 1918 private network (10.0.0.1, 192.168.1.1).""" + import httpx + from backend.app.services.ssrf_validator import safe_http_get + + requested_urls = [] + def handler(request: httpx.Request): + requested_urls.append(str(request.url)) + return httpx.Response(302, headers={"Location": "http://10.0.0.1/admin"}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + with patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 80))]): + success, resp, error = await safe_http_get("http://example.com/login", client=client) + + assert not success + assert "10.0.0.1" in error + assert "http://10.0.0.1/admin" not in requested_urls + +@pytest.mark.asyncio +async def test_safe_http_get_redirect_to_ipv6_loopback_or_private_blocked(): + """Public URL redirects to IPv6 loopback ([::1]) or ULA ([fc00::1]).""" + import httpx + from backend.app.services.ssrf_validator import safe_http_get + + requested_urls = [] + def handler(request: httpx.Request): + requested_urls.append(str(request.url)) + return httpx.Response(302, headers={"Location": "http://[::1]:8080/metrics"}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + with patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 80))]): + success, resp, error = await safe_http_get("http://example.com/stats", client=client) + + assert not success + assert "::1" in error + assert "http://[::1]:8080/metrics" not in requested_urls + +@pytest.mark.asyncio +async def test_safe_http_get_multiple_redirect_hops_and_limit(): + """Allows safe multiple hops up to max_redirects, but halts if exceeded.""" + import httpx + from backend.app.services.ssrf_validator import safe_http_get + + hops = [] + def handler(request: httpx.Request): + hops.append(str(request.url)) + idx = int(request.url.path.strip("/").split("-")[-1]) + if idx < 10: + return httpx.Response(302, headers={"Location": f"http://example.com/hop-{idx + 1}"}) + return httpx.Response(200, text="FINISH") + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + with patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 80))]): + success, resp, error = await safe_http_get("http://example.com/hop-1", client=client, max_redirects=3) + + assert not success + assert "Maximum redirect limit (3) exceeded" in error + assert len(hops) == 4 # hop-1, hop-2, hop-3, hop-4 (4th hop triggers limit check) + +@pytest.mark.asyncio +async def test_safe_http_get_redirect_loop_detected(): + """Detects and prevents infinite redirect loops (A -> B -> A).""" + import httpx + from backend.app.services.ssrf_validator import safe_http_get + + hops = [] + def handler(request: httpx.Request): + url_str = str(request.url) + hops.append(url_str) + if "loop-a" in url_str: + return httpx.Response(302, headers={"Location": "http://example.com/loop-b"}) + else: + return httpx.Response(302, headers={"Location": "http://example.com/loop-a"}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + with patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 80))]): + success, resp, error = await safe_http_get("http://example.com/loop-a", client=client) + + assert not success + assert "Redirect loop detected" in error + assert len(hops) <= 3 + +@pytest.mark.asyncio +async def test_safe_http_get_malformed_location_header(): + """Handles malformed or empty Location headers gracefully.""" + import httpx + from backend.app.services.ssrf_validator import safe_http_get + + def handler(request: httpx.Request): + return httpx.Response(302, headers={"Location": ""}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + with patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 80))]): + success, resp, error = await safe_http_get("http://example.com/malformed", client=client) + + assert not success + assert "Malformed empty redirect Location" in error + +@pytest.mark.asyncio +async def test_safe_http_get_redirect_to_prohibited_scheme(): + """Rejects redirects to prohibited schemes (file://, javascript:, gopher://).""" + import httpx + from backend.app.services.ssrf_validator import safe_http_get + + for bad_scheme in ["file:///etc/passwd", "javascript:alert(1)", "gopher://127.0.0.1"]: + def handler(request: httpx.Request, loc=bad_scheme): + return httpx.Response(302, headers={"Location": loc}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + with patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 80))]): + success, resp, error = await safe_http_get("http://example.com/exploit", client=client) + + assert not success + assert any(term in error.lower() for term in ["prohibited scheme", "unsupported protocol", "invalidurl", "for absolute urls"]) + +@pytest.mark.asyncio +async def test_email_scraper_with_ssrf_redirect_protection(): + """ + Regression test for Finding 2: + scrape_website_contacts crawls homepage that redirects to cloud metadata. + Must block the redirect and never expose internal metadata. + """ + import httpx + from backend.app.services.email_scraper import scrape_website_contacts + + requested_urls = [] + def handler(request: httpx.Request): + url_str = str(request.url) + requested_urls.append(url_str) + if "innocent-looking-site.com" in url_str: + return httpx.Response(302, headers={"Location": "http://169.254.169.254/latest/meta-data"}) + return httpx.Response(200, text="IAM_ROLE_CREDENTIALS") + + mock_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + with patch("httpx.AsyncClient", return_value=mock_client), \ + patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 80))]): + res = await scrape_website_contacts("https://innocent-looking-site.com") + + assert "http://169.254.169.254/latest/meta-data" not in requested_urls + assert res["emails"] == [] + +# ========================================================================= +# LEVEL 1: COMPREHENSIVE ADVERSARIAL AND CONNECTION-PINNING UNIT TESTS +# ========================================================================= + +def test_mixed_public_and_private_dns_answers_rejected(): + """ + Adversarial test: + DNS returns a mix of a valid public IP and a prohibited private IP. + Policy: Reject the entire answer set to prevent attacker-directed fallback. + """ + from backend.app.services.ssrf_validator import resolve_and_pin_host, SSRFSecurityError + with patch("socket.getaddrinfo", return_value=[ + (None, None, None, None, ("93.184.216.34", 80)), + (None, None, None, None, ("10.0.0.1", 80)) + ]): + with pytest.raises(SSRFSecurityError) as exc_info: + resolve_and_pin_host("dual-homed-attack.com", 80) + assert "prohibited ip" in str(exc_info.value).lower() or "security policy" in str(exc_info.value).lower() + +def test_ipv4_mapped_ipv6_addresses_prohibited(): + """ + Adversarial test: + Attackers often encode prohibited IPv4 addresses inside IPv6 mapped prefixes (::ffff:...). + Must extract the mapped IPv4 and enforce private/loopback/cloud-metadata rules. + """ + from backend.app.services.ssrf_validator import is_ip_allowed, validate_target_url + + # IPv4 loopback mapped into IPv6 + allowed, reason = is_ip_allowed("::ffff:127.0.0.1") + assert not allowed + assert "prohibited" in reason.lower() or "loopback" in reason.lower() + + # AWS metadata mapped into IPv6 + allowed, reason = is_ip_allowed("::ffff:169.254.169.254") + assert not allowed + assert "prohibited" in reason.lower() or "metadata" in reason.lower() or "link-local" in reason.lower() + + # RFC 1918 10.x mapped into IPv6 + allowed, reason = is_ip_allowed("::ffff:10.50.1.1") + assert not allowed + assert "prohibited" in reason.lower() or "private" in reason.lower() + + # Direct URL validation + is_safe, reason, _, _ = validate_target_url("http://[::ffff:127.0.0.1]/") + assert not is_safe + +def test_multiple_public_dns_answers_deterministic_pinning(): + """ + When DNS returns multiple legitimate public IP addresses (both IPv4 and IPv6), + the validator deterministically pins a valid public address without failing. + """ + from backend.app.services.ssrf_validator import resolve_and_pin_host + with patch("socket.getaddrinfo", return_value=[ + (None, None, None, None, ("2606:2800:220:1:248:1893:25c8:1946", 443)), + (None, None, None, None, ("93.184.216.34", 443)), + (None, None, None, None, ("93.184.216.35", 443)) + ]): + pinned_ip, resolved = resolve_and_pin_host("example.com", 443) + assert pinned_ip == "93.184.216.34" # Deterministic IPv4 preference + assert len(resolved) == 3 + +def test_dns_resolution_failure_handling(): + """DNS resolution errors raise predictable SSRFSecurityError in resolve_and_pin_host.""" + import socket + from backend.app.services.ssrf_validator import resolve_and_pin_host, SSRFSecurityError + with patch("socket.getaddrinfo", side_effect=socket.gaierror(11001, "getaddrinfo failed")): + with pytest.raises(SSRFSecurityError) as exc_info: + resolve_and_pin_host("unregistered-bogus-domain.invalid", 80) + assert "dns resolution failed" in str(exc_info.value).lower() + +@pytest.mark.asyncio +async def test_cross_origin_authorization_header_stripping(): + """ + Redirect security: + When redirecting across different origins (e.g. source.com -> other-host.com), + sensitive Authorization and Cookie headers must be stripped to prevent credential leakage. + """ + import httpx + from backend.app.services.ssrf_validator import safe_http_get + + captured_headers = {} + def handler(request: httpx.Request): + url_str = str(request.url) + if "source.com" in url_str: + return httpx.Response(302, headers={"Location": "https://other-domain.com/landing"}) + captured_headers["auth"] = request.headers.get("Authorization") + captured_headers["cookie"] = request.headers.get("Cookie") + return httpx.Response(200, text="OK") + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + with patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 443))]): + headers = {"Authorization": "Bearer SECRET_TOKEN", "Cookie": "session=ABC"} + success, resp, _ = await safe_http_get("https://source.com/start", headers=headers, client=client) + + assert success + assert captured_headers.get("auth") is None + assert captured_headers.get("cookie") is None + +@pytest.mark.asyncio +async def test_response_size_limit_truncation(): + """ + Resource limits: + Oversized responses or compression bombs exceeding max_bytes are truncated, + flagged as was_truncated=True, and do not exhaust application memory. + """ + import httpx + from backend.app.services.ssrf_validator import safe_http_get + + def handler(request: httpx.Request): + # 100 KB payload + return httpx.Response(200, text="A" * 102400) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + with patch("socket.getaddrinfo", return_value=[(None, None, None, None, ("93.184.216.34", 80))]): + # Cap at 10 KB + success, resp, _ = await safe_http_get("http://example.com/large", max_bytes=10240, client=client) + + assert success + assert resp.was_truncated is True + assert len(resp.text) <= 10240 + 8192 # bounded by chunk size + +def test_unix_domain_sockets_explicitly_blocked(): + """Connection backend strictly rejects Unix domain socket connections.""" + import pytest + from backend.app.services.ssrf_validator import PinnedAsyncNetworkBackend, SSRFSecurityError + backend = PinnedAsyncNetworkBackend() + with pytest.raises(SSRFSecurityError) as exc_info: + import asyncio + asyncio.run(backend.connect_unix_socket("/var/run/docker.sock")) + assert "unix domain sockets are prohibited" in str(exc_info.value).lower() + diff --git a/src/context/AppContext.tsx b/src/context/AppContext.tsx index ecd8054..9210147 100644 --- a/src/context/AppContext.tsx +++ b/src/context/AppContext.tsx @@ -964,38 +964,10 @@ Research & Technical Auditing Lead`, confirmed: boolean; }): Promise<{ success: boolean; messageId?: string; provider?: string; error?: string }> => { try { - let isSimulated = false; - let data: any = null; - + // 1. Explicit offline / mock mode fallback only when backend is recognized as offline if (backendOnline === false) { - isSimulated = true; - } else { - try { - const res = await fetch('/api/email/send', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - draft_id: payload.draftId, - recipient_email: payload.recipient, - subject: payload.subject, - body: payload.body, - confirmed: payload.confirmed - }) - }); - - if (res.ok) { - data = await res.json(); - } else { - isSimulated = true; - } - } catch { - isSimulated = true; - } - } - - if (isSimulated) { await new Promise((r) => setTimeout(r, 600)); - const messageId = 'demo-msg-' + Math.random().toString(36).substring(2, 10); + const demoMessageId = 'demo-msg-' + Math.random().toString(36).substring(2, 10); setEmailDrafts((prev) => prev.map((d) => (d.id === payload.draftId ? { ...d, status: 'submitted', updatedAt: new Date().toISOString() } : d)) ); @@ -1009,13 +981,73 @@ Research & Technical Auditing Lead`, date: 'Just now', status: 'sent', provider: 'Gmail API (OAuth 2.0)', - providerMessageId: messageId, + providerMessageId: demoMessageId, bodySnippet: payload.body.substring(0, 100) + '...', createdAt: new Date().toISOString() }; setEmailHistory((prev) => [histItem, ...prev]); - addToast(`Proposal dispatched via demo gateway (Message ID: ${messageId})`, 'success'); - return { success: true, messageId, provider: 'Gmail API (OAuth 2.0)' }; + addToast(`Proposal dispatched via demo gateway (Message ID: ${demoMessageId})`, 'success'); + return { success: true, messageId: demoMessageId, provider: 'Gmail API (OAuth 2.0)' }; + } + + // 2. Connected live backend dispatch + let res: Response; + try { + res = await fetch('/api/email/send', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + draft_id: payload.draftId, + recipient_email: payload.recipient, + subject: payload.subject, + body: payload.body, + confirmed: payload.confirmed + }) + }); + } catch (networkErr: any) { + // Live backend is unreachable + const errMsg = networkErr?.message || 'Network error: Backend server is unreachable.'; + addToast(`Email dispatch failed: ${errMsg}`, 'error'); + await refreshDrafts(); + return { success: false, error: errMsg }; + } + + // 3. Handle live HTTP error response (never simulate success on backend error) + if (!res.ok) { + let errorDetail = `Backend dispatch failed (HTTP ${res.status})`; + try { + const errData = await res.json(); + if (errData && (errData.detail || errData.message)) { + errorDetail = errData.detail || errData.message; + } + } catch { + const errText = await res.text().catch(() => ''); + if (errText) errorDetail = errText; + } + + const isUncertain = errorDetail.toLowerCase().includes('uncertain'); + const toastType = res.status === 409 || isUncertain ? 'warning' : 'error'; + addToast(errorDetail, toastType); + await refreshDrafts(); + await fetchEmailDispatchStatus(); + return { success: false, error: errorDetail }; + } + + const data = await res.json(); + + // Check if backend recorded an uncertain delivery outcome + if (data.status === 'uncertain') { + const warnMsg = data.message || 'Dispatch completed with uncertain delivery status. Please inspect your email provider Sent folder before retrying.'; + addToast(warnMsg, 'warning'); + await refreshDrafts(); + await refreshEmailHistory(); + await fetchEmailDispatchStatus(); + return { + success: false, + messageId: data.gmail_message_id, + provider: data.provider || 'Gmail API', + error: warnMsg + }; } const messageId = data.gmail_message_id; @@ -1028,7 +1060,7 @@ Research & Technical Auditing Lead`, addToast(`Proposal accepted by ${provider} gateway (Message ID: ${messageId})`, 'success'); return { success: true, messageId, provider }; } catch (err: any) { - const errMsg = err.message || 'Network error dispatching email'; + const errMsg = err.message || 'Unexpected error dispatching email'; addToast(errMsg, 'error'); await refreshDrafts(); return { success: false, error: errMsg }; diff --git a/src/pages/Integrations.tsx b/src/pages/Integrations.tsx index 8558006..307b43e 100644 --- a/src/pages/Integrations.tsx +++ b/src/pages/Integrations.tsx @@ -643,7 +643,7 @@ export const Integrations: React.FC = () => { onChange={(e) => setSelectedModel({ ...selectedModel, [p.provider_id]: e.target.value })} className="bg-[#111214] border border-[#2B2F36] rounded px-2.5 py-1.5 text-[12px] font-mono text-[#F3F4F6] focus:outline-none focus:border-[#D8E7A2]" > - {p.available_models.map((m) => ( + {Array.from(new Set(p.available_models || [])).map((m) => (