Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ All notable changes to this project are documented here. The format follows
- Documentation site with screenshots, pre-commit hooks, a Docker image, a GitHub Action and
release automation (`Prepare release` workflow, PyPI publishing, action repository sync).

- Framework packs: Django, FastAPI, Flask, NestJS, Spring Boot, ASP.NET Core, Terraform and
CloudFormation, with detectors, 20 new rules (SH014-SH020, SH106-SH109, SH206-SH208, SH303,
SH401-SH405), fixtures and docs. SH001-SH003 now cover these stacks too, and `devshield fix`
creates starter CI workflows for Python, Java and .NET projects.

### Changed

- Scans of large repositories are much faster: files that cannot match a rule are ruled out with
Expand Down
20 changes: 20 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,25 @@ auto_fix:
```


# Supported frameworks

| Stack | Detected from | Highlights |
|-------|---------------|------------|
| Laravel | `artisan`, `laravel/framework` | debug mode, CORS, cookies, rate limiting |
| Node, Express | `package.json` | CORS, headers, JWT, sessions, dynamic code |
| React, Next.js | `package.json` | XSS sinks, secrets in the bundle, CSP, bundle size |
| NestJS | `@nestjs/core` | helmet, ValidationPipe, CORS |
| Django | `django`, `manage.py` | `DEBUG`, `ALLOWED_HOSTS`, `SECRET_KEY`, `SECURE_*`, CSRF, cookies |
| FastAPI | `fastapi` | CORS, `/docs` exposure, secrets, SQL injection |
| Flask | `flask` | `debug=True`, `SECRET_KEY`, cookies, CSRF, Flask-CORS |
| Spring Boot | `pom.xml`, Gradle | Actuator, `application.properties` secrets, CSRF, `permitAll`, `ddl-auto` |
| ASP.NET Core | `Microsoft.NET.Sdk.Web` | developer exception page, `appsettings.json` secrets, CORS, Swagger |
| Terraform, CloudFormation | `*.tf`, templates | public S3, IAM `*`, open security groups, unencrypted or public databases |
| Docker, GitHub Actions | `Dockerfile`, workflows | root user, pinned actions, token permissions |

See [docs/frameworks.md](docs/frameworks.md) for the rules that apply to each stack.


# Rule Categories

| Rules | Category |
Expand All @@ -473,6 +492,7 @@ auto_fix:
| SH100–SH199 | Authentication |
| SH200–SH299 | Secrets |
| SH300–SH399 | Configuration |
| SH400–SH499 | Cloud and infrastructure as code |
| PERF001–PERF099 | Performance |
| PERF100–PERF199 | Docker |
| PERF200–PERF299 | Frontend |
Expand Down
102 changes: 100 additions & 2 deletions devshield/core/context.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

import ast
import json
import os
import re
Expand All @@ -14,26 +15,45 @@
from devshield.core.git import tracked_files
from devshield.core.models import DetectedFramework, Framework
from devshield.core.parsers import (
ConfigEntry,
EnvEntry,
Instruction,
flatten_json,
flatten_yaml,
json_line_map,
load_template,
parse_dockerfile,
parse_dotenv,
parse_hcl,
parse_module,
parse_pipfile,
parse_properties,
parse_pyproject,
parse_requirements_txt,
yaml_line_map,
)
from devshield.core.parsers.hcl import HclBlock

DEFAULT_EXCLUDES = frozenset(
{
".eggs",
".git",
".gradle",
".mypy_cache",
".next",
".nuxt",
".pytest_cache",
".ruff_cache",
".terraform",
".tox",
".venv",
"__pycache__",
"build",
"coverage",
"dist",
"node_modules",
"obj",
"target",
"vendor",
"venv",
}
Expand All @@ -43,6 +63,12 @@
COMPOSE_PATTERNS = ("docker-compose*.yml", "docker-compose*.yaml", "compose.yml", "compose.yaml")
WORKFLOW_PATTERNS = (".github/workflows/*.yml", ".github/workflows/*.yaml")
SOURCE_PATTERNS = ("*.js", "*.jsx", "*.ts", "*.tsx", "*.mjs", "*.cjs")
PYTHON_PATTERNS = ("*.py",)
JAVA_PATTERNS = ("*.java", "*.kt")
CSHARP_PATTERNS = ("*.cs",)
LOCKFILE_NAMES = frozenset({"package-lock.json", "composer.lock", "yarn.lock", "pnpm-lock.yaml"})
TEMPLATE_MAX_BYTES = 512 * 1024
REQUIREMENTS_PATTERNS = ("requirements*.txt", "requirements/*.txt")
EXAMPLE_SUFFIXES = (".example", ".sample", ".template", ".dist", ".defaults")
TEST_DIRS = frozenset({"test", "tests", "__tests__", "spec", "e2e", "cypress", "fixtures"})

Expand All @@ -61,8 +87,14 @@ def _is_example(name: str) -> bool:

def is_test_path(rel: str) -> bool:
*directories, name = rel.split("/")
return bool(TEST_DIRS.intersection(directories)) or bool(
re.search(r"\.(?:test|spec)\.|_test\.|^test_", name)
return (
bool(TEST_DIRS.intersection(directories))
or bool(
re.search(
r"\.(?:test|spec)\.|_test\.|^test_|^conftest\.py$|Tests?\.(?:cs|java|kt)$", name
)
)
or any(d.endswith((".Tests", ".UnitTests", ".IntegrationTests")) for d in directories)
)


Expand All @@ -82,6 +114,7 @@ def __init__(
self._resolved: dict[str, Path | None] = {}
self._listed: frozenset[str] | None = None
self._ignored: dict[str, bool] = {}
self._ast: dict[str, ast.Module | None] = {}
self._line_maps: dict[str, dict[tuple[str | int, ...], int]] = {}

@cached_property
Expand Down Expand Up @@ -200,6 +233,67 @@ def _env_files(self, *, example: bool) -> list[str]:
def source_files(self) -> list[str]:
return [f for f in self.match(*SOURCE_PATTERNS) if not is_test_path(f)]

@cached_property
def python_files(self) -> list[str]:
return [f for f in self.match(*PYTHON_PATTERNS) if not is_test_path(f)]

@cached_property
def java_files(self) -> list[str]:
return [f for f in self.match(*JAVA_PATTERNS) if not is_test_path(f)]

@cached_property
def csharp_files(self) -> list[str]:
return [f for f in self.match(*CSHARP_PATTERNS) if not is_test_path(f)]

@cached_property
def python_dependencies(self) -> dict[str, str]:
found: dict[str, str] = {}
for file in self.match(*REQUIREMENTS_PATTERNS):
found.update(parse_requirements_txt(self.read_text(file) or ""))
for file in self.match("pyproject.toml"):
found.update(parse_pyproject(self.read_text(file) or ""))
for file in self.match("Pipfile"):
found.update(parse_pipfile(self.read_text(file) or ""))
return found

def pyast(self, rel: str) -> ast.Module | None:
if rel not in self._ast:
self._ast[rel] = parse_module(self.read_text(rel) or "")
return self._ast[rel]

@cached_property
def terraform_files(self) -> list[str]:
return self.match("*.tf")

def terraform(self, rel: str) -> list[HclBlock]:
return parse_hcl(self.read_text(rel) or "")

@cached_property
def cloudformation_files(self) -> list[str]:
workflows = set(self.workflow_files)
found: list[str] = []
for file in self.match("*.yml", "*.yaml", "*.json", "*.template"):
name = file.rsplit("/", 1)[-1]
if file in workflows or name in LOCKFILE_NAMES or self.size(file) > TEMPLATE_MAX_BYTES:
continue
text = self.read_text(file) or ""
if "AWS::" in text and "Resources" in text:
found.append(file)
return found

def template(self, rel: str) -> dict[str, Any] | None:
"""A parsed CloudFormation template (YAML with tags, or JSON)."""
return load_template(self.read_text(rel) or "", is_json=rel.endswith(".json"))

def config_entries(self, rel: str) -> list[ConfigEntry]:
"""Dotted key/value pairs of a .properties, YAML or JSON config file."""
text = self.read_text(rel) or ""
if rel.endswith(".properties"):
return parse_properties(text)
if rel.endswith((".yml", ".yaml")):
return flatten_yaml(text)
return flatten_json(text) if rel.endswith(".json") else []

@cached_property
def tracked(self) -> frozenset[str] | None:
return tracked_files(self.root)
Expand Down Expand Up @@ -262,6 +356,10 @@ def frameworks(self) -> tuple[DetectedFramework, ...]:
def has_framework(self, framework: Framework) -> bool:
return any(d.framework == framework for d in self.frameworks)

def has_framework_any(self, frameworks: Iterable[Framework]) -> bool:
wanted = set(frameworks)
return any(d.framework in wanted for d in self.frameworks)


def _join(parent: str, name: str) -> str:
return f"{parent}/{name}" if parent else name
63 changes: 63 additions & 0 deletions devshield/core/detect.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
from __future__ import annotations

import re
from collections.abc import Callable
from typing import TYPE_CHECKING

from devshield.core.models import Confidence, DetectedFramework, Framework
from devshield.core.search import search

if TYPE_CHECKING:
from devshield.core.context import ProjectContext
Expand Down Expand Up @@ -43,6 +45,59 @@ def _node(ctx: ProjectContext) -> DetectedFramework | None:
return _detected(Framework.NODE, ["package.json"] if ctx.exists("package.json") else [])


def _python_framework(framework: Framework, package: str, imports: str) -> Detector:
pattern = re.compile(rf"^\s*(?:from|import)\s+{imports}\b")

def detect(ctx: ProjectContext) -> DetectedFramework | None:
evidence = []
if package in ctx.python_dependencies:
evidence.append(f"Python dependency {package}")
hit = next(search(ctx, ctx.python_files, pattern), None)
if hit:
evidence.append(f"{hit.file} imports {imports}")
return _detected(framework, evidence, high_at=2)

return detect


def _django(ctx: ProjectContext) -> DetectedFramework | None:
evidence = []
if "django" in ctx.python_dependencies:
evidence.append("Python dependency django")
if "DJANGO_SETTINGS_MODULE" in (ctx.read_text("manage.py") or ""):
evidence.append("manage.py")
return _detected(Framework.DJANGO, evidence, high_at=2)


def _spring(ctx: ProjectContext) -> DetectedFramework | None:
evidence = []
if "spring-boot" in (ctx.read_text("pom.xml") or ""):
evidence.append("pom.xml uses spring-boot")
for gradle in ("build.gradle", "build.gradle.kts"):
text = ctx.read_text(gradle) or ""
if "org.springframework.boot" in text or "spring-boot-starter" in text:
evidence.append(f"{gradle} uses Spring Boot")
return _detected(Framework.SPRING, evidence)


def _aspnet(ctx: ProjectContext) -> DetectedFramework | None:
evidence = [
f
for f in ctx.match("*.csproj")
if "Microsoft.NET.Sdk.Web" in (ctx.read_text(f) or "")
or "Microsoft.AspNetCore" in (ctx.read_text(f) or "")
]
return _detected(Framework.ASPNET, evidence)


def _terraform(ctx: ProjectContext) -> DetectedFramework | None:
return _detected(Framework.TERRAFORM, ctx.terraform_files[:3])


def _cloudformation(ctx: ProjectContext) -> DetectedFramework | None:
return _detected(Framework.CLOUDFORMATION, ctx.cloudformation_files[:3])


def _docker(ctx: ProjectContext) -> DetectedFramework | None:
return _detected(Framework.DOCKER, [*ctx.dockerfiles, *ctx.compose_files])

Expand All @@ -57,6 +112,14 @@ def _github_actions(ctx: ProjectContext) -> DetectedFramework | None:
_node_package(Framework.EXPRESS, "express"),
_node_package(Framework.REACT, "react"),
_node_package(Framework.NEXTJS, "next"),
_node_package(Framework.NESTJS, "@nestjs/core"),
_django,
_python_framework(Framework.FASTAPI, "fastapi", "fastapi"),
_python_framework(Framework.FLASK, "flask", "flask"),
_spring,
_aspnet,
_terraform,
_cloudformation,
_docker,
_github_actions,
)
Expand Down
8 changes: 8 additions & 0 deletions devshield/core/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ class Framework(StrEnum):
NEXTJS = "nextjs"
DOCKER = "docker"
GITHUB_ACTIONS = "github_actions"
DJANGO = "django"
FASTAPI = "fastapi"
FLASK = "flask"
NESTJS = "nestjs"
SPRING = "spring"
ASPNET = "aspnet"
TERRAFORM = "terraform"
CLOUDFORMATION = "cloudformation"


class Confidence(StrEnum):
Expand Down
39 changes: 39 additions & 0 deletions devshield/core/parsers/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
from devshield.core.parsers.appconfig import (
ConfigEntry,
flatten_json,
flatten_yaml,
parse_properties,
)
from devshield.core.parsers.cloudformation import load_template
from devshield.core.parsers.dockerfile import (
Instruction,
effective_user,
Expand All @@ -7,21 +14,53 @@
stages,
)
from devshield.core.parsers.dotenv import EnvEntry, parse_dotenv
from devshield.core.parsers.hcl import HclBlock, parse_hcl
from devshield.core.parsers.php import PhpEnv, PhpExpr, parse_php_config
from devshield.core.parsers.pyast import (
MISSING,
literal,
module_assignments,
parse_module,
secret_assignments,
string_literals,
)
from devshield.core.parsers.pythondeps import (
normalize,
parse_pipfile,
parse_pyproject,
parse_requirements_txt,
)
from devshield.core.parsers.structured import json_line_map, yaml_line_map

__all__ = [
"MISSING",
"ConfigEntry",
"EnvEntry",
"HclBlock",
"Instruction",
"PhpEnv",
"PhpExpr",
"effective_user",
"final_stage",
"flatten_json",
"flatten_yaml",
"from_parts",
"json_line_map",
"literal",
"load_template",
"module_assignments",
"normalize",
"parse_dockerfile",
"parse_dotenv",
"parse_hcl",
"parse_module",
"parse_php_config",
"parse_pipfile",
"parse_properties",
"parse_pyproject",
"parse_requirements_txt",
"secret_assignments",
"stages",
"string_literals",
"yaml_line_map",
]
Loading
Loading