diff --git a/.github/workflows/release-cli.yaml b/.github/workflows/release-cli.yaml index b185e6e5ff..9f755307cf 100644 --- a/.github/workflows/release-cli.yaml +++ b/.github/workflows/release-cli.yaml @@ -191,6 +191,22 @@ jobs: GORELEASER_CURRENT_TAG: v${{ steps.release_version.outputs.RELEASE_VERSION }} HOMEBREW_TAP_OWNER: ${{ github.repository_owner }} HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} + - + name: Bundle go-ssa-server into release archives + if: ${{ steps.release_version.outputs.status == 'succeeded' }} + working-directory: cli + env: + GH_TOKEN: ${{ secrets.SEQRA_GITHUB_TOKEN }} + run: | + bash scripts/bundle-go-server.sh dist ${{ github.repository }} + RELEASE_TAG="v${{ steps.release_version.outputs.RELEASE_VERSION }}" + # Re-upload the modified "opentaint" archives. The "opentaint-full" + # archives are re-uploaded by the JRE step, which runs after this one. + for archive in dist/opentaint_*.tar.gz dist/opentaint_*.zip; do + [ -f "$archive" ] || continue + echo "Re-uploading $archive..." + gh release upload "$RELEASE_TAG" "$archive" --clobber + done - name: Bundle JREs into release archives if: ${{ steps.release_version.outputs.status == 'succeeded' }} diff --git a/.github/workflows/update-cli-versions.yaml b/.github/workflows/update-cli-versions.yaml index 6ce36afee7..5ebb68e2bc 100644 --- a/.github/workflows/update-cli-versions.yaml +++ b/.github/workflows/update-cli-versions.yaml @@ -15,12 +15,17 @@ on: description: 'Rules version tag (e.g. rules/v0.1.0). Leave empty for latest.' required: false type: string + go_server_version: + description: 'Go server version tag (e.g. go-server/2026.03.19.adeb616). Leave empty for latest.' + required: false + type: string workflow_run: workflows: - Publish Analyzer and Models - Publish Autobuilder - Release Rules + - Publish Go Server branches: [main] types: [completed] @@ -66,19 +71,26 @@ jobs: RULES=$(git tag -l "rules/v[0-9]*.[0-9]*.[0-9]*" --sort=-v:refname | head -1) fi + GO_SERVER="${{ inputs.go_server_version }}" + if [ -z "$GO_SERVER" ]; then + GO_SERVER=$(git tag -l "go-server/[0-9]*" --sort=-creatordate | head -1) + fi + if [ -z "$ANALYZER" ] || [ -z "$AUTOBUILDER" ] || [ -z "$RULES" ]; then - echo "::error::Failed to resolve versions (analyzer=${ANALYZER:-}, autobuilder=${AUTOBUILDER:-}, rules=${RULES:-})" + echo "::error::Failed to resolve versions (analyzer=${ANALYZER:-}, autobuilder=${AUTOBUILDER:-}, rules=${RULES:-}, go-server=${GO_SERVER:-})" exit 1 fi echo "analyzer=$ANALYZER" >> "$GITHUB_OUTPUT" echo "autobuilder=$AUTOBUILDER" >> "$GITHUB_OUTPUT" echo "rules=$RULES" >> "$GITHUB_OUTPUT" + echo "go_server=$GO_SERVER" >> "$GITHUB_OUTPUT" echo "Resolved versions:" echo " Analyzer: $ANALYZER" echo " Autobuilder: $AUTOBUILDER" echo " Rules: $RULES" + echo " Go server: $GO_SERVER" - name: Read current versions id: current @@ -87,6 +99,9 @@ jobs: echo "analyzer=$(grep '^analyzer:' cli/internal/globals/versions.yaml | awk '{print $2}' | tr -d '\"')" >> "$GITHUB_OUTPUT" echo "autobuilder=$(grep '^autobuilder:' cli/internal/globals/versions.yaml | awk '{print $2}' | tr -d '\"')" >> "$GITHUB_OUTPUT" echo "rules=$(grep '^rules:' cli/internal/globals/versions.yaml | awk '{print $2}' | tr -d '\"')" >> "$GITHUB_OUTPUT" + if [ -n "${{ steps.versions.outputs.go_server }}" ]; then + echo "go_server=$(grep '^go-server:' cli/internal/globals/versions.yaml | awk '{print $2}' | tr -d '\"')" >> "$GITHUB_OUTPUT" + fi - name: Check for changes id: diff @@ -103,6 +118,9 @@ jobs: if [ "${{ steps.versions.outputs.rules }}" != "${{ steps.current.outputs.rules }}" ]; then CHANGES="${CHANGES}- **rules**: \`${{ steps.current.outputs.rules }}\` → \`${{ steps.versions.outputs.rules }}\`"$'\n' fi + if [ -n "${{ steps.versions.outputs.go_server }}" ] && [ "${{ steps.versions.outputs.go_server }}" != "${{ steps.current.outputs.go_server }}" ]; then + CHANGES="${CHANGES}- **go-server**: \`${{ steps.current.outputs.go_server }}\` → \`${{ steps.versions.outputs.go_server }}\`"$'\n' + fi if [ -z "$CHANGES" ]; then echo "changed=false" >> "$GITHUB_OUTPUT" @@ -124,6 +142,9 @@ jobs: sed -i 's|^analyzer: .*|analyzer: "${{ steps.versions.outputs.analyzer }}"|' cli/internal/globals/versions.yaml sed -i 's|^autobuilder: .*|autobuilder: "${{ steps.versions.outputs.autobuilder }}"|' cli/internal/globals/versions.yaml sed -i 's|^rules: .*|rules: "${{ steps.versions.outputs.rules }}"|' cli/internal/globals/versions.yaml + if [ -n "${{ steps.versions.outputs.go_server }}" ]; then + sed -i 's|^go-server: .*|go-server: "${{ steps.versions.outputs.go_server }}"|' cli/internal/globals/versions.yaml + fi echo "Updated versions.yaml:" cat cli/internal/globals/versions.yaml diff --git a/Makefile b/Makefile index 29904da2e2..f26dd43f48 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ MAKE ?= make GRADLEW := $(CURDIR)/core/gradlew INSTALL ?= install +APT_GET ?= apt-get PREFIX ?= /usr/local BINDIR ?= $(PREFIX)/bin @@ -13,19 +14,26 @@ CLI_BINARY_NAME := opentaint CLI_DEV_BINARY_NAME := opentaint-dev ANALYZER_TASK := :projectAnalyzerJar AUTOBUILDER_TASK := opentaint-jvm-autobuilder:projectAutoBuilderJar +GO_SERVER_TASK := :opentaint-ir:go:buildGoServer ANALYZER_JAR := $(CORE_DIR)/build/libs/opentaint-project-analyzer.jar AUTOBUILDER_JAR := $(CORE_DIR)/opentaint-jvm-autobuilder/build/libs/opentaint-project-auto-builder.jar +GO_SERVER_BINARY := $(CORE_DIR)/opentaint-ir/go/go-ssa-server/go-ssa-server RULES_SRC := rules/ruleset INSTALLED_ANALYZER_JAR := $(LIBDIR)/$(notdir $(ANALYZER_JAR)) INSTALLED_AUTOBUILDER_JAR := $(LIBDIR)/$(notdir $(AUTOBUILDER_JAR)) INSTALLED_RULES_DIR := $(LIBDIR)/rules INSTALLED_CLI_BINARY := $(BINDIR)/$(CLI_BINARY_NAME) INSTALLED_DEV_BINARY := $(BINDIR)/$(CLI_DEV_BINARY_NAME) +INSTALLED_GO_SERVER := $(LIBDIR)/$(notdir $(GO_SERVER_BINARY)) -.PHONY: all core projectAnalyzerJar core/autobuilder cli install clean +.PHONY: all core projectAnalyzerJar core/autobuilder go-server cli install install-deps clean -all: core cli +all: core go-server cli + +install-deps: + $(APT_GET) update + $(APT_GET) install -y protobuf-compiler core: cd $(CORE_DIR) && $(GRADLEW) $(ANALYZER_TASK) $(AUTOBUILDER_TASK) @@ -36,14 +44,18 @@ projectAnalyzerJar: core/autobuilder: cd $(CORE_DIR) && $(GRADLEW) $(AUTOBUILDER_TASK) +go-server: + cd $(CORE_DIR) && $(GRADLEW) $(GO_SERVER_TASK) + cli: $(MAKE) -C $(CLI_DIR) build -install: core +install: core go-server mkdir -p $(BINDIR) $(LIBDIR) $(MAKE) -C $(CLI_DIR) install PREFIX=$(PREFIX) BINDIR=$(abspath $(BINDIR)) $(INSTALL) -m 0644 $(ANALYZER_JAR) $(INSTALLED_ANALYZER_JAR) $(INSTALL) -m 0644 $(AUTOBUILDER_JAR) $(INSTALLED_AUTOBUILDER_JAR) + $(INSTALL) -m 0755 $(GO_SERVER_BINARY) $(INSTALLED_GO_SERVER) rm -rf $(INSTALLED_RULES_DIR) mkdir -p $(INSTALLED_RULES_DIR) cp -R $(RULES_SRC)/. $(INSTALLED_RULES_DIR)/ @@ -54,7 +66,7 @@ install: core 'BIN_DIR=$$(CDPATH= cd -- "$$(dirname -- "$$SELF")" && pwd -P)' \ 'PREFIX_DIR=$$(CDPATH= cd -- "$$BIN_DIR/.." && pwd)' \ 'LIB_DIR="$$PREFIX_DIR/lib"' \ - 'exec "$$BIN_DIR/$(CLI_BINARY_NAME)" --experimental --analyzer-jar "$$LIB_DIR/$(notdir $(ANALYZER_JAR))" --autobuilder-jar "$$LIB_DIR/$(notdir $(AUTOBUILDER_JAR))" "$$@"' \ + 'exec "$$BIN_DIR/$(CLI_BINARY_NAME)" --experimental --analyzer-jar "$$LIB_DIR/$(notdir $(ANALYZER_JAR))" --autobuilder-jar "$$LIB_DIR/$(notdir $(AUTOBUILDER_JAR))" --go-server-binary "$$LIB_DIR/$(notdir $(GO_SERVER_BINARY))" "$$@"' \ > $(INSTALLED_DEV_BINARY) chmod 0755 $(INSTALLED_DEV_BINARY) $(INSTALLED_CLI_BINARY) pull diff --git a/cli/cmd/analyzer_inputs.go b/cli/cmd/analyzer_inputs.go index d9fabfe8b4..a61ee05e99 100644 --- a/cli/cmd/analyzer_inputs.go +++ b/cli/cmd/analyzer_inputs.go @@ -14,7 +14,7 @@ func addDataflowApproximations(b *AnalyzerBuilder, paths []string, analyzerJarPa absPaths := make([]string, 0, len(paths)) for _, approxPath := range paths { - absPaths = append(absPaths, log.AbsPathOrExit(approxPath, "dataflow-approximations")) + absPaths = append(absPaths, log.AbsPathOrExit(approxPath, "java-models")) } classDirs, err := approximation.ResolveAll(absPaths, newApproximationBuilder(analyzerJarPath)) @@ -28,6 +28,6 @@ func addDataflowApproximations(b *AnalyzerBuilder, paths []string, analyzerJarPa func addPassthroughApproximations(b *AnalyzerBuilder, paths []string) { for _, passthrough := range paths { - b.AddPassthroughApproximations(log.AbsPathOrExit(passthrough, "passthrough-approximations")) + b.AddPassthroughApproximations(log.AbsPathOrExit(passthrough, "passthrough-models")) } } diff --git a/cli/cmd/approximation.go b/cli/cmd/approximation.go index 61d5050c88..14e8657285 100644 --- a/cli/cmd/approximation.go +++ b/cli/cmd/approximation.go @@ -18,7 +18,7 @@ methods the analyzer cannot see into, together with the exact dependency version models are written against. Compile one with 'opentaint compile approximations', and apply it with the ---dataflow-approximations flag of 'opentaint scan' or 'opentaint test approximation run'.`, +--java-models flag of 'opentaint scan' or 'opentaint test dataflow-model run'.`, } var approximationInitCmd = &cobra.Command{ diff --git a/cli/cmd/compile.go b/cli/cmd/compile.go index 3d04172990..c63dc01098 100644 --- a/cli/cmd/compile.go +++ b/cli/cmd/compile.go @@ -32,21 +32,33 @@ func currentCompileBuilder(projectPath string) *utils.OpentaintCommandBuilder { // dockerCompileSuggestion builds the "try Docker-based compilation" fallback hint. func dockerCompileSuggestion() output.Suggestion { return output.Suggestion{ - Description: dockerFallbackHintPrefix + "compilation:", + Description: "If the required Java is missing, set JAVA_HOME or compile in a container instead:", Command: utils.BuildCompileCommandWithDocker(currentCompileBuilder(""), ProjectPath, OutputProjectModelPath), } } // compileCmd represents the compile command var compileCmd = &cobra.Command{ - Use: "compile project", - Short: "Compile your Java or Kotlin project", + Use: "compile ", + Short: "Compile a project into a reusable project model", Args: cobra.ExactArgs(1), // require exactly one argument - Long: `This command takes a required path to the project, automatically detects Java/Kotlin build system, modules and dependencies and compiles project model. + Long: `Compile a project into a project model that you can scan many times. OpenTaint finds the build system, collects the modules and dependencies, and builds the project. -Arguments: - project - Path to a project to compile (required) -`, +The project argument is the path to the project root. It is required. Use --output to set the project model directory. This directory must not exist before the command runs. + +Later scans can use the model without a new build. This makes repeated scans fast. + +To scan the model, use "opentaint scan --project-model".`, + Example: ` # Compile the current directory into a project model + opentaint compile . -o ./model + + # Make sure the inputs are correct, without a build + opentaint compile . -o ./model --dry-run + + # Recipe: compile one time, then scan with different settings + opentaint compile ./my-app -o ./model + opentaint scan --project-model ./model --ruleset ./team-rules -o team.sarif + opentaint scan --project-model ./model --severity error -o errors.sarif`, Annotations: map[string]string{"PrintConfig": "true"}, Run: func(cmd *cobra.Command, args []string) { ProjectPath = args[0] @@ -72,25 +84,25 @@ Arguments: sb.Line() } sb.FieldNode("Project", absProjectRoot). - FieldNode("Output project model", absOutputProjectModelPath). + FieldNode("Project model", absOutputProjectModelPath). FieldNode("Autobuilder", utils.ArtifactVersionWithPath(globals.ArtifactByKind("autobuilder"))). Render() if DryRunCompile { out.Blank() failOnInvalidInputs(func() error { return validation.ValidateCompileInputs(absProjectRoot, absOutputProjectModelPath) }) - runDryRun("Compilation") + runDryRun("compilation") return } autobuilderJarPath, err := ensureAutobuilderAvailable() if err != nil { - out.Fatalf("Native compile preparation failed: %s", err) + failf("Native compile preparation failed: %s", err) } compileJavaRunner := newAutobuilderJavaRunner() if _, err := compileJavaRunner.EnsureJava(); err != nil { - out.Fatalf("Failed to resolve Java for compilation: %s", err) + failf("Failed to resolve Java for compilation: %s", err) } if err := out.RunWithSpinner("Compiling project model", func() error { @@ -98,7 +110,7 @@ Arguments: }); err == nil { out.Blank() printCompileSummary(absOutputProjectModelPath) - suggest("To scan project run", utils.BuildScanCommandFromCompile(projectRoot, absOutputProjectModelPath)) + suggest("To scan the compiled project model, run:", utils.BuildScanCommandFromCompile(projectRoot, absOutputProjectModelPath)) } else { out.InteractiveBlank() failWith(1, fmt.Sprintf("Native compile has failed: %s", err), dockerCompileSuggestion()) @@ -109,7 +121,7 @@ Arguments: func init() { rootCmd.AddCommand(compileCmd) - compileCmd.Flags().StringVarP(&OutputProjectModelPath, "output", "o", "", `Path to the result project model`) + compileCmd.Flags().StringVarP(&OutputProjectModelPath, "output", "o", "", `Path to the project model directory to create (required, must not exist)`) _ = compileCmd.MarkFlagRequired("output") compileCmd.Flags().BoolVar(&DryRunCompile, "dry-run", false, "Validate inputs and show what would run without compiling") compileCmd.Flags().StringVar(&CompileLogFile, "log-file", "", "Path to the log file (default: /logs/.log)") diff --git a/cli/cmd/compile_approximations_cmd.go b/cli/cmd/compile_approximations_cmd.go index d9ab54bcaf..b1c20cd4af 100644 --- a/cli/cmd/compile_approximations_cmd.go +++ b/cli/cmd/compile_approximations_cmd.go @@ -53,7 +53,7 @@ needed to compile ahead of time or to see compilation errors on their own.`, Field("Compiled models written to", approximation.ClassesDir(outputPath)). Render() suggest("To apply these models in a scan run", - fmt.Sprintf("opentaint scan --dataflow-approximations %s", args[0])) + fmt.Sprintf("opentaint scan --java-models %s", args[0])) }, } diff --git a/cli/cmd/dry_run.go b/cli/cmd/dry_run.go index e0c2f7276a..dd2edaa238 100644 --- a/cli/cmd/dry_run.go +++ b/cli/cmd/dry_run.go @@ -1,6 +1,9 @@ package cmd -import "fmt" +import ( + "os" + "strings" +) func failOnInvalidInputs(validate func() error) { if err := validate(); err != nil { @@ -9,5 +12,34 @@ func failOnInvalidInputs(validate func() error) { } func runDryRun(skippedAction string) { - out.Print(fmt.Sprintf("Dry run mode. Inputs validated. %s skipped.", skippedAction)) + out.Printf("Dry run complete. Inputs validated, %s skipped.", skippedAction) + suggest("To run for real, run:", rerunWithoutDryRun()) +} + +func rerunWithoutDryRun() string { + args := []string{"opentaint"} + for _, arg := range os.Args[1:] { + if arg == "--dry-run" || strings.HasPrefix(arg, "--dry-run=") { + continue + } + args = append(args, shellQuote(arg)) + } + return strings.Join(args, " ") +} + +func shellQuote(arg string) string { + if arg != "" && !strings.ContainsFunc(arg, shellUnsafe) { + return arg + } + return "'" + strings.ReplaceAll(arg, "'", `'\''`) + "'" +} + +func shellUnsafe(r rune) bool { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return false + case strings.ContainsRune("_@%+=:,./-", r): + return false + } + return true } diff --git a/cli/cmd/exit_codes.go b/cli/cmd/exit_codes.go new file mode 100644 index 0000000000..7f950d70c5 --- /dev/null +++ b/cli/cmd/exit_codes.go @@ -0,0 +1,28 @@ +package cmd + +import ( + "fmt" + + "github.com/seqra/opentaint/internal/analyzer" +) + +func analyzerExitCodeRows() string { + rows := "" + for _, code := range []int{analyzer.ExitException, analyzer.ExitOOM, analyzer.ExitTimeout, analyzer.ExitConfigError} { + rows += fmt.Sprintf("\n %-3d %s", code, analyzer.ExitMessage(code)) + } + return rows +} + +func scanExitCodesHelp(completedLine string) string { + return `Exit codes: + 0 ` + completedLine + ` + 1 General failure (configuration or infrastructure error)` + analyzerExitCodeRows() +} + +func testExitCodesHelp(passedLine string) string { + return `Exit codes: + 0 ` + passedLine + ` + 1 General failure (configuration or infrastructure error) + 2 One or more tests failed (false negatives, false positives, or skipped samples)` + analyzerExitCodeRows() +} diff --git a/cli/cmd/flag_alias.go b/cli/cmd/flag_alias.go new file mode 100644 index 0000000000..d2d03d75dd --- /dev/null +++ b/cli/cmd/flag_alias.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "strings" + + "github.com/spf13/pflag" +) + +type renamedStringArray struct { + target *[]string +} + +func (v renamedStringArray) String() string { + if len(*v.target) == 0 { + return "" + } + return "[" + strings.Join(*v.target, ",") + "]" +} + +func (v renamedStringArray) Set(s string) error { + *v.target = append(*v.target, s) + return nil +} + +func (v renamedStringArray) Type() string { + return "stringArray" +} + +func addRenamedStringArrayFlag(fs *pflag.FlagSet, target *[]string, name, deprecated, usage string) { + fs.Var(renamedStringArray{target}, name, usage) + fs.Var(renamedStringArray{target}, deprecated, usage) + if err := fs.MarkDeprecated(deprecated, "use --"+name); err != nil { + panic(err) + } +} diff --git a/cli/cmd/flag_alias_test.go b/cli/cmd/flag_alias_test.go new file mode 100644 index 0000000000..f0220ad99f --- /dev/null +++ b/cli/cmd/flag_alias_test.go @@ -0,0 +1,56 @@ +package cmd + +import ( + "reflect" + "testing" + + "github.com/spf13/pflag" +) + +func parseRenamed(t *testing.T, args []string) []string { + t.Helper() + var target []string + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + addRenamedStringArrayFlag(fs, &target, "passthrough-models", "passthrough-approximations", "usage") + if err := fs.Parse(args); err != nil { + t.Fatalf("parse %v: %v", args, err) + } + return target +} + +func TestRenamedFlagAccumulatesAcrossBothSpellings(t *testing.T) { + cases := [][]string{ + {"--passthrough-models", "a.yaml", "--passthrough-approximations", "b.yaml"}, + {"--passthrough-approximations", "a.yaml", "--passthrough-models", "b.yaml"}, + } + for _, args := range cases { + got := parseRenamed(t, args) + if len(got) != 2 { + t.Errorf("args %v: got %v, want both values kept", args, got) + } + } +} + +func TestRenamedFlagKeepsRepeatsInOrder(t *testing.T) { + got := parseRenamed(t, []string{ + "--passthrough-models", "a.yaml", + "--passthrough-models", "b.yaml", + "--passthrough-approximations", "c.yaml", + }) + if want := []string{"a.yaml", "b.yaml", "c.yaml"}; !reflect.DeepEqual(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestRenamedFlagAliasIsDeprecatedAndHidden(t *testing.T) { + var target []string + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + addRenamedStringArrayFlag(fs, &target, "passthrough-models", "passthrough-approximations", "usage") + alias := fs.Lookup("passthrough-approximations") + if alias == nil || alias.Deprecated == "" { + t.Fatal("alias must be registered and marked deprecated") + } + if fs.Lookup("passthrough-models").Deprecated != "" { + t.Error("the new spelling must not be deprecated") + } +} diff --git a/cli/cmd/health.go b/cli/cmd/health.go index 0954bbbf47..79d0b943a2 100644 --- a/cli/cmd/health.go +++ b/cli/cmd/health.go @@ -15,6 +15,7 @@ var ( healthAutobuilder bool healthAnalyzer bool healthRules bool + healthGoServer bool healthRuntime bool ) @@ -27,15 +28,29 @@ type healthComponent struct { var healthCmd = &cobra.Command{ Use: "health", - Short: "Show resolved dependency paths", - Long: `Show the on-disk paths OpenTaint uses for the autobuilder, analyzer, -built-in rules, and Java runtime. + Short: "Show dependency paths and report missing components", + Long: `Show the paths of the components on this computer. The components are the autobuilder, the analyzer, the built-in rules, the go-ssa-server, and the Java runtime. The command shows if each component is present. -Use --autobuilder, --analyzer, --rules, or --runtime to select components. When -exactly one component is selected, only its path is printed. The command does -not download artifacts except built-in rules, which are fetched on demand. +To select components, use --autobuilder, --analyzer, --rules, --go-server, or --runtime. With no flag, the command shows all five components. If you select exactly one component, only its path is printed. This output is good for scripts. -The exit code is non-zero when any selected component is missing.`, +Only the built-in rules are downloaded when they are missing. No other component is downloaded. + +If a selected component is missing, the command exits with a code that is not zero. To download the missing components, run "opentaint pull".`, + Example: ` # Show all components and their paths + opentaint health + + # Print only the analyzer JAR path, for a script + opentaint health --analyzer + + # Make sure the Java runtime is present + opentaint health --runtime + + # Make sure the go-ssa-server for Go analysis is present + opentaint health --go-server + + # Recipe: use the built-in rules path in a script + RULES=$(opentaint health --rules) + opentaint scan . --ruleset "$RULES" --ruleset ./extra-rules -o report.sarif`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { return runHealth() @@ -47,6 +62,7 @@ func init() { healthCmd.Flags().BoolVar(&healthAutobuilder, "autobuilder", false, "Print only the autobuilder JAR path") healthCmd.Flags().BoolVar(&healthAnalyzer, "analyzer", false, "Print only the analyzer JAR path") healthCmd.Flags().BoolVar(&healthRules, "rules", false, "Print only the built-in rules path, downloading rules if needed") + healthCmd.Flags().BoolVar(&healthGoServer, "go-server", false, "Print only the go-ssa-server binary path") healthCmd.Flags().BoolVar(&healthRuntime, "runtime", false, "Print only the Java runtime path") } @@ -61,11 +77,14 @@ func runHealth() error { if healthRules { requested = append(requested, "rules") } + if healthGoServer { + requested = append(requested, "goserver") + } if healthRuntime { requested = append(requested, "runtime") } if len(requested) == 0 { - requested = []string{"autobuilder", "analyzer", "rules", "runtime"} + requested = []string{"autobuilder", "analyzer", "rules", "goserver", "runtime"} } components := make([]healthComponent, 0, len(requested)) @@ -105,6 +124,7 @@ func runHealth() error { } sb.Render() if len(missing) > 0 { + out.Suggest("To download the missing components, run:", "opentaint pull") return fmt.Errorf("missing components: %s", strings.Join(missing, ", ")) } return nil @@ -112,8 +132,8 @@ func runHealth() error { func resolveHealthComponent(key string) healthComponent { switch key { - case "autobuilder", "analyzer": - return resolveJarComponent(key) + case "autobuilder", "analyzer", "goserver": + return resolveArtifactComponent(key) case "rules": return resolveRulesComponent() case "runtime": @@ -123,7 +143,9 @@ func resolveHealthComponent(key string) healthComponent { } } -func resolveJarComponent(kind string) healthComponent { +// resolveArtifactComponent resolves a downloadable artifact (a JAR or the +// go-ssa-server binary) through the standard tiers and the dev override. +func resolveArtifactComponent(kind string) healthComponent { def := globals.ArtifactByKind(kind) path, err := utils.ResolveJarPath(def) version := utils.ArtifactVersion(def) diff --git a/cli/cmd/java_runners.go b/cli/cmd/java_runners.go index 560e342e8d..e03d02d6c1 100644 --- a/cli/cmd/java_runners.go +++ b/cli/cmd/java_runners.go @@ -5,11 +5,12 @@ import ( "github.com/seqra/opentaint/internal/utils/java" ) -func newAnalyzerJavaRunner() java.JavaRunner { +func newAnalyzerJavaRunner(env map[string]string) java.JavaRunner { return java.NewJavaRunner(). WithSkipVerify(globals.Config.SkipVerify). WithDebugOutput(out.DebugStream("Analyzer")). WithImageType(java.AdoptiumImageJRE). + WithExtraEnv(env). TrySpecificVersion(globals.DefaultJavaVersion) } diff --git a/cli/cmd/project.go b/cli/cmd/project.go index 60e31d5d20..53b9229cf5 100644 --- a/cli/cmd/project.go +++ b/cli/cmd/project.go @@ -181,7 +181,7 @@ func (c *JavaAutobuilderConfig) printProjectSummary(config *project.Config) erro projectYamlPath := filepath.Join(c.outputDir, "project.yaml") c.logProjectSummary(projectYamlPath, config) - suggest("To scan project run", utils.BuildScanCommandFromCompile(c.outputDir, c.outputDir)) + suggest("To scan the generated model, run:", utils.BuildScanCommandFromCompile(c.outputDir, c.outputDir)) return nil } @@ -218,16 +218,28 @@ var ( var projectCmd = &cobra.Command{ Use: "project", - Short: "Create a project model directory containing a project.yaml configuration from precompiled JARs or classes", - Long: `Create a project model directory containing a project.yaml configuration from precompiled JARs or classes. + Short: "Create a project model from precompiled JARs or classes", + Long: `Create a project model from JARs or classes that are already compiled. No build occurs. OpenTaint examines the classpath, finds the modules and dependencies, and writes a project.yaml file. -This command generates a project model, automatically detecting dependencies and project structure. -Additional packages have to be specified to enhance the generated configuration. +Use this command when you have compiled artifacts but no build. To build a model from sources, use "opentaint compile". -Examples: - # Classpath analysis - opentaint project --output ./project-model --source-root /path/to/source \ - --classpath /path/to/app.jar --package com.example`, +All inputs are flags. Give the source path with --source-root. Give the compiled classes or JARs with --classpath. Give the packages to include with --package. Add more JAR files with --dependency. + +Use --output to set the project model directory. This directory must not exist before the command runs. + +Before the first run, run "opentaint pull" one time. To scan the model, use "opentaint scan --project-model".`, + Example: ` # Create a project model from a compiled JAR + opentaint project --source-root ./src --classpath ./app.jar --package com.example -o ./model + + # Add more dependency JARs to the classpath + opentaint project --source-root ./src --classpath ./app.jar --dependency ./lib.jar --package com.example -o ./model + + # Make sure the inputs are correct, without a model + opentaint project --source-root ./src --classpath ./app.jar --package com.example -o ./model --dry-run + + # Recipe: scan a vendor JAR that you cannot build + opentaint project --source-root ./src --classpath ./vendor-app.jar --package com.vendor -o ./model + opentaint scan --project-model ./model -o report.sarif`, Run: func(cmd *cobra.Command, args []string) { config := NewJavaAutobuilder(). WithOutputDir(OutputDir). @@ -269,12 +281,12 @@ Examples: if DryRunProject { failOnInvalidInputs(config.validate) - runDryRun("Project generation") + runDryRun("project-model generation") return } if err := config.Execute(); err != nil { - out.Fatalf("Failed to generate project configuration: %s", err) + failf("Failed to generate project configuration: %s", err) } }, } @@ -282,15 +294,15 @@ Examples: func init() { rootCmd.AddCommand(projectCmd) - projectCmd.Flags().StringVarP(&OutputDir, "output", "o", "", "Output directory for project.yaml") + projectCmd.Flags().StringVarP(&OutputDir, "output", "o", "", "Directory to write the generated project model (required, must not exist)") _ = projectCmd.MarkFlagRequired("output") - projectCmd.Flags().StringVar(&SourceRoot, "source-root", "", "Source root directory") + projectCmd.Flags().StringVar(&SourceRoot, "source-root", "", "Path to the project source root") _ = projectCmd.MarkFlagRequired("source-root") - projectCmd.Flags().StringArrayVar(&Dependencies, "dependency", []string{}, "Project dependencies (JAR files)") - projectCmd.Flags().StringArrayVar(&Packages, "package", []string{}, "Project packages") + projectCmd.Flags().StringArrayVar(&Dependencies, "dependency", []string{}, "Additional dependency JAR file on the compile classpath (repeatable)") + projectCmd.Flags().StringArrayVar(&Packages, "package", []string{}, "Package to include in the generated model (repeatable)") _ = projectCmd.MarkFlagRequired("package") - projectCmd.Flags().StringArrayVar(&Classpaths, "classpath", []string{}, "Classpath entries (classes or JAR files)") + projectCmd.Flags().StringArrayVar(&Classpaths, "classpath", []string{}, "Classpath entry: a compiled classes directory or a JAR file (repeatable)") _ = projectCmd.MarkFlagRequired("classpath") - projectCmd.Flags().BoolVar(&DryRunProject, "dry-run", false, "Validate inputs and show what would run without generating project model") + projectCmd.Flags().BoolVar(&DryRunProject, "dry-run", false, "Validate inputs and show what would run without generating the project model") projectCmd.Flags().StringVar(&ProjectLogFile, "log-file", "", "Path to the log file (default: /logs/.log)") } diff --git a/cli/cmd/prune.go b/cli/cmd/prune.go index a4da4e6b84..aab398e193 100644 --- a/cli/cmd/prune.go +++ b/cli/cmd/prune.go @@ -57,24 +57,27 @@ func resolveCategories() (utils.PruneCategory, error) { var pruneCmd = &cobra.Command{ Use: "prune", - Short: "Remove stale downloaded artifacts from ~/.opentaint", - Long: `Remove stale downloaded artifacts from the local cache (~/.opentaint). - -Identifies artifacts that are no longer needed: -- Old versions of analyzer JARs, autobuilder JARs, and rules -- Downloaded JDK/JRE versions that don't match the current version -- Cached project models - -Use category flags to prune selectively: - --artifacts Stale analyzer and autobuilder JARs - --rules Stale rules directories - --jdk Old JDK/JRE versions - --models Cached project models - --logs Project log files - --install Install-tier lib and JRE artifacts (requires re-download) - -Without category flags, prunes: artifacts + rules + jdk + models. -With --all: prunes everything including logs and install-tier.`, + Short: "Remove old downloaded artifacts from the cache", + Long: `Remove old downloaded artifacts from the local cache (~/.opentaint). The command removes analyzer and autobuilder JARs that a newer version replaced, old rules, JDK and JRE versions that do not match the configuration, and cached project models. + +To select categories, use --artifacts, --rules, --jdk, --models, --logs, or --install. With no category flag, the command removes artifacts, rules, jdk, and models. The --all flag removes everything, with logs and install-tier artifacts included. Do not give --all together with a category flag. + +To see the deletions without a removal, use --dry-run. To skip the confirmation prompt, use --yes. To download the toolchain again, run "opentaint pull".`, + Example: ` # Remove the default categories, with a confirmation prompt + opentaint prune + + # Remove only the old JDK and JRE versions + opentaint prune --jdk + + # Remove everything, with logs and install-tier artifacts included + opentaint prune --all + + # See what the command would delete, without a deletion + opentaint prune --dry-run + + # Recipe: get disk space back, keep the current toolchain + opentaint prune --dry-run + opentaint prune --yes`, Run: func(cmd *cobra.Command, args []string) { categories, err := resolveCategories() if err != nil { @@ -84,23 +87,23 @@ With --all: prunes everything including logs and install-tier.`, // Acquire global prune lock pruneLockPath, err := utils.PruneLockPath() if err != nil { - out.Fatalf("Failed to resolve prune lock path: %s", err) + failf("Failed to resolve prune lock path: %s", err) } pruneLock, err := utils.TryLockExclusive(pruneLockPath, utils.LockMeta{ PID: os.Getpid(), Command: "prune", }) if err == utils.ErrLocked { - out.Fatal("Another prune is already running") + failWith(1, "Another prune is already running") } if err != nil { - out.Fatalf("Failed to acquire prune lock: %s", err) + failf("Failed to acquire prune lock: %s", err) } defer pruneLock.Unlock() result, err := utils.ScanForStaleArtifacts(categories) if err != nil { - out.Fatalf("Failed to scan for stale artifacts: %s", err) + failf("Failed to scan for stale artifacts: %s", err) } // Display skipped projects @@ -130,22 +133,27 @@ With --all: prunes everything including logs and install-tier.`, Render() if pruneDryRun { - out.Print("Dry run mode. No files were deleted.") + out.Print("Dry run complete. No files were deleted.") + suggest("To delete these artifacts, run:", withFlag(rerunWithoutDryRun(), "--yes")) return } if !pruneYes { if !out.Confirm("Delete these artifacts?", false) { out.Print("Prune cancelled.") + suggest("To prune without confirming, run:", withFlag(rerunWithoutDryRun(), "--yes")) return } } if err := utils.DeleteArtifacts(result.Stale); err != nil { - out.Fatalf("Failed to delete artifacts: %s", err) + failf("Failed to delete artifacts: %s", err) } out.Successf("Pruned %d items, freed %s", result.TotalCount, output.FormatSize(result.TotalSize)) + if pruneInstall || pruneAll { + suggest("To restore the removed components, run:", "opentaint pull") + } }, } diff --git a/cli/cmd/pull.go b/cli/cmd/pull.go index ff95db860f..d4608c723d 100644 --- a/cli/cmd/pull.go +++ b/cli/cmd/pull.go @@ -17,20 +17,27 @@ import ( var pullCmd = &cobra.Command{ Use: "pull", - Short: "Download autobuilder, analyzer binaries, rules and Java runtime", - Long: `Download all necessary binaries and assets: -- OpenTaint autobuilder JAR -- OpenTaint analyzer JAR -- OpenTaint rules archive -- Java runtime (Temurin JRE) - -This prepares the environment with all required dependencies for offline analysis. -When bundled artifacts are present (from a release archive), they will be used directly.`, + Short: "Download the analysis toolchain and Java runtime", + Long: `Download the toolchain into the local cache. The toolchain contains the analyzer, the autobuilder, the built-in rules, the go-ssa-server, and a Java runtime. After the download, OpenTaint can build and scan projects without network access. + +If a release archive supplied bundled artifacts, OpenTaint uses them. They are not downloaded again. + +Run "opentaint pull" one time before your first scan. To remove old downloads, use "opentaint prune".`, + Example: ` # Download the toolchain before the first scan + opentaint pull + + # Download a different Java runtime version + opentaint pull --java-version 17 + + # Recipe: prepare a machine that will have no network access + opentaint pull + opentaint health`, Run: func(cmd *cobra.Command, args []string) { out.Section("OpenTaint Pull"). Field("Autobuilder", globals.Config.Autobuilder.Version). Field("Analyzer", globals.Config.Analyzer.Version). Field("Rules", globals.Config.Rules.Version). + Field("GoServer", globals.Config.GoServer.Version). Field("Java", globals.Config.Java.Version). Render() @@ -40,7 +47,7 @@ When bundled artifacts are present (from a release archive), they will be used d installCurrent := utils.IsInstallCurrent() if !installCurrent { if err := utils.CleanInstallDir(); err != nil { - out.Fatalf("Failed to clean install directory: %s", err) + failf("Failed to clean install directory: %s", err) } } @@ -50,26 +57,29 @@ When bundled artifacts are present (from a release archive), they will be used d for _, spec := range artifacts { node, err := downloadArtifact(spec, installNextToBinary, installCurrent) if err != nil { - out.Fatalf("Failed to download %s: %s", spec.Kind(), err) + failf("Failed to download %s: %s", spec.Kind(), err) } summaryNodes = append(summaryNodes, node) } javaNode, err := downloadJava(installNextToBinary, installCurrent) if err != nil { - out.Fatalf("Failed to download Java: %s", err) + failf("Failed to download Java: %s", err) } summaryNodes = append(summaryNodes, javaNode) // Write version marker after all downloads succeed if err := utils.WriteInstallVersionMarker(); err != nil { - out.Fatalf("Failed to write install version marker: %s", err) + failf("Failed to write install version marker: %s", err) } out.Blank() out.Section("Pull Summary"). Child(summaryNodes...). Render() + + out.Successf("Pull completed.") + suggest("To scan your project, run:", "opentaint scan .") }, } @@ -114,6 +124,12 @@ func downloadArtifact(spec globals.ArtifactDef, installNextToBinary, installCurr if err := download(t.Path); err != nil { return node, err } + // go-ssa-server is a single native binary and must be executable on unix. + if spec.Kind() == "goserver" && runtime.GOOS != "windows" { + if err := os.Chmod(t.Path, 0o755); err != nil { + return node, fmt.Errorf("failed to mark %s executable: %w", spec.Kind(), err) + } + } node.Child(fmt.Sprintf("Downloaded to %s", t.Path)) return node, nil } diff --git a/cli/cmd/rerun_test.go b/cli/cmd/rerun_test.go new file mode 100644 index 0000000000..a57cd8aee5 --- /dev/null +++ b/cli/cmd/rerun_test.go @@ -0,0 +1,158 @@ +package cmd + +import ( + "os" + "testing" + "time" + + "github.com/seqra/opentaint/internal/analyzer" +) + +func withOSArgs(t *testing.T, args []string) { + t.Helper() + saved := os.Args + os.Args = args + t.Cleanup(func() { os.Args = saved }) +} + +func TestRerunWithoutDryRunStripsFlag(t *testing.T) { + withOSArgs(t, []string{"/usr/bin/opentaint", "scan", "./proj", "--dry-run", "--color", "never"}) + got := rerunWithoutDryRun() + want := "opentaint scan ./proj --color never" + if got != want { + t.Fatalf("rerunWithoutDryRun() = %q, want %q", got, want) + } +} + +func TestRerunWithoutDryRunStripsEqualsForm(t *testing.T) { + withOSArgs(t, []string{"opentaint", "compile", ".", "--dry-run=true", "-o", "./model"}) + got := rerunWithoutDryRun() + want := "opentaint compile . -o ./model" + if got != want { + t.Fatalf("rerunWithoutDryRun() = %q, want %q", got, want) + } +} + +func TestRerunWithoutDryRunQuotesSpaces(t *testing.T) { + withOSArgs(t, []string{"opentaint", "scan", "my project", "--dry-run"}) + got := rerunWithoutDryRun() + want := "opentaint scan 'my project'" + if got != want { + t.Fatalf("rerunWithoutDryRun() = %q, want %q", got, want) + } +} + +func TestWithFlag(t *testing.T) { + if got := withFlag("opentaint prune", "--yes"); got != "opentaint prune --yes" { + t.Fatalf("withFlag append = %q", got) + } + if got := withFlag("opentaint prune --yes", "--yes"); got != "opentaint prune --yes" { + t.Fatalf("withFlag no-op = %q", got) + } + if got := withFlag("opentaint scan path--debug", "--debug"); got != "opentaint scan path--debug --debug" { + t.Fatalf("withFlag substring match = %q", got) + } +} + +func TestRerunReplacingFlagValueForm(t *testing.T) { + withOSArgs(t, []string{"opentaint", "scan", ".", "--max-memory", "8G"}) + got := rerunReplacingFlag("16G", "--max-memory") + want := "opentaint scan . --max-memory 16G" + if got != want { + t.Fatalf("rerunReplacingFlag() = %q, want %q", got, want) + } +} + +func TestRerunReplacingFlagAliasAndEqualsForm(t *testing.T) { + withOSArgs(t, []string{"opentaint", "scan", ".", "-t", "15m", "--timeout=10m"}) + got := rerunReplacingFlag("30m0s", "--timeout", "-t") + want := "opentaint scan . --timeout 30m0s" + if got != want { + t.Fatalf("rerunReplacingFlag() = %q, want %q", got, want) + } +} + +func TestRerunReplacingFlagAppendsWhenAbsent(t *testing.T) { + withOSArgs(t, []string{"opentaint", "test", "rule", "run", "./model"}) + got := rerunReplacingFlag("16G", "--max-memory") + want := "opentaint test rule run ./model --max-memory 16G" + if got != want { + t.Fatalf("rerunReplacingFlag() = %q, want %q", got, want) + } +} + +func TestDoubleMemory(t *testing.T) { + cases := map[string]string{ + "8G": "16G", + "12G": "16G", + "15G": "16G", + "1024m": "2048m", + "0G": "16G", + "weird": "16G", + "": "16G", + } + for in, want := range cases { + if got := doubleMemory(in); got != want { + t.Fatalf("doubleMemory(%q) = %q, want %q", in, got, want) + } + } +} + +func TestRetrySuggestion(t *testing.T) { + withOSArgs(t, []string{"opentaint", "scan", ".", "--max-memory", "8G"}) + + oom, ok := retrySuggestion(analyzer.ExitOOM, 900e9, "8G") + if !ok || oom.Description != "To retry with more memory, run:" || oom.Command != "opentaint scan . --max-memory 16G" { + t.Fatalf("OOM retry = %+v ok=%t", oom, ok) + } + + timeoutRetry, ok := retrySuggestion(analyzer.ExitTimeout, 10*time.Minute, "8G") + if !ok || timeoutRetry.Description != "To retry with a longer timeout, run:" { + t.Fatalf("timeout retry = %+v ok=%t", timeoutRetry, ok) + } + want := "opentaint scan . --max-memory 8G --timeout 15m0s" + if timeoutRetry.Command != want { + t.Fatalf("timeout retry command = %q, want %q", timeoutRetry.Command, want) + } + if _, ok := retrySuggestion(analyzer.ExitTimeout, 15*time.Minute, "8G"); ok { + t.Fatal("timeout at cap must not produce a retry suggestion") + } + if _, ok := retrySuggestion(analyzer.ExitOOM, 15*time.Minute, "16G"); ok { + t.Fatal("memory at cap must not produce a retry suggestion") + } + if retry, ok := retrySuggestion(analyzer.ExitTimeout, 10*time.Minute, "8G"); !ok || retry.Command != "opentaint scan . --max-memory 8G --timeout 15m0s" { + t.Fatalf("timeout cap retry = %+v ok=%t", retry, ok) + } + + if _, ok := retrySuggestion(analyzer.ExitException, 900e9, "8G"); ok { + t.Fatal("exception exit code must not produce a retry suggestion") + } +} + +func TestShellQuoteLeavesInertArgumentsAlone(t *testing.T) { + for _, arg := range []string{"opentaint", "report.sarif", "--max-memory=8G", "path/to/file.yaml", "a-b_c.d,e:f@g%h+i"} { + if got := shellQuote(arg); got != arg { + t.Errorf("shellQuote(%q) = %q, want unchanged", arg, got) + } + } +} + +func TestShellQuoteQuotesShellMetacharacters(t *testing.T) { + cases := map[string]string{ + "demo-rule-*": "'demo-rule-*'", + "java/security/**": "'java/security/**'", + "$HOME": "'$HOME'", + "a;b": "'a;b'", + "a|b": "'a|b'", + "a b": "'a b'", + "it's": `'it'\''s'`, + "": "''", + "a>b": "'a>b'", + "`cmd`": "'`cmd`'", + } + for in, want := range cases { + if got := shellQuote(in); got != want { + t.Errorf("shellQuote(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 214028ec8b..af3574eb4b 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -33,9 +33,14 @@ var updateHintCh = make(chan string, 1) // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ - Use: "opentaint", - Short: "OpenTaint Analyzer", - Long: `OpenTaint is a CLI tool that analyzes Java and Kotlin projects to find vulnerabilities`, + Use: "opentaint", + Short: "Find vulnerabilities in source code with taint analysis", + Long: `OpenTaint finds vulnerabilities in your code. It follows tainted data from untrusted sources to dangerous sinks. Java, Kotlin, and Go projects are supported. + +Quick start: + 1. Run "opentaint pull" one time. This downloads the toolchain. + 2. Run "opentaint scan ." to scan a project. + 3. Run "opentaint summary --show-findings" to read the findings.`, SilenceErrors: true, SilenceUsage: true, @@ -128,6 +133,14 @@ func init() { _ = rootCmd.PersistentFlags().MarkHidden("rules-version") _ = viper.BindPFlag("rules.version", rootCmd.PersistentFlags().Lookup("rules-version")) + rootCmd.PersistentFlags().StringVar(&globals.Config.GoServer.Version, "go-server-version", globals.GoServerBindVersion, "The go-ssa-server version to download") + _ = rootCmd.PersistentFlags().MarkHidden("go-server-version") + _ = viper.BindPFlag("go-server.version", rootCmd.PersistentFlags().Lookup("go-server-version")) + + rootCmd.PersistentFlags().StringVar(&globals.Config.GoServer.Binary, "go-server-binary", "", "Path to a go-ssa-server binary (developer override, no download)") + _ = rootCmd.PersistentFlags().MarkHidden("go-server-binary") + _ = viper.BindPFlag("go-server.binary", rootCmd.PersistentFlags().Lookup("go-server-binary")) + rootCmd.PersistentFlags().IntVar(&globals.Config.Java.Version, "java-version", globals.DefaultJavaVersion, "Java version to use for running analyzer") _ = viper.BindPFlag("java.version", rootCmd.PersistentFlags().Lookup("java-version")) diff --git a/cli/cmd/scan.go b/cli/cmd/scan.go index 30f51ed2fa..fdb5e2cc05 100644 --- a/cli/cmd/scan.go +++ b/cli/cmd/scan.go @@ -3,7 +3,11 @@ package cmd import ( "fmt" "os" + "os/exec" "path/filepath" + "runtime" + "slices" + "strings" "time" "github.com/seqra/opentaint/internal/analyzer" @@ -73,15 +77,40 @@ func (p scanPlan) title() string { // scanCmd represents the scan command var scanCmd = &cobra.Command{ Use: "scan [source-path]", - Short: "Scan your Java or Kotlin project", + Short: "Scan a project for vulnerabilities", Args: cobra.MaximumNArgs(1), - Long: `This command automatically detects Java/Kotlin build systems, builds the project, and analyzes it + Long: `Scan a project and find vulnerabilities. OpenTaint finds the build system, builds the project, and does a taint analysis. -Arguments: - source-path - Path to the project sources (default: current directory) +The source-path argument is the project root. It is optional. The default is the current directory. To scan a project model that is already compiled, use --project-model. Do not give source-path and --project-model together. -Use --project-model to scan a pre-compiled project model instead of compiling from sources. -`, +OpenTaint writes the findings to a SARIF report. Use --output to set the report path. If --output is not set, the report goes into the project model directory. A summary is shown when the scan completes. + +Before your first scan, run "opentaint pull" one time. For a Go project, the "go" command must also be on your PATH. To read a report again later, use "opentaint summary". + +` + scanExitCodesHelp("Scan completed"), + Example: ` # Scan the current directory with the built-in rules + opentaint scan . + + # Scan a project and write the report to a known path + opentaint scan ./my-app -o report.sarif + + # Scan a project model that is already compiled + opentaint scan --project-model ./model -o report.sarif + + # Use your own rules and show only errors + opentaint scan . --ruleset ./rules --severity error -o report.sarif + + # Give a large project more time and memory + opentaint scan . --timeout 30m --max-memory 16G -o report.sarif + + # Recipe: first scan on a new machine + opentaint pull + opentaint scan . -o report.sarif + opentaint summary report.sarif --show-findings + + # Recipe: build one time, then scan many times + opentaint compile ./my-app -o ./model + opentaint scan --project-model ./model -o report.sarif`, Annotations: map[string]string{"PrintConfig": "true"}, Run: func(cmd *cobra.Command, args []string) { if scanFlags.DebugRunAnalysisOnSelectedEntryPoints != "" { @@ -94,7 +123,7 @@ Use --project-model to scan a pre-compiled project model instead of compiling fr func prepareScanConfig(cfg ScanConfig, args []string) ScanConfig { if len(args) > 0 && cfg.ProjectModelPath != "" { out.Error("Cannot use both a source path argument and --project-model flag") - suggest("Use either a source path or --project-model", + suggest("Use either a source path or --project-model:", utils.NewScanCommand("").Build()+"\n "+utils.NewScanCommand("").WithProjectModel("").Build()) os.Exit(1) } @@ -123,28 +152,28 @@ func addEntryPointsFlag(cmd *cobra.Command) { } func addRuleIDFlag(cmd *cobra.Command) { - cmd.Flags().StringArrayVar(&scanFlags.RuleID, "rule-id", nil, "Filter active rules by ID (repeatable)") + cmd.Flags().StringArrayVar(&scanFlags.RuleID, "rule-id", nil, "Run only rules with this ID (repeatable)") } func addScanFlags(cmd *cobra.Command) { - cmd.Flags().DurationVarP(&globals.Config.Scan.Timeout, "timeout", "t", 900*time.Second, "Timeout for analysis") + cmd.Flags().DurationVarP(&globals.Config.Scan.Timeout, "timeout", "t", 900*time.Second, "Maximum wall-clock time for analysis (e.g. 30m, 1h)") - cmd.Flags().StringArrayVar(&scanFlags.Ruleset, "ruleset", []string{"builtin"}, "YAML rules file, directory of YAML rules files ending in .yml or .yaml, or `builtin` to scan with built-in rules") + cmd.Flags().StringArrayVar(&scanFlags.Ruleset, "ruleset", []string{"builtin"}, "Rules to run: a YAML file, a directory of .yml or .yaml files, or builtin for the built-in rules (repeatable)") - cmd.Flags().BoolVar(&scanFlags.SemgrepCompatibilitySarif, "semgrep-compatibility-sarif", true, "Use Semgrep compatible ruleId") - cmd.Flags().StringVarP(&scanFlags.SarifReportPath, "output", "o", "", "Path to the SARIF-report output file") + cmd.Flags().BoolVar(&scanFlags.SemgrepCompatibilitySarif, "semgrep-compatibility-sarif", true, "Use Semgrep-compatible rule IDs in the SARIF report") + cmd.Flags().StringVarP(&scanFlags.SarifReportPath, "output", "o", "", "Path to write the SARIF report") - cmd.Flags().StringArrayVar(&scanFlags.Severity, "severity", []string{"warning", "error"}, "Report findings only from rules matching the supplied severity level. By default only warning and error rules are run (note, warning, error)") - cmd.Flags().StringVar(&globals.Config.Scan.MaxMemory, "max-memory", "8G", "Maximum memory for the analyzer (e.g., 1024m, 8G, 81920k, 83886080)") + cmd.Flags().StringArrayVar(&scanFlags.Severity, "severity", []string{"warning", "error"}, "Run only rules at these severity levels: note, warning, error (repeatable)") + cmd.Flags().StringVar(&globals.Config.Scan.MaxMemory, "max-memory", "8G", "Maximum analyzer heap size (e.g. 8G, 1024m)") cmd.Flags().Int64Var(&globals.Config.Scan.CodeFlowLimit, "code-flow-limit", 0, "Maximum number of code flows to include in the report (0 = unlimited)") cmd.Flags().BoolVar(&scanFlags.DryRun, "dry-run", false, "Validate inputs and show what would run without compiling or scanning") cmd.Flags().BoolVar(&scanFlags.Recompile, "recompile", false, "Force recompilation even if a cached project model exists") cmd.Flags().StringVar(&scanFlags.ProjectModelPath, "project-model", "", "Path to a pre-compiled project model (skips compilation)") cmd.Flags().StringVar(&scanFlags.LogFile, "log-file", "", "Path to the log file (default: /logs/.log)") - cmd.Flags().StringArrayVar(&scanFlags.PassthroughApproximations, "passthrough-approximations", nil, "Pass-through approximation YAML file or directory (repeatable)") + addRenamedStringArrayFlag(cmd.Flags(), &scanFlags.PassthroughApproximations, "passthrough-models", "passthrough-approximations", "Pass-through models: a YAML file or a directory of them (repeatable)") - cmd.Flags().StringArrayVar(&scanFlags.DataflowApproximations, "dataflow-approximations", nil, "Dataflow approximation project, build output, or class directory (repeatable)") + addRenamedStringArrayFlag(cmd.Flags(), &scanFlags.DataflowApproximations, "java-models", "dataflow-approximations", "Java dataflow models: a model project, build output, or class directory (repeatable)") cmd.Flags().BoolVar(&scanFlags.TrackExternalMethods, "track-external-methods", false, "Write external-method coverage files next to the SARIF report") @@ -176,7 +205,7 @@ func isDefaultSeverity(sev []string) bool { // dockerScanSuggestion builds the "try Docker-based scan" fallback hint. func dockerScanSuggestion(cfg ScanConfig, projectRoot, sarifReportPath string) output.Suggestion { return output.Suggestion{ - Description: dockerFallbackHintPrefix + "scan:", + Description: "If the required Java is missing, set JAVA_HOME or scan in a container instead:", Command: utils.BuildScanCommandWithDocker(currentScanBuilder(cfg, ""), projectRoot, sarifReportPath, cfg.Ruleset), } } @@ -194,7 +223,7 @@ func runScan(cmd *cobra.Command, cfg ScanConfig) { if err := validation.ValidateSourceProject(absUserProjectRoot); err != nil { if validation.IsProjectModel(absUserProjectRoot) { out.ErrorErr(err) - suggest("Use --project-model to scan a pre-compiled model", currentScanBuilder(cfg, "").WithProjectModel(absUserProjectRoot).Build()) + suggest("Use --project-model to scan a pre-compiled model:", currentScanBuilder(cfg, "").WithProjectModel(absUserProjectRoot).Build()) os.Exit(1) } out.FatalErr(err) @@ -287,7 +316,7 @@ func runScan(cmd *cobra.Command, cfg ScanConfig) { } if cfg.DryRun { - runDryRun("Compilation and analysis") + runDryRun("the build and scan") return } @@ -298,27 +327,49 @@ func runScan(cmd *cobra.Command, cfg ScanConfig) { break } } + + // Go projects: the analyzer starts a go-ssa-server process, and that server + // runs the `go` toolchain. The decision about the Go wiring occurs here, + // after the --dry-run return and before the compile and the analyzer + // download. Thus a dry run causes no side effects, and a Go-only scan with + // no `go` toolchain fails fast, not after a long compile. + // + // For a source scan, sourceRoot is the live source tree. For a + // --project-model scan, it is the sourceRoot from project.yaml. Thus the + // detection covers both paths. For a precompiled model, the recorded source + // root must still exist on disk with its go.mod. If it does not, the + // detection finds no languages. + // + // The `go` preflight does not depend on the --go-server-binary override, + // because go-ssa-server always runs `go`. It applies when Go is detected. + // If Go is the only language, the command stops with an error. If other + // languages are present, the command shows a warning and does not do the + // Go wiring. The resolution of the binary occurs later (see + // resolveGoServerEnv), after the compile. Thus a missing `go` is reported + // before the long compile runs. + needGoServer := goServerRequired(sourceRoot) + if hasBuiltin { if _, err := utils.EnsureRulesPath(out); err != nil { - out.Fatalf("Failed to prepare built-in rules: %s", err) + failf("Failed to prepare built-in rules: %s", err) } } if plan.needsCompilation { autobuilderJarPath, err := ensureAutobuilderAvailable() if err != nil { - out.Fatalf("Native compile preparation failed: %s", err) + failf("Native compile preparation failed: %s", err) } compileJavaRunner := newAutobuilderJavaRunner() if _, err := compileJavaRunner.EnsureJava(); err != nil { - out.Fatalf("Failed to resolve Java for compilation: %s", err) + failf("Failed to resolve Java for compilation: %s", err) } // Wipe any residue from a prior crashed compile before writing new output. if plan.projectCachePath != "" { if err := os.RemoveAll(plan.absProjectModel); err != nil { - out.Fatalf("Failed to prepare cache directory: %s", err) + failf("Failed to prepare cache directory: %s", err) } } @@ -337,7 +388,7 @@ func runScan(cmd *cobra.Command, cfg ScanConfig) { if plan.projectCachePath != "" { if err := utils.MarkCompileComplete(plan.projectCachePath); err != nil { _ = os.RemoveAll(plan.absProjectModel) - out.Fatalf("Failed to mark model complete: %s", err) + failf("Failed to mark model complete: %s", err) } if err := plan.cacheLock.Downgrade(); err != nil { output.LogInfof("Cache lock downgrade failed, continuing under exclusive: %v", err) @@ -348,7 +399,7 @@ func runScan(cmd *cobra.Command, cfg ScanConfig) { } if err := utils.EnsureParentDir(absSarifReportPath); err != nil { - out.Fatalf("Failed to create output directory: %s", err) + failf("Failed to create output directory: %s", err) } // Update builder with native paths for native execution @@ -404,15 +455,28 @@ func runScan(cmd *cobra.Command, cfg ScanConfig) { analyzerJarPath, err := ensureAnalyzerAvailable() if err != nil { - out.Fatalf("Native scan preparation failed: %s", err) + failf("Native scan preparation failed: %s", err) } nativeBuilder.SetJarPath(analyzerJarPath) + // Process --java-models: build model projects if needed addDataflowApproximations(nativeBuilder, cfg.DataflowApproximations, analyzerJarPath) - analyzerJavaRunner := newAnalyzerJavaRunner() + // Go projects: get the go-ssa-server binary (a download, or the + // --go-server-binary override) and give its absolute path to the analyzer + // in GOIR_SERVER_BINARY. The needGoServer decision occurred early, after + // the --dry-run return, where the `go` preflight ran. This block only + // resolves the binary and sets the env. A scan with no Go, and a polyglot + // scan where `go` was missing, keep goServerEnv nil. Then + // WithExtraEnv(nil) does nothing and those runs do not change. + var goServerEnv map[string]string + if needGoServer { + goServerEnv = resolveGoServerEnv() + } + + analyzerJavaRunner := newAnalyzerJavaRunner(goServerEnv) if _, err := analyzerJavaRunner.EnsureJava(); err != nil { - out.Fatalf("Failed to resolve Java for analyzer: %s", err) + failf("Failed to resolve Java for analyzer: %s", err) } var analyzerFail *analyzer.Error @@ -467,15 +531,41 @@ func runScan(cmd *cobra.Command, cfg ScanConfig) { var suggestions []output.Suggestion if analyzerFail != nil { suggestions = appendLogSuggestion(suggestions) + if retry, ok := retrySuggestion(analyzerFail.ExitCode, globals.Config.Scan.Timeout, globals.Config.Scan.MaxMemory); ok { + suggestions = append(suggestions, retry) + } } if report != nil { // Scan does not expose summary's filter/group flags, so pass zero values: // no filtering, default group dimension, first-flow code-flow selection. printSarifSummary(report, absSarifReportPath, sarif.Filters{}, sarif.ListingOptions{MaxNestingLevel: -1}) - suggestions = append(suggestions, output.Suggestion{ - Description: "To view findings run", - Command: utils.NewSummaryCommand(absSarifReportPath).WithShowFindings().Build(), - }) + switch { + case cfg.DebugFactReachabilitySarif: + if analyzerFail == nil { + out.Successf("Reachability analysis completed.") + } + reachabilityReportPath := filepath.Join(filepath.Dir(absSarifReportPath), "debug-ifds-fact-reachability.sarif") + suggestions = append(suggestions, output.Suggestion{ + Description: "To view the reachability report, run:", + Command: utils.NewSummaryCommand(reachabilityReportPath).WithShowFindings().Build(), + }) + case sarif.GenerateSummary(report).TotalFindings > 0: + if analyzerFail == nil { + out.Successf("Scan completed.") + } + suggestions = append(suggestions, output.Suggestion{ + Description: "To view the findings, run:", + Command: utils.NewSummaryCommand(absSarifReportPath).WithShowFindings().Build(), + }) + case analyzerFail == nil: + out.Successf("Scan completed. No %s findings were reported.", strings.Join(cfg.Severity, " or ")) + if isDefaultSeverity(cfg.Severity) { + suggestions = append(suggestions, output.Suggestion{ + Description: "To also check note-level rules, run:", + Command: noteSeverityScanCommand(cfg), + }) + } + } } out.Suggestions(suggestions...) @@ -484,6 +574,18 @@ func runScan(cmd *cobra.Command, cfg ScanConfig) { } } +func noteSeverityScanCommand(cfg ScanConfig) string { + sourcePath := cfg.UserProjectPath + if cfg.ProjectModelPath != "" { + sourcePath = "" + } + b := currentScanBuilder(cfg, sourcePath).WithSeverity([]string{"note"}) + if cfg.ProjectModelPath != "" { + b.WithProjectModel(cfg.ProjectModelPath) + } + return b.Build() +} + func resolveScanPlan(cfg ScanConfig, absUserProjectRoot string) scanPlan { if cfg.ProjectModelPath != "" { return scanPlan{ @@ -543,7 +645,7 @@ func resolveScanPlan(cfg ScanConfig, absUserProjectRoot string) scanPlan { } else { out.Error("Another scan is currently analyzing this project") } - suggest("To scan an existing model instead", utils.NewScanCommand("").WithProjectModel("").Build()) + suggest("To scan an existing model instead, run:", utils.NewScanCommand("").WithProjectModel("").Build()) os.Exit(1) } if lockErr != nil { @@ -599,6 +701,134 @@ func setupSemgrepRuleLoadTrace(traceDir string) string { return absSemgrepRuleLoadTracePath } +// EnsureGoServerAvailable returns the absolute path of the go-ssa-server +// binary. If the binary is absent, the function downloads the asset for this +// platform and, on unix, makes the file executable. +// +// The release tag is the configured go-server version ("go-server/", +// default globals.GoServerBindVersion). The asset is the platform-specific +// globals.GoServerAssetName() ("go-ssa-server__", with ".exe" +// on windows). DownloadGithubReleaseAsset examines the checksums. +// +// This is the public entry point for Go projects and for "opentaint pull". +// The function does not set environment variables. +// +// The --go-server-binary override (globals.Config.GoServer.Binary) has +// priority. With it, the function makes sure that the given path exists, +// converts the path to an absolute path, and returns it. No download, no +// version tag, and no chmod occur. The user-supplied file does not change. +// This is the same behavior as the --analyzer-jar override in +// ensureAnalyzerAvailable(). +func EnsureGoServerAvailable() (string, error) { + if globals.Config.GoServer.Binary != "" { + info, err := os.Stat(globals.Config.GoServer.Binary) + if err != nil { + return "", fmt.Errorf("go-ssa-server binary not found at %s: %w", globals.Config.GoServer.Binary, err) + } + if info.IsDir() { + return "", fmt.Errorf("go-ssa-server path %s is a directory, expected an executable file", globals.Config.GoServer.Binary) + } + if runtime.GOOS != "windows" && info.Mode()&0o111 == 0 { + return "", fmt.Errorf("go-ssa-server binary %s is not executable (run: chmod +x %s)", globals.Config.GoServer.Binary, globals.Config.GoServer.Binary) + } + absPath, err := filepath.Abs(globals.Config.GoServer.Binary) + if err != nil { + return "", fmt.Errorf("failed to resolve absolute path to the go-ssa-server: %w", err) + } + return absPath, nil + } + + version := globals.Config.GoServer.Version + if version == "" { + version = globals.GoServerBindVersion + } + + assetName := globals.GoServerAssetName() + + goServerPath, err := utils.GetGoServerPath(version) + if err != nil { + return "", fmt.Errorf("failed to construct path to the go-ssa-server: %w", err) + } + + if err := ensureArtifactAvailable("go-ssa-server", version, goServerPath, func() error { + return utils.DownloadGithubReleaseAsset(globals.Config.Owner, globals.Config.Repo, version, assetName, goServerPath, globals.Config.Github.Token, globals.Config.SkipVerify, out) + }); err != nil { + return "", err + } + + // The go-ssa-server is a single native binary and must be executable to run. + if runtime.GOOS != "windows" { + if err := os.Chmod(goServerPath, 0o755); err != nil { + return "", fmt.Errorf("failed to mark go-ssa-server executable at %s: %w", goServerPath, err) + } + } + + absPath, err := filepath.Abs(goServerPath) + if err != nil { + return "", fmt.Errorf("failed to resolve absolute path to the go-ssa-server: %w", err) + } + + return absPath, nil +} + +// goServerRequired reports whether the analyzer needs the go-ssa-server for +// the project at sourceRoot. As a side effect, it does the `go` toolchain +// preflight. If Go is present but `go` is missing, a Go-only project causes +// an error, and a polyglot project causes a warning and a false result. The +// preflight does not depend on the --go-server-binary override, because +// go-ssa-server always runs `go`. If Go is absent, the result is false. +// +// This function is the "decide" half of the Go wiring, and resolveGoServerEnv +// is the "resolve" half. The scan command keeps the two apart. Thus the +// preflight can run before the autobuilder compile (fail fast) and the +// download after it. A caller with no compile step (for example, the rule +// tests) calls the two functions one after the other. +func goServerRequired(sourceRoot string) bool { + langs := validation.DetectLanguages(sourceRoot) + if !slices.Contains(langs, "Go") { + return false + } + if _, lookErr := exec.LookPath("go"); lookErr != nil { + if len(langs) == 1 { + out.Fatal("A Go project needs the Go toolchain, but `go` was not found on your PATH.\n" + + "Install Go (https://go.dev/dl/), make sure `go` is on your PATH, and run the command again.") + } + out.Warnf("A Go module (go.mod) was found, but `go` is not on your PATH. " + + "The scan continues with the other detected languages and does not analyze the Go code. " + + "To analyze Go, install Go (https://go.dev/dl/).") + return false + } + return true +} + +// resolveGoServerEnv gets the go-ssa-server binary (a download, or the +// --go-server-binary override) and returns the analyzer env that points +// GOIR_SERVER_BINARY at its absolute path. Call it only when +// goServerRequired returned true. +func resolveGoServerEnv() map[string]string { + goServerPath, err := EnsureGoServerAvailable() + if err != nil { + out.Fatalf("Failed to resolve go-ssa-server: %s", err) + } + return map[string]string{"GOIR_SERVER_BINARY": goServerPath} +} + +// goServerEnvForModel does goServerRequired and then resolveGoServerEnv for +// a compiled project model. It reads the source root that the model recorded. +// It returns the go-ssa-server env when the model needs it, and nil when not. +// The result goes directly to newAnalyzerJavaRunner, where WithExtraEnv(nil) +// does nothing. +func goServerEnvForModel(projectModelPath string) map[string]string { + sourceRoot, err := project.GetSourceRoot(projectModelPath) + if err != nil { + out.Fatalf("Failed to parse sourceRoot from project.yaml: %v", err) + } + if !goServerRequired(sourceRoot) { + return nil + } + return resolveGoServerEnv() +} + func scanProject(analyzerBuilder *AnalyzerBuilder, javaRunner java.JavaRunner) (*java.JavaCommandError, error) { analyzerCommand := analyzerBuilder.BuildNativeCommand() diff --git a/cli/cmd/scan_test.go b/cli/cmd/scan_test.go new file mode 100644 index 0000000000..14c6c95764 --- /dev/null +++ b/cli/cmd/scan_test.go @@ -0,0 +1,94 @@ +package cmd + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/seqra/opentaint/internal/globals" +) + +// The --go-server-binary override short-circuits EnsureGoServerAvailable: when +// globals.Config.GoServer.Binary is set it validates the path exists, returns +// its absolute form, and performs no download. These tests exercise only that +// override branch, so no network access is involved. + +func TestEnsureGoServerAvailable_BinaryOverrideReturnsAbsPath(t *testing.T) { + orig := globals.Config.GoServer.Binary + t.Cleanup(func() { globals.Config.GoServer.Binary = orig }) + + dir := t.TempDir() + bin := filepath.Join(dir, "go-ssa-server") + if err := os.WriteFile(bin, []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + globals.Config.GoServer.Binary = bin + + got, err := EnsureGoServerAvailable() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !filepath.IsAbs(got) { + t.Errorf("EnsureGoServerAvailable() = %q, want an absolute path", got) + } + want, _ := filepath.Abs(bin) + if got != want { + t.Errorf("EnsureGoServerAvailable() = %q, want %q", got, want) + } +} + +func TestEnsureGoServerAvailable_BinaryOverrideRelativePathResolvedToAbs(t *testing.T) { + orig := globals.Config.GoServer.Binary + t.Cleanup(func() { globals.Config.GoServer.Binary = orig }) + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "go-ssa-server"), []byte("x"), 0o755); err != nil { + t.Fatal(err) + } + t.Chdir(dir) + + globals.Config.GoServer.Binary = "go-ssa-server" // relative to cwd + + got, err := EnsureGoServerAvailable() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !filepath.IsAbs(got) { + t.Errorf("EnsureGoServerAvailable() = %q, want absolute path from relative override", got) + } + if filepath.Base(got) != "go-ssa-server" { + t.Errorf("EnsureGoServerAvailable() basename = %q, want %q", filepath.Base(got), "go-ssa-server") + } +} + +func TestEnsureGoServerAvailable_BinaryOverrideMissingReturnsError(t *testing.T) { + orig := globals.Config.GoServer.Binary + t.Cleanup(func() { globals.Config.GoServer.Binary = orig }) + + globals.Config.GoServer.Binary = filepath.Join(t.TempDir(), "does-not-exist") + + got, err := EnsureGoServerAvailable() + if err == nil { + t.Fatalf("expected error for non-existent override binary, got path %q", got) + } + if !strings.Contains(err.Error(), "not found") { + t.Errorf("error should mention binary not found, got: %v", err) + } +} + +func TestEnsureGoServerAvailable_BinaryOverrideDirectoryReturnsError(t *testing.T) { + orig := globals.Config.GoServer.Binary + t.Cleanup(func() { globals.Config.GoServer.Binary = orig }) + + // Point the override at a directory instead of an executable file. + globals.Config.GoServer.Binary = t.TempDir() + + got, err := EnsureGoServerAvailable() + if err == nil { + t.Fatalf("expected error when override points at a directory, got path %q", got) + } + if !strings.Contains(err.Error(), "directory") { + t.Errorf("error should mention the path is a directory, got: %v", err) + } +} diff --git a/cli/cmd/suggest.go b/cli/cmd/suggest.go index 9508a86cb0..70b9283c51 100644 --- a/cli/cmd/suggest.go +++ b/cli/cmd/suggest.go @@ -3,23 +3,135 @@ package cmd import ( "fmt" "os" + "strconv" + "strings" + "time" + "github.com/seqra/opentaint/internal/analyzer" "github.com/seqra/opentaint/internal/globals" "github.com/seqra/opentaint/internal/output" ) -// dockerFallbackHintPrefix is the shared lead-in for the Docker-based fallback -// hints emitted when native compilation can't find a suitable Java. compile and -// scan complete it with their respective "compilation:" / "scan:" suffix. -const dockerFallbackHintPrefix = "If native compilation fails due to missing required Java, set JAVA_HOME according to the project's requirements or try Docker-based " - func suggest(description, command string) { out.Suggest(description, command) } -// logSuggestion returns a Suggestion pointing at the active log file. The -// second result is false when no log file is active (e.g. failures that occur -// before logging is activated), in which case callers omit it. +const ( + maxRetryMemory = 16 + maxRetryTimeout = 15 * time.Minute +) + +func withFlag(command, flag string) string { + for _, token := range strings.Fields(command) { + if token == flag || strings.HasPrefix(token, flag+"=") { + return command + } + } + return command + " " + flag +} + +func retrySuggestion(exitCode int, timeout time.Duration, maxMemory string) (output.Suggestion, bool) { + switch exitCode { + case analyzer.ExitOOM: + if memoryGib(maxMemory) >= maxRetryMemory { + return output.Suggestion{}, false + } + next := doubleMemory(maxMemory) + return output.Suggestion{ + Description: "To retry with more memory, run:", + Command: rerunReplacingFlag(next, "--max-memory"), + }, true + case analyzer.ExitTimeout: + if timeout >= maxRetryTimeout { + return output.Suggestion{}, false + } + next := timeout * 2 + if next <= timeout || next > maxRetryTimeout { + next = maxRetryTimeout + } + return output.Suggestion{ + Description: "To retry with a longer timeout, run:", + Command: rerunReplacingFlag(next.String(), "--timeout", "-t"), + }, true + } + return output.Suggestion{}, false +} + +func rerunReplacingFlag(value string, names ...string) string { + args := []string{"opentaint"} + skipNext := false + for _, arg := range os.Args[1:] { + if skipNext { + skipNext = false + continue + } + matched := false + for _, name := range names { + if arg == name { + matched = true + skipNext = true + break + } + if strings.HasPrefix(arg, name+"=") { + matched = true + break + } + } + if matched { + continue + } + args = append(args, shellQuote(arg)) + } + args = append(args, names[0], shellQuote(value)) + return strings.Join(args, " ") +} + +func doubleMemory(value string) string { + digits := 0 + for digits < len(value) && value[digits] >= '0' && value[digits] <= '9' { + digits++ + } + suffix := value[digits:] + if digits == 0 || len(suffix) > 1 { + return "16G" + } + n, err := strconv.ParseInt(value[:digits], 10, 64) + if err != nil || n <= 0 { + return "16G" + } + if memoryGib(value)*2 >= maxRetryMemory { + return "16G" + } + return fmt.Sprintf("%d%s", n*2, suffix) +} + +func memoryGib(value string) int { + digits := 0 + for digits < len(value) && value[digits] >= '0' && value[digits] <= '9' { + digits++ + } + suffix := value[digits:] + if digits == 0 { + return 0 + } + n, err := strconv.ParseInt(value[:digits], 10, 64) + if err != nil || n <= 0 { + return 0 + } + switch suffix { + case "G", "g": + return int(n) + case "M", "m": + return int(n / 1024) + case "K", "k": + return int(n / (1024 * 1024)) + case "": + return int(n / (1024 * 1024 * 1024)) + default: + return 0 + } +} + func logSuggestion() (output.Suggestion, bool) { if globals.LogPath == "" { return output.Suggestion{}, false @@ -30,9 +142,6 @@ func logSuggestion() (output.Suggestion, bool) { }, true } -// appendLogSuggestion appends the log-file pointer to s when a log file is -// active and returns s unchanged otherwise. It centralizes the "lead with the -// log" idiom shared by buildFailSuggestions and the scan summary tail. func appendLogSuggestion(s []output.Suggestion) []output.Suggestion { if logSug, ok := logSuggestion(); ok { return append(s, logSug) @@ -40,29 +149,16 @@ func appendLogSuggestion(s []output.Suggestion) []output.Suggestion { return s } -// buildFailSuggestions returns a pointer to the log file (when one exists) -// followed by the contextual hints. The log pointer leads so the user always -// sees where to look for full details first. Building onto a fresh slice avoids -// mutating a caller's backing array when contextual is passed via the spread form. func buildFailSuggestions(contextual []output.Suggestion) []output.Suggestion { return append(appendLogSuggestion(nil), contextual...) } -// failWith prints an error message, renders a single Suggestions block leading -// with a pointer to the log file (when one exists) followed by any contextual -// hints, then exits the process with the given code. Use it for operational -// compile/scan failures; pure input/argument errors stay on out.Fatalf with -// their own usage hints. func failWith(code int, message string, contextual ...output.Suggestion) { out.Error(message) out.Suggestions(buildFailSuggestions(contextual)...) os.Exit(code) } -// failf formats an error message and fails with exit code 1 and no contextual -// suggestion. The log-file pointer is still added when a log file exists. Use -// it as the drop-in for bare out.Fatalf at operational compile/scan failure -// sites; see failWith. func failf(format string, args ...any) { failWith(1, fmt.Sprintf(format, args...)) } diff --git a/cli/cmd/summary.go b/cli/cmd/summary.go index 0959b5257c..9b6da780ce 100644 --- a/cli/cmd/summary.go +++ b/cli/cmd/summary.go @@ -9,14 +9,34 @@ import ( // summaryCmd represents the summary command var summaryCmd = &cobra.Command{ - Use: "summary sarif", - Short: "Print summary of your sarif", + Use: "summary ", + Short: "Show a summary of a SARIF report", Args: cobra.ExactArgs(1), // require exactly one argument - Long: `Print summary of your sarif file + Long: `Show a summary of a SARIF report in the terminal. The summary counts the findings by severity. It also shows which rules ran and which rules found problems. -Arguments: - sarif - Path to a sarif file -`, +The sarif-report argument is the path to a SARIF report. It is required. Use a report from "opentaint scan" or "opentaint test". + +To see each finding, use --show-findings. To make the list smaller, use --severity, --rule-id, or --path. To see the full data flow, use --verbose-flow and --show-code-snippets. + +This command only reads the report. It does not write files.`, + Example: ` # Show a summary of a report + opentaint summary report.sarif + + # Show each finding with its location + opentaint summary report.sarif --show-findings + + # Show only the error-level findings + opentaint summary report.sarif --show-findings --severity error + + # Group the findings by rule + opentaint summary report.sarif --show-findings --group-by rule-id + + # Recipe: examine one rule in full detail + opentaint summary report.sarif --show-findings --group-by rule-id + opentaint summary report.sarif --show-findings --rule-id --verbose-flow --show-code-snippets + + # Recipe: read the findings for one part of the code + opentaint summary report.sarif --show-findings --path "src/main/**" --severity error`, Run: func(cmd *cobra.Command, args []string) { for _, s := range summarySeverities { @@ -39,6 +59,13 @@ Arguments: out.Fatalf("Failed to load SARIF report: %s", err) } printSarifSummary(report, absSarifPath, summaryFilters(), summaryListingOptions(dim, codeFlowSel)) + + if !showFindings && sarif.GenerateSummary(report.Filter(summaryFilters())).TotalFindings > 0 { + out.Suggest( + "To list the findings, run:", + currentSummaryBuilder(absSarifPath).WithShowFindings().Build(), + ) + } }, } @@ -58,16 +85,16 @@ var summaryCodeFlow string func init() { rootCmd.AddCommand(summaryCmd) - summaryCmd.Flags().BoolVar(&showFindings, "show-findings", false, "Show all issues from Sarif file") + summaryCmd.Flags().BoolVar(&showFindings, "show-findings", false, "Show every finding in the SARIF report") summaryCmd.Flags().BoolVar(&showCodeSnippets, "show-code-snippets", false, "Show finding related code snippets") summaryCmd.Flags().BoolVar(&verboseFlow, "verbose-flow", false, "Show full code flow steps for findings") summaryCmd.Flags().StringArrayVar(&summaryPaths, "path", nil, "Show only findings whose file path matches this glob (** supported, repeatable)") - summaryCmd.Flags().StringArrayVar(&summarySeverities, "severity", nil, "Show only findings of this SARIF level: error, warning, note, none (repeatable)") - summaryCmd.Flags().StringArrayVar(&summaryRuleIDs, "rule-id", nil, "Show only findings for this rule: full id, leaf name, or glob (repeatable)") + summaryCmd.Flags().StringArrayVar(&summarySeverities, "severity", nil, "Show only findings at these SARIF levels: note, warning, error, none (repeatable)") + summaryCmd.Flags().StringArrayVar(&summaryRuleIDs, "rule-id", nil, "Show only findings from this rule: full id, leaf name, or glob (repeatable)") summaryCmd.Flags().StringArrayVar(&summaryFingerprints, "partial-fingerprint", nil, "Show only findings whose partial fingerprint starts with this value (git-hash style, repeatable)") - summaryCmd.Flags().StringVar(&summaryFingerprintKey, "partial-fingerprint-key", "", "partialFingerprints key matched by --partial-fingerprint (default vulnerabilityWithTraceHash/v1)") + summaryCmd.Flags().StringVar(&summaryFingerprintKey, "partial-fingerprint-key", "", "partialFingerprints key matched by --partial-fingerprint (defaults to vulnerabilityWithTraceHash/v1)") summaryCmd.Flags().IntVar(&summaryMaxNestingLevel, "max-nesting-level", -1, "Collapse code-flow steps deeper than this call-nesting level (-1 = no cap)") - summaryCmd.Flags().StringVar(&summaryGroupBy, "group-by", "", "Group the --show-findings listing by: severity, rule-id, file-path (default file-path)") + summaryCmd.Flags().StringVar(&summaryGroupBy, "group-by", "", "Group the --show-findings listing by: severity, rule-id, file-path (defaults to file-path)") summaryCmd.Flags().StringVar(&summaryCodeFlow, "code-flow", "", "Render code flows: \"all\", a 1-based index, or unset (first only)") } @@ -137,7 +164,7 @@ func printSarifSummary(report *sarif.Report, absSarifPath string, filters sarif. if showFindings && hasOmittedFlow && !verboseFlow { out.Suggest( - "To see full code flow and code snippets, use:", + "To see the full code flow and code snippets, run:", currentSummaryBuilder(absSarifPath).WithVerboseFlow().WithShowCodeSnippets().Build(), ) } diff --git a/cli/cmd/test.go b/cli/cmd/test.go index a31a3cc73b..9fffee8f1a 100644 --- a/cli/cmd/test.go +++ b/cli/cmd/test.go @@ -8,40 +8,56 @@ import ( var testCmd = &cobra.Command{ Use: "test", - Short: "Create and run rule and approximation tests", - Long: `Tools for creating test projects, running annotated rule and approximation tests, and debugging rule reachability.`, + Short: "Create and run rule and model tests", + Long: `Create and run tests for detection rules and dataflow models. Rule tests make sure that a rule finds the positive samples and ignores the negative samples. Dataflow model tests make sure that a model moves taint from source to sink. + +Workflow: + 1. Create a test project with init. + 2. Compile the project with "opentaint compile". + 3. Run the samples with "test rule run" or "test dataflow-model run". + +To see why one rule does or does not fire, use "test rule reachability".`, } var testRuleCmd = &cobra.Command{ Use: "rule", Short: "Create, run, and debug detection-rule tests", + Long: `Create, run, and debug tests for taint detection rules. A rule test makes sure that a rule finds the positive samples and ignores the negative samples. + +Workflow: + 1. Create a test project with "test rule init". + 2. Compile the project with "opentaint compile". + 3. Run the samples with "test rule run". + +To see why one rule does or does not fire, use "test rule reachability".`, } -var testApproximationCmd = &cobra.Command{ - Use: "approximation", - Short: "Create and run dataflow-approximation tests", +var testModelCmd = &cobra.Command{ + Use: "dataflow-model", + Aliases: []string{"approximation"}, + Short: "Create and run dataflow model tests", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + if cmd.Parent().CalledAs() == "approximation" { + out.Warnf("command 'test approximation' is deprecated; use 'test dataflow-model' instead") + } + }, + Long: `Create and run tests for dataflow models. A model test makes sure that a model moves taint from source to sink in your samples. + +Workflow: + 1. Create a test project with "test dataflow-model init". + 2. Compile the project with "opentaint compile". + 3. Run the samples with "test dataflow-model run --java-models ".`, } func init() { rootCmd.AddCommand(testCmd) testCmd.AddCommand(testRuleCmd) - testCmd.AddCommand(testApproximationCmd) -} - -func testExitCodesHelp(passedLine string) string { - return `Exit codes: - 0 ` + passedLine + ` - 1 General failure (configuration or infrastructure error) - 2 One or more tests failed (false negatives/positives or skipped samples) - 252 Unhandled analyzer exception - 253 Out of memory (try increasing --max-memory) - 254 Analysis timed out (try increasing --timeout) - 255 Project configuration error` + testCmd.AddCommand(testModelCmd) } func addTestRunFlags(cmd *cobra.Command, outputDir *string, timeout *time.Duration, maxMemory *string, dataflow *[]string) { cmd.Flags().StringVarP(outputDir, "output", "o", "", "Directory for test-result.json and test-results.sarif") - cmd.Flags().DurationVar(timeout, "timeout", 600*time.Second, "Analysis timeout") - cmd.Flags().StringVar(maxMemory, "max-memory", "8G", "Maximum analyzer heap size (e.g., 8G)") - cmd.Flags().StringArrayVar(dataflow, "dataflow-approximations", nil, "Dataflow approximation project, build output, or class directory (repeatable)") + cmd.Flags().DurationVar(timeout, "timeout", 600*time.Second, "Maximum wall-clock time for analysis (e.g. 30m, 1h)") + cmd.Flags().StringVar(maxMemory, "max-memory", "8G", "Maximum analyzer heap size (e.g. 8G, 1024m)") + addRenamedStringArrayFlag(cmd.Flags(), dataflow, "java-models", "dataflow-approximations", "Java dataflow models: a model project, build output, or class directory (repeatable)") } diff --git a/cli/cmd/test_approximation_run.go b/cli/cmd/test_approximation_run.go index 018e16c03d..d2928e8d20 100644 --- a/cli/cmd/test_approximation_run.go +++ b/cli/cmd/test_approximation_run.go @@ -15,15 +15,27 @@ var ( testApproxDataflow []string ) -var testApproximationRunCmd = &cobra.Command{ +var testModelRunCmd = &cobra.Command{ Use: "run ", - Short: "Run dataflow approximation tests on a compiled project model", - Long: `Run the samples specified in rule-test.yaml with the supplied dataflow approximations applied. + Short: "Run dataflow model tests on a compiled project model", + Long: `Run the samples that rule-test.yaml declares, with your dataflow models applied. The command reports which samples passed. A fixed source-to-sink harness rule is applied automatically. Positive samples point to it with the id model-rule. -A built-in source-to-sink harness rule is applied automatically; positive samples reference the -approximation-rule.yaml rule with id "approximation-rule". +The project-model argument is a compiled project model directory from "opentaint compile". Give the model under test with --java-models. -` + testExitCodesHelp("All approximation tests passed"), +The command writes test-result.json and a test-results.sarif report to --output. If --output is not set, it writes to a temporary directory. + +Compile the test project before you run the tests. To read the results, use "opentaint summary". + +` + testExitCodesHelp("All model tests passed"), + Example: ` # Run a dataflow model test on a compiled model + opentaint test dataflow-model run ./model-test/model --java-models ./my-model + + # Write the results to a directory + opentaint test dataflow-model run ./model-test/model --java-models ./my-model -o ./results + + # Recipe: change a model, then make sure the tests stay green + opentaint test dataflow-model run ./model-test/model --java-models ./my-model -o ./results + opentaint summary ./results/test-results.sarif --show-findings`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { ruleDir, err := os.MkdirTemp("", "opentaint-approx-rule-*") @@ -35,7 +47,8 @@ approximation-rule.yaml rule with id "approximation-rule". } runTestProject(args[0], testProjectOptions{ - label: "Approximation tests", + label: "Model tests", + passedLine: "All model tests passed.", tempDir: "opentaint-test-approximations-*", rulesets: []string{ruleDir}, outputDir: testApproxOutputDir, @@ -47,6 +60,6 @@ approximation-rule.yaml rule with id "approximation-rule". } func init() { - testApproximationCmd.AddCommand(testApproximationRunCmd) - addTestRunFlags(testApproximationRunCmd, &testApproxOutputDir, &testApproxTimeout, &testApproxMaxMemory, &testApproxDataflow) + testModelCmd.AddCommand(testModelRunCmd) + addTestRunFlags(testModelRunCmd, &testApproxOutputDir, &testApproxTimeout, &testApproxMaxMemory, &testApproxDataflow) } diff --git a/cli/cmd/test_init.go b/cli/cmd/test_init.go index 2c4966f822..0a3df6daa2 100644 --- a/cli/cmd/test_init.go +++ b/cli/cmd/test_init.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" + "github.com/seqra/opentaint/internal/output" "github.com/seqra/opentaint/internal/testapprox" "github.com/seqra/opentaint/internal/testproject" "github.com/seqra/opentaint/internal/testrule" @@ -18,19 +19,26 @@ var initRuleSourcesOnly bool var testRuleInitCmd = &cobra.Command{ Use: "init ", Short: "Create rule test projects with source and sink harnesses", - Long: `Create one or two Gradle test projects under . The sinks -project tests sink rules against a generic Taint source; the sources project -tests source rules against a generic Taint sink. Use --sinks-only or ---sources-only when only one project is needed. + Long: `Create one or two Gradle test projects for detection-rule tests. The sinks project tests sink rules with a generic taint source. The sources project tests source rules with a generic taint sink. -Each project includes: - - build.gradle.kts with compile-only dependencies, settings.gradle.kts - - src/main/java/test/ with Taint.java (the generic source()/sink()) for test sample sources - - test-rules/java/lib/test/generic-{source,sink}.yaml marker rules for test-only joins +The output-dir argument is the parent directory for the new projects. The default creates the two projects, in output-dir/sinks and output-dir/sources. To create one project only, use --sinks-only or --sources-only. To add compile-only Maven dependencies for the samples, use --dependency. -Positive and negative samples are specified via rule-test.yaml. +Each project contains a rule-test.yaml file and a Taint.java harness. Declare your positive and negative samples in rule-test.yaml. -Use --dependency to add compile-only Maven dependencies for the samples.`, +Then compile the project with "opentaint compile" and run the samples with "opentaint test rule run".`, + Example: ` # Create the sinks and the sources test projects + opentaint test rule init ./rule-tests + + # Create only the sinks project + opentaint test rule init ./rule-tests --sinks-only + + # Add a compile-only dependency for the samples + opentaint test rule init ./rule-tests --dependency + + # Recipe: from an empty directory to a first test run + opentaint test rule init ./rule-tests + opentaint compile ./rule-tests/sinks -o ./rule-tests/sinks/model + opentaint test rule run ./rule-tests/sinks/model --ruleset ./my-rules`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { if initRuleSinksOnly && initRuleSourcesOnly { @@ -50,37 +58,58 @@ Use --dependency to add compile-only Maven dependencies for the samples.`, if err := testrule.Scaffold(dir); err != nil { out.Fatalf("Failed to scaffold rule test project: %s", err) } - fmt.Printf("Rule test project (%s) initialized at %s\n", kind, dir) + out.Printf("Rule test project (%s) initialized at %s", kind, dir) } + var suggestions []output.Suggestion + for _, kind := range kinds { + dir := filepath.Join(args[0], kind) + modelDir := filepath.Join(dir, "model") + suggestions = append(suggestions, + output.Suggestion{Description: fmt.Sprintf("To add %s test samples, edit:", kind), Command: filepath.Join(dir, "rule-test.yaml")}, + output.Suggestion{Description: fmt.Sprintf("To compile the %s test project, run:", kind), Command: fmt.Sprintf("opentaint compile %s -o %s", shellQuote(dir), shellQuote(modelDir))}, + output.Suggestion{Description: fmt.Sprintf("To run the %s tests, run:", kind), Command: fmt.Sprintf("opentaint test rule run %s", shellQuote(modelDir))}, + ) + } + out.Suggestions(suggestions...) }, } -var testApproximationInitCmd = &cobra.Command{ +var testModelInitCmd = &cobra.Command{ Use: "init ", - Short: "Create a dataflow approximation test project", - Long: `Create a minimal Gradle project for testing OpenTaint dataflow approximations. + Short: "Create a dataflow model test project", + Long: `Create a Gradle test project for dataflow model tests. The project contains a fixed source-to-sink rule. The samples are checked against this rule. + +The output-dir argument is the directory for the new project. To add compile-only Maven dependencies for the samples, use --dependency. The model under test is not part of the project. Give it at run time with --java-models. -The project includes: - - build.gradle.kts with compile-only dependencies - - settings.gradle.kts - - approximation-rule.yaml, the fixed source-to-sink rule the samples are checked against - - src/main/java/test/ with Taint.java (the fixed source() and sink()) for test sample sources +The project contains a rule-test.yaml file, a Taint.java source and sink, and a fixed source-to-sink harness rule. Declare your positive and negative samples in rule-test.yaml. -Positive and negative samples are specified via rule-test.yaml. +Then compile the project with "opentaint compile" and run the samples with "opentaint test dataflow-model run".`, + Example: ` # Create a dataflow model test project + opentaint test dataflow-model init ./model-test -The approximation under test is supplied separately at test time with ---dataflow-approximations. + # Add a compile-only dependency for the samples + opentaint test dataflow-model init ./model-test --dependency -Use --dependency to add compile-only Maven dependencies for the samples.`, + # Recipe: from an empty directory to a first test run + opentaint test dataflow-model init ./model-test + opentaint compile ./model-test -o ./model-test/model + opentaint test dataflow-model run ./model-test/model --java-models ./my-model`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - if err := testproject.Bootstrap(args[0], "approximation-test-project", initApproxProjectDeps); err != nil { + if err := testproject.Bootstrap(args[0], "model-test-project", initApproxProjectDeps); err != nil { out.Fatalf("Failed to bootstrap test project: %s", err) } if err := testapprox.Scaffold(args[0]); err != nil { - out.Fatalf("Failed to scaffold approximation project: %s", err) + out.Fatalf("Failed to scaffold model test project: %s", err) } - fmt.Printf("Approximation test project initialized at %s\n", args[0]) + out.Printf("Model test project initialized at %s", args[0]) + dir := args[0] + modelDir := filepath.Join(dir, "model") + out.Suggestions( + output.Suggestion{Description: "To add your test samples, edit:", Command: filepath.Join(dir, "rule-test.yaml")}, + output.Suggestion{Description: "To compile the test project, run:", Command: fmt.Sprintf("opentaint compile %s -o %s", shellQuote(dir), shellQuote(modelDir))}, + output.Suggestion{Description: "To run the tests, run:", Command: fmt.Sprintf("opentaint test dataflow-model run %s --java-models ", shellQuote(modelDir))}, + ) }, } @@ -93,7 +122,7 @@ func init() { testRuleInitCmd.Flags().BoolVar(&initRuleSourcesOnly, "sources-only", false, "Create only the sources test project") - testApproximationCmd.AddCommand(testApproximationInitCmd) - testApproximationInitCmd.Flags().StringArrayVar(&initApproxProjectDeps, "dependency", nil, + testModelCmd.AddCommand(testModelInitCmd) + testModelInitCmd.Flags().StringArrayVar(&initApproxProjectDeps, "dependency", nil, "Compile-only Maven dependency coordinates for generated samples (repeatable)") } diff --git a/cli/cmd/test_rule_reachability.go b/cli/cmd/test_rule_reachability.go index ecb0bf3231..20714bdf83 100644 --- a/cli/cmd/test_rule_reachability.go +++ b/cli/cmd/test_rule_reachability.go @@ -8,12 +8,31 @@ var reachabilityEntryPoint string var testRuleReachabilityCmd = &cobra.Command{ Use: "reachability [source-path]", - Short: "Trace why a rule can or cannot reach its facts", - Long: `Scan a project with one rule and write a sibling fact-reachability SARIF -report (debug-ifds-fact-reachability.sarif) next to the main one. Use this to -debug why a rule does or does not fire. + Short: "Show why a rule does or does not fire", + Long: `Scan a project with one rule and write a fact-reachability SARIF report. The report shows why the rule does or does not fire. Library source and sink rules that the rule points to are included automatically. -Referenced library source and sink rules are collected and analyzed automatically.`, +The rule-id argument selects the rule. The source-path argument is the project root. It is optional. The default is the current directory. To use a compiled model, use --project-model. Do not give source-path and --project-model together. To start the analysis from one method, use --entry-points. + +The report name is debug-ifds-fact-reachability.sarif. It is written adjacent to the main SARIF report. + +Before the first run, run "opentaint pull" one time. To read the report, use "opentaint summary". + +` + scanExitCodesHelp("Reachability analysis completed"), + Example: ` # Show why a rule does or does not fire on the current directory + opentaint test rule reachability . + + # Examine a rule on a compiled project model + opentaint test rule reachability --project-model ./model + + # Start the analysis from one entry-point method + opentaint test rule reachability . --entry-points com.example.App#main + + # Make sure the inputs are correct, without a scan + opentaint test rule reachability . --dry-run + + # Recipe: find why a new rule stays silent + opentaint test rule reachability . -o report.sarif + opentaint summary debug-ifds-fact-reachability.sarif --show-findings --verbose-flow`, Annotations: map[string]string{"PrintConfig": "true"}, Args: cobra.RangeArgs(1, 2), Run: func(cmd *cobra.Command, args []string) { diff --git a/cli/cmd/test_rule_run.go b/cli/cmd/test_rule_run.go index 6c7261e0fa..f20e798c20 100644 --- a/cli/cmd/test_rule_run.go +++ b/cli/cmd/test_rule_run.go @@ -7,6 +7,7 @@ import ( "time" "github.com/seqra/opentaint/internal/analyzer" + "github.com/seqra/opentaint/internal/output" "github.com/seqra/opentaint/internal/utils" "github.com/seqra/opentaint/internal/utils/log" "github.com/spf13/cobra" @@ -25,14 +26,32 @@ var ( var testRuleRunCmd = &cobra.Command{ Use: "run ", Short: "Run detection-rule tests on a compiled project model", - Long: `Run detection rules against the samples specified in rule-test.yaml in the -compiled project model. + Long: `Run detection rules on the samples that rule-test.yaml declares. The command reports which samples passed. The built-in rules are always included. + +The project-model argument is a compiled project model directory from "opentaint compile". To add your own rules, use --ruleset. To run only specified rules, use --rule-id. To apply models, use --java-models or --passthrough-models. + +The command writes test-result.json and a test-results.sarif report to --output. If --output is not set, it writes to a temporary directory. + +Compile the test project before you run the tests. To read the results, use "opentaint summary". ` + testExitCodesHelp("All rule tests passed"), + Example: ` # Run the built-in rules on a compiled model + opentaint test rule run ./rule-tests/sinks/model + + # Test your own rules and write the results to a directory + opentaint test rule run ./rule-tests/sinks/model --ruleset ./rules -o ./results + + # Run only one rule + opentaint test rule run ./rule-tests/sinks/model --rule-id + + # Recipe: change a rule, then make sure the tests stay green + opentaint test rule run ./rule-tests/sinks/model --ruleset ./rules -o ./results + opentaint summary ./results/test-results.sarif --show-findings`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { runTestProject(args[0], testProjectOptions{ label: "Rule tests", + passedLine: "All rule tests passed.", tempDir: "opentaint-test-rules-*", rulesets: testRulesRuleset, outputDir: testRulesOutputDir, @@ -48,6 +67,7 @@ compiled project model. type testProjectOptions struct { label string + passedLine string tempDir string rulesets []string outputDir string @@ -104,7 +124,7 @@ func runTestProject(projectModelArg string, opts testProjectOptions) { if opts.includeBuiltinRules { rulesPath, err := utils.EnsureRulesPath(out) if err != nil { - out.Fatalf("Failed to prepare built-in rules: %s", err) + failf("Failed to prepare built-in rules: %s", err) } builder.AddRuleSet(rulesPath) } @@ -124,54 +144,68 @@ func runTestProject(projectModelArg string, opts testProjectOptions) { analyzerJarPath, err := ensureAnalyzerAvailable() if err != nil { - out.Fatalf("Failed to resolve analyzer: %s", err) + failf("Failed to resolve analyzer: %s", err) } builder.SetJarPath(analyzerJarPath) addDataflowApproximations(builder, opts.dataflowApprox, analyzerJarPath) addPassthroughApproximations(builder, opts.passthroughApprox) - javaRunner := newAnalyzerJavaRunner() + javaRunner := newAnalyzerJavaRunner(goServerEnvForModel(projectPath)) if _, err := javaRunner.EnsureJava(); err != nil { - out.Fatalf("Failed to resolve Java for analyzer: %s", err) + failf("Failed to resolve Java for analyzer: %s", err) } cmdErr, err := scanProject(builder, javaRunner) if err != nil { - out.Fatalf("%s failed: %s", opts.label, err) + failf("%s failed: %s", opts.label, err) } analyzerFail := analyzer.Classify(cmdErr) - if analyzerFail != nil { - out.Error(analyzerFail.Message) - } resultPath := filepath.Join(outputDir, "test-result.json") - fmt.Printf("Results directory: %s\n", outputDir) - fmt.Printf("Test results: %s\n", resultPath) + out.Printf("Results directory: %s", outputDir) + out.Printf("Test results: %s", resultPath) if analyzerFail != nil { + out.Error(analyzerFail.Message) + hint := output.Suggestion{ + Description: "To stream the analyzer output, re-run with --debug:", + Command: withFlag(rerunWithoutDryRun(), "--debug"), + } + if retry, ok := retrySuggestion(analyzerFail.ExitCode, opts.timeout, opts.maxMemory); ok { + hint = retry + } + out.Suggestions(append(appendLogSuggestion(nil), hint)...) os.Exit(analyzerFail.ExitCode) } tr, err := analyzer.LoadTestResult(resultPath) if err != nil { - out.Fatalf("%s produced no readable test-result.json: %s", opts.label, err) + failf("%s produced no readable test-result.json: %s", opts.label, err) } - fmt.Printf("Passed: %d, failed: %d (false negatives: %d, false positives: %d, skipped: %d), disabled: %d\n", + out.Printf("Passed: %d, failed: %d (false negatives: %d, false positives: %d, skipped: %d), disabled: %d", len(tr.Success), tr.Failed(), len(tr.FalseNegative), len(tr.FalsePositive), len(tr.Skipped), len(tr.Disabled)) + + viewResultsCommand := utils.NewSummaryCommand(filepath.Join(outputDir, "test-results.sarif")).WithShowFindings().Build() + if tr.Failed() > 0 { out.Error(fmt.Sprintf("%s failed", opts.label)) + out.Suggestions(append(appendLogSuggestion(nil), output.Suggestion{ + Description: "To inspect the failing samples, run:", + Command: viewResultsCommand, + })...) os.Exit(2) } - fmt.Printf("%s completed successfully\n", opts.label) + out.Successf("%s", opts.passedLine) + suggest("To view the test results, run:", viewResultsCommand) } func init() { testRuleCmd.AddCommand(testRuleRunCmd) - testRuleRunCmd.Flags().StringArrayVar(&testRulesRuleset, "ruleset", nil, "Ruleset file or directory to test (repeatable)") + testRuleRunCmd.Flags().StringArrayVar(&testRulesRuleset, "ruleset", nil, "Ruleset to test: a YAML file or a directory of .yml or .yaml files (repeatable)") addTestRunFlags(testRuleRunCmd, &testRulesOutputDir, &testRulesTimeout, &testRulesMaxMemory, &testRulesDataflow) testRuleRunCmd.Flags().StringArrayVar(&testRulesRuleID, "rule-id", nil, "Run only rules with this ID (repeatable)") - testRuleRunCmd.Flags().StringArrayVar(&testRulesPassthrough, "passthrough-approximations", nil, "Pass-through approximation YAML file or directory (repeatable)") + addRenamedStringArrayFlag(testRuleRunCmd.Flags(), &testRulesPassthrough, "passthrough-models", "passthrough-approximations", "Pass-through models: a YAML file or a directory of them (repeatable)") } diff --git a/cli/cmd/update.go b/cli/cmd/update.go index 1d3a5c1bda..6ef0484d9b 100644 --- a/cli/cmd/update.go +++ b/cli/cmd/update.go @@ -20,13 +20,25 @@ var ( var updateCmd = &cobra.Command{ Use: "update [version]", Short: "Update opentaint to the latest version", - Long: `Update opentaint to the latest version (or a specific version). + Long: `Update the opentaint binary to the latest release. To get a specified version, give the version argument. Only upgrades are possible. The command refuses a version that is older than the current one. -This command detects how opentaint was installed and provides appropriate -instructions for package manager installations. For binary installations, -it performs an in-place update. +If opentaint was installed with Homebrew or npm, the command does not change the binary. It shows the correct package-manager command. -Only upgrades are supported — downgrading to an older version is refused.`, +To see the latest version without a download, use --check. To skip the confirmation prompt, use --yes. + +After a successful update, remove the old artifacts with "opentaint prune".`, + Example: ` # Update to the latest release + opentaint update + + # See if a newer version is available, without a download + opentaint update --check + + # Update to a specified version without a prompt + opentaint update 1.2.3 --yes + + # Recipe: update, then remove the artifacts of the old version + opentaint update --yes + opentaint prune --yes`, Args: cobra.MaximumNArgs(1), Run: func(cmd *cobra.Command, args []string) { // Check installation method first @@ -35,11 +47,11 @@ Only upgrades are supported — downgrading to an older version is refused.`, switch method { case utils.InstallMethodHomebrew: out.Print("opentaint was installed via Homebrew.") - out.Print("Run: brew upgrade --cask opentaint") + suggest("To update, run:", "brew upgrade --cask opentaint") return case utils.InstallMethodNpm: out.Print("opentaint was installed via npm.") - out.Print("Run: npm install -g @seqra/opentaint@latest") + suggest("To update, run:", "npm install -g @seqra/opentaint@latest") return } @@ -75,7 +87,7 @@ Only upgrades are supported — downgrading to an older version is refused.`, out.Warnf("Could not compare versions: %s", err) out.Printf("Current: %s, Latest: %s", currentVersion, targetVersion) if !updateYes { - out.Print("Use --yes to proceed anyway.") + suggest("To proceed anyway, run:", withFlag(rerunWithoutDryRun(), "--yes")) return } } @@ -94,9 +106,8 @@ Only upgrades are supported — downgrading to an older version is refused.`, out.Section("Update Available"). Field("Current version", fmt.Sprintf("v%s", currentVersion)). Field("Latest version", fmt.Sprintf("v%s", targetVersion)). - Line(). - Text("Run 'opentaint update' to update."). Render() + suggest("To update, run:", "opentaint update") return } @@ -107,6 +118,7 @@ Only upgrades are supported — downgrading to an older version is refused.`, if !updateYes { if !out.Confirm("Proceed with update?", false) { out.Print("Update cancelled.") + suggest("To update without confirming, run:", "opentaint update --yes") return } } @@ -132,7 +144,7 @@ Only upgrades are supported — downgrading to an older version is refused.`, } out.Successf("Successfully updated to v%s", targetVersion) - suggest("To clean up old artifacts run", "opentaint prune") + suggest("To clean up old artifacts, run:", "opentaint prune") }, } diff --git a/cli/internal/analyzer/exit.go b/cli/internal/analyzer/exit.go index 6d38e75992..9e97afbdad 100644 --- a/cli/internal/analyzer/exit.go +++ b/cli/internal/analyzer/exit.go @@ -37,9 +37,9 @@ func ExitMessage(code int) string { case ExitConfigError: return "project configuration error" case ExitTimeout: - return "analysis timed out — try increasing --timeout or --max-memory" + return "analysis timed out: try increasing --timeout or --max-memory" case ExitOOM: - return "out of memory — try increasing --max-memory (e.g. --max-memory 16G)" + return "out of memory: try increasing --max-memory (e.g. --max-memory 16G)" case ExitException: return "unhandled analyzer exception" default: diff --git a/cli/internal/approximation/resolve.go b/cli/internal/approximation/resolve.go index c0505e3474..a4225a9e91 100644 --- a/cli/internal/approximation/resolve.go +++ b/cli/internal/approximation/resolve.go @@ -147,7 +147,7 @@ func ambiguousDirError(dir, classFile string) error { "%s holds compiled classes of its own (e.g. %s) and an approximation project or build "+ "output below it, so it is neither a class directory nor a directory of them.\n"+ "Move the classes into a directory of their own, or pass each approximation path to "+ - "--dataflow-approximations separately", + "--java-models separately", dir, classFile) } diff --git a/cli/internal/globals/artifacts.go b/cli/internal/globals/artifacts.go index 95487b32ab..07778b17ac 100644 --- a/cli/internal/globals/artifacts.go +++ b/cli/internal/globals/artifacts.go @@ -1,6 +1,9 @@ package globals -import "strings" +import ( + "runtime" + "strings" +) // ArtifactDef declaratively describes a downloadable artifact. type ArtifactDef struct { @@ -26,6 +29,16 @@ func (a ArtifactDef) Kind() string { return strings.ToLower(a.Name) } +// TagPrefix returns the release-tag prefix of this artifact, for example +// "analyzer/". The go-ssa-server tag uses "go-server/", which differs from +// the artifact kind. +func (a ArtifactDef) TagPrefix() string { + if a.Kind() == "goserver" { + return "go-server/" + } + return a.Kind() + "/" +} + // IsBindVersion reports whether the configured version matches the compile-time bind version. func (a ArtifactDef) IsBindVersion() bool { return a.Version == a.BindVersion @@ -40,6 +53,14 @@ func (a ArtifactDef) WithVersion(v string) ArtifactDef { // Artifacts returns the list of artifact definitions, reading current globals and Config. // This is a function (not a var) so it picks up mutable bind version vars and Config at call time. func Artifacts() []ArtifactDef { + // go-ssa-server is a single native binary (Unpack:false). The bundled and + // install tiers store it under its platform asset name. The per-machine + // cache file keeps the windows .exe suffix. Thus the resolved path stays + // executable on windows. + goServerCacheSuffix := "" + if runtime.GOOS == "windows" { + goServerCacheSuffix = ".exe" + } return []ArtifactDef{ { Name: "Autobuilder", @@ -74,6 +95,18 @@ func Artifacts() []ArtifactDef { Version: Config.Rules.Version, Unpack: true, }, + { + Name: "GoServer", + RepoName: Config.Repo, + AssetName: GoServerAssetName(), + LibSubpath: GoServerAssetName(), + CachePrefix: GoServerAssetNamePrefix, + CacheSuffix: goServerCacheSuffix, + BindVersion: GoServerBindVersion, + Version: Config.GoServer.Version, + Override: Config.GoServer.Binary, + Unpack: false, + }, } } diff --git a/cli/internal/globals/artifacts_test.go b/cli/internal/globals/artifacts_test.go index 1b5c2d3128..b1dc99e317 100644 --- a/cli/internal/globals/artifacts_test.go +++ b/cli/internal/globals/artifacts_test.go @@ -23,11 +23,11 @@ func TestArtifacts(t *testing.T) { Config.Rules.Version = "v3.0.0" arts := Artifacts() - if len(arts) != 3 { - t.Fatalf("expected 3 artifacts, got %d", len(arts)) + if len(arts) != 4 { + t.Fatalf("expected 4 artifacts, got %d", len(arts)) } - names := []string{"Autobuilder", "Analyzer", "Rules"} + names := []string{"Autobuilder", "Analyzer", "Rules", "GoServer"} for i, want := range names { if arts[i].Name != want { t.Errorf("Artifacts()[%d].Name = %q, want %q", i, arts[i].Name, want) @@ -120,3 +120,14 @@ func TestArtifactByKindPanics(t *testing.T) { }() ArtifactByKind("unknown") } + +func TestGoServerDefCarriesTheBinaryOverride(t *testing.T) { + old := Config.GoServer.Binary + defer func() { Config.GoServer.Binary = old }() + Config.GoServer.Binary = "/tmp/dev-go-ssa-server" + + def := ArtifactByKind("goserver") + if def.Override != "/tmp/dev-go-ssa-server" { + t.Errorf("Override = %q, want the --go-server-binary value", def.Override) + } +} diff --git a/cli/internal/globals/global.go b/cli/internal/globals/global.go index 8bcba3f393..fb23d6b5aa 100644 --- a/cli/internal/globals/global.go +++ b/cli/internal/globals/global.go @@ -3,6 +3,7 @@ package globals import ( _ "embed" "fmt" + "runtime" "time" "gopkg.in/yaml.v2" @@ -15,6 +16,7 @@ type versions struct { Analyzer string `yaml:"analyzer"` Autobuilder string `yaml:"autobuilder"` Rules string `yaml:"rules"` + GoServer string `yaml:"go-server"` Java int `yaml:"java"` } @@ -30,6 +32,7 @@ var ( AnalyzerBindVersion = BindVersions.Analyzer AutobuilderBindVersion = BindVersions.Autobuilder RulesBindVersion = BindVersions.Rules + GoServerBindVersion = BindVersions.GoServer DefaultJavaVersion = BindVersions.Java ) @@ -40,6 +43,21 @@ const AutobuilderAssetName = "opentaint-project-auto-builder.jar" const AnalyzerAssetName = "opentaint-project-analyzer.jar" const RulesAssetName = "opentaint-rules.tar.gz" +// GoServerAssetNamePrefix is the per-platform go-ssa-server asset filename prefix. +const GoServerAssetNamePrefix = "go-ssa-server_" + +// GoServerAssetName returns the platform-specific name of the go-ssa-server +// release asset. The name follows the contract of the release pipeline: +// "go-ssa-server__", with ".exe" added on windows. It uses the +// raw runtime.GOOS and runtime.GOARCH values (no Adoptium-style mapping). +func GoServerAssetName() string { + name := fmt.Sprintf("%s%s_%s", GoServerAssetNamePrefix, runtime.GOOS, runtime.GOARCH) + if runtime.GOOS == "windows" { + name += ".exe" + } + return name +} + type Scan struct { Timeout time.Duration `mapstructure:"timeout"` MaxMemory string `mapstructure:"max_memory"` @@ -70,6 +88,11 @@ type Rules struct { Version string `mapstructure:"version"` } +type GoServer struct { + Version string `mapstructure:"version"` + Binary string `mapstructure:"binary"` +} + type Java struct { Version int `mapstructure:"version"` } @@ -82,6 +105,7 @@ type ConfigType struct { Analyzer Analyzer `mapstructure:"analyzer"` Autobuilder Autobuilder `mapstructure:"autobuilder"` Rules Rules `mapstructure:"rules"` + GoServer GoServer `mapstructure:"go-server"` Java Java `mapstructure:"java"` Owner string `mapstructure:"owner"` Repo string `mapstructure:"repo"` diff --git a/cli/internal/globals/versions.yaml b/cli/internal/globals/versions.yaml index 2a0cc00711..890f00c661 100644 --- a/cli/internal/globals/versions.yaml +++ b/cli/internal/globals/versions.yaml @@ -1,4 +1,5 @@ analyzer: "analyzer/2026.10.26.c624add" autobuilder: "autobuilder/2026.07.16.7a834a0" rules: "rules/v0.3.0" +go-server: "go-server/2026.06.19.cf7e218" java: 21 diff --git a/cli/internal/utils/display_version.go b/cli/internal/utils/display_version.go index a92dfa7d1b..1e5d02ab7f 100644 --- a/cli/internal/utils/display_version.go +++ b/cli/internal/utils/display_version.go @@ -13,7 +13,7 @@ func ArtifactDisplayVersion(def globals.ArtifactDef) string { } func ArtifactVersionWithPath(def globals.ArtifactDef) string { - return strings.TrimPrefix(ArtifactDisplayVersion(def), def.Kind()+"/") + return strings.TrimPrefix(ArtifactDisplayVersion(def), def.TagPrefix()) } func ArtifactVersion(def globals.ArtifactDef) string { @@ -21,7 +21,7 @@ func ArtifactVersion(def globals.ArtifactDef) string { if isCustomArtifact(def.Version, def.Override, tier, bundledRelease) { return "custom" } - return strings.TrimPrefix(def.Version, def.Kind()+"/") + return strings.TrimPrefix(def.Version, def.TagPrefix()) } func artifactResolution(def globals.ArtifactDef) (tier, path string, bundledRelease bool) { diff --git a/cli/internal/utils/java/runner.go b/cli/internal/utils/java/runner.go index 392fd72621..599f94bf6d 100644 --- a/cli/internal/utils/java/runner.go +++ b/cli/internal/utils/java/runner.go @@ -40,6 +40,12 @@ type JavaRunner interface { WithImageType(imageType AdoptiumImageType) JavaRunner WithSkipVerify(skipVerify bool) JavaRunner WithDebugOutput(writer DebugLineWriter) JavaRunner + // WithExtraEnv adds KEY=VALUE environment entries that are appended to the + // child process environment for every ExecuteJavaCommand call. Entries are + // applied regardless of the Java resolution strategy (including System) and + // override any inherited values with the same key. A nil or empty map is a + // no-op. Repeated calls merge their entries (later calls win per key). + WithExtraEnv(env map[string]string) JavaRunner GetJavaResolutions() []JavaResolution // EnsureJava resolves and downloads Java if needed, returning the path. // Call this before wrapping ExecuteJavaCommand in a spinner to avoid @@ -59,6 +65,7 @@ type javaRunner struct { skipVerify bool resolvedJavaPath string debugOutput DebugLineWriter + extraEnv map[string]string } type JavaResolution func() (string, ResolutionStrategy, error) @@ -205,6 +212,21 @@ func (j *javaRunner) executeWithJava(javaPath string, strategy ResolutionStrateg output.LogDebug("Using clean environment for managed Java version strategy") } + // Append any per-call extra environment variables. These apply regardless of + // the resolution strategy. On the System strategy cmd.Env is otherwise left + // nil (the child inherits the parent env), so seed it from os.Environ() first + // to avoid dropping inherited variables. Appended entries win over earlier + // ones with the same key (os/exec uses the last occurrence). + if len(j.extraEnv) > 0 { + if cmd.Env == nil { + cmd.Env = os.Environ() + } + for key, value := range j.extraEnv { + cmd.Env = append(cmd.Env, key+"="+value) + output.LogDebugf("Injecting %s into command environment", key) + } + } + output.LogDebugf("Executing Java command: %s %v (full: %s)", javaPath, args, strings.Join(cmdArgs, " ")) // Create pipes for stdout and stderr @@ -292,6 +314,19 @@ func (j *javaRunner) WithDebugOutput(writer DebugLineWriter) JavaRunner { return j } +func (j *javaRunner) WithExtraEnv(env map[string]string) JavaRunner { + if len(env) == 0 { + return j + } + if j.extraEnv == nil { + j.extraEnv = make(map[string]string, len(env)) + } + for key, value := range env { + j.extraEnv[key] = value + } + return j +} + // unsetJavaEnvironmentVariables unsets Java-related environment variables // to ensure a clean environment when using specific Java versions func unsetJavaEnvironmentVariables() { diff --git a/cli/internal/utils/java/runner_test.go b/cli/internal/utils/java/runner_test.go index fa0815d0a4..60ccdeb1b0 100644 --- a/cli/internal/utils/java/runner_test.go +++ b/cli/internal/utils/java/runner_test.go @@ -1,7 +1,9 @@ package java import ( + "fmt" "os" + "path/filepath" "strings" "testing" ) @@ -392,3 +394,158 @@ func TestEnvironmentVariableList(t *testing.T) { } _ = os.Unsetenv("NON_JAVA") } + +// --- WithExtraEnv: builder semantics --- + +func TestJavaRunner_WithExtraEnv_NilAndEmptyAreNoOps(t *testing.T) { + j := &javaRunner{} + + ret := j.WithExtraEnv(nil) + if jr, ok := ret.(*javaRunner); !ok || jr != j { + t.Fatalf("WithExtraEnv(nil) should return the same usable *javaRunner") + } + if j.extraEnv != nil { + t.Errorf("WithExtraEnv(nil) should not allocate extraEnv, got %v", j.extraEnv) + } + + ret = j.WithExtraEnv(map[string]string{}) + if jr, ok := ret.(*javaRunner); !ok || jr != j { + t.Fatalf("WithExtraEnv(empty) should return the same usable *javaRunner") + } + if j.extraEnv != nil { + t.Errorf("WithExtraEnv(empty) should not allocate extraEnv, got %v", j.extraEnv) + } +} + +func TestJavaRunner_WithExtraEnv_MergeLaterWins(t *testing.T) { + j := &javaRunner{} + j.WithExtraEnv(map[string]string{"A": "1", "B": "2"}) + j.WithExtraEnv(map[string]string{"B": "20", "C": "3"}) + + want := map[string]string{"A": "1", "B": "20", "C": "3"} + if len(j.extraEnv) != len(want) { + t.Fatalf("extraEnv = %v, want %v", j.extraEnv, want) + } + for k, v := range want { + if j.extraEnv[k] != v { + t.Errorf("extraEnv[%q] = %q, want %q (later WithExtraEnv call should win)", k, j.extraEnv[k], v) + } + } +} + +// --- executeWithJava: extra env reaches the child process --- +// +// These tests re-execute the test binary itself as the "java" process via +// executeWithJava and have the child (TestHelperProcess) dump its environment +// to a file, so we can assert what actually ended up in cmd.Env on each +// resolution strategy. No real JVM is required. + +// runHelperEnv runs executeWithJava with the test binary as the child process +// and returns the environment that child observed. The GO_WANT_HELPER_PROCESS +// gate is injected through the runner's own extra-env mechanism, so if extra +// env failed to reach the child the helper would never activate and the env +// file would be missing (surfaced as a clear failure here). +func runHelperEnv(t *testing.T, j *javaRunner, strategy ResolutionStrategy) map[string]string { + t.Helper() + + outFile := filepath.Join(t.TempDir(), "child-env.txt") + j.WithExtraEnv(map[string]string{"GO_WANT_HELPER_PROCESS": "1"}) + + args := []string{"-test.run=TestHelperProcess", "--", outFile} + cmdErr, err := j.executeWithJava(os.Args[0], strategy, args, func(error) bool { return true }) + if err != nil { + t.Fatalf("executeWithJava returned setup error: %v", err) + } + if cmdErr != nil { + t.Fatalf("helper process exited non-zero: %v", cmdErr) + } + + raw, err := os.ReadFile(outFile) + if err != nil { + t.Fatalf("reading child env file (extra env likely not propagated to child): %v", err) + } + + env := map[string]string{} + for _, line := range strings.Split(string(raw), "\n") { + if line == "" { + continue + } + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + env[parts[0]] = parts[1] + } + } + return env +} + +func TestJavaRunner_ExtraEnv_SystemStrategy_AppendsAndPreservesInherited(t *testing.T) { + // Set an inherited variable in the parent environment. On the System path + // cmd.Env starts nil and must be seeded from os.Environ() so inherited + // variables survive alongside the appended extra entry. + t.Setenv("OPENTAINT_INHERITED_SYS", "inherited-value") + + j := (&javaRunner{}).WithExtraEnv(map[string]string{ + "OPENTAINT_EXTRA_ENV_TEST": "extra-value", + }).(*javaRunner) + + env := runHelperEnv(t, j, System) + + if got := env["OPENTAINT_EXTRA_ENV_TEST"]; got != "extra-value" { + t.Errorf("child OPENTAINT_EXTRA_ENV_TEST = %q, want %q (extra env not appended on System path)", got, "extra-value") + } + if got := env["OPENTAINT_INHERITED_SYS"]; got != "inherited-value" { + t.Errorf("child OPENTAINT_INHERITED_SYS = %q, want %q (inherited env dropped; cmd.Env not seeded from os.Environ() on System path)", got, "inherited-value") + } +} + +func TestJavaRunner_ExtraEnv_SpecificStrategy_AppendsAfterCleanEnv(t *testing.T) { + // Specific strategy uses the clean environment (Java vars stripped) and then + // appends the extra entries. + t.Setenv("OPENTAINT_INHERITED_SPECIFIC", "kept-value") + t.Setenv("JAVA_HOME", "/fake/java/home") + + j := (&javaRunner{}).WithExtraEnv(map[string]string{ + "OPENTAINT_EXTRA_ENV_TEST": "extra-value", + }).(*javaRunner) + + env := runHelperEnv(t, j, Specific) + + if got := env["OPENTAINT_EXTRA_ENV_TEST"]; got != "extra-value" { + t.Errorf("child OPENTAINT_EXTRA_ENV_TEST = %q, want %q (extra env not appended on Specific path)", got, "extra-value") + } + if got := env["OPENTAINT_INHERITED_SPECIFIC"]; got != "kept-value" { + t.Errorf("child OPENTAINT_INHERITED_SPECIFIC = %q, want %q (non-Java inherited env should survive clean env)", got, "kept-value") + } + if _, present := env["JAVA_HOME"]; present { + t.Errorf("child JAVA_HOME should be stripped by clean environment on Specific path, but it was present") + } +} + +// TestHelperProcess is not a real test. It is re-executed as the child "java" +// process by the env-injection tests above. When the GO_WANT_HELPER_PROCESS +// gate (passed through the runner's extra env) is present it writes the child's +// full environment to the file named after "--" and exits; otherwise it is a +// no-op for normal test runs. +func TestHelperProcess(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + + args := os.Args + for len(args) > 0 { + if args[0] == "--" { + args = args[1:] + break + } + args = args[1:] + } + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "helper: missing output file argument") + os.Exit(2) + } + if err := os.WriteFile(args[0], []byte(strings.Join(os.Environ(), "\n")), 0o644); err != nil { + fmt.Fprintln(os.Stderr, "helper: write env:", err) + os.Exit(3) + } + os.Exit(0) +} diff --git a/cli/internal/utils/opentaint_command_builder.go b/cli/internal/utils/opentaint_command_builder.go index 1eb81a09c6..6117c91b80 100644 --- a/cli/internal/utils/opentaint_command_builder.go +++ b/cli/internal/utils/opentaint_command_builder.go @@ -184,7 +184,7 @@ func (cb *OpentaintCommandBuilder) WithRuleID(ruleIDs []string) *OpentaintComman func (cb *OpentaintCommandBuilder) WithPassthroughApproximations(paths []string) *OpentaintCommandBuilder { for _, p := range paths { if p != "" { - cb.arrayFlags["passthrough-approximations"] = append(cb.arrayFlags["passthrough-approximations"], p) + cb.arrayFlags["passthrough-models"] = append(cb.arrayFlags["passthrough-models"], p) } } return cb @@ -193,7 +193,7 @@ func (cb *OpentaintCommandBuilder) WithPassthroughApproximations(paths []string) func (cb *OpentaintCommandBuilder) WithDataflowApproximations(paths []string) *OpentaintCommandBuilder { for _, p := range paths { if p != "" { - cb.arrayFlags["dataflow-approximations"] = append(cb.arrayFlags["dataflow-approximations"], p) + cb.arrayFlags["java-models"] = append(cb.arrayFlags["java-models"], p) } } return cb diff --git a/cli/internal/utils/opentaint_home.go b/cli/internal/utils/opentaint_home.go index c397e10ce7..513069867e 100644 --- a/cli/internal/utils/opentaint_home.go +++ b/cli/internal/utils/opentaint_home.go @@ -210,3 +210,10 @@ func ResolveJarPath(def globals.ArtifactDef) (string, error) { func GetRulesPath(version string) (string, error) { return resolveArtifactPath(globals.ArtifactByKind("rules").WithVersion(version)) } + +// GetGoServerPath resolves the on-disk path of the go-ssa-server binary for the +// given version using the standard artifact tier resolution (bundled, install, cache). +// The returned path may not yet exist on disk if the binary has not been downloaded. +func GetGoServerPath(version string) (string, error) { + return resolveArtifactPath(globals.ArtifactByKind("goserver").WithVersion(version)) +} diff --git a/cli/internal/validation/source_project.go b/cli/internal/validation/source_project.go index c63ca863f2..4fdd9c48ea 100644 --- a/cli/internal/validation/source_project.go +++ b/cli/internal/validation/source_project.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" ) @@ -57,6 +58,10 @@ var supportedLanguages = []LanguageMarkers{ "gradlew.bat", }, }, + { + Name: "Go", + Markers: []string{"go.mod"}, + }, } // markerToLanguage builds a lookup from marker name to language name. @@ -186,3 +191,23 @@ func ValidateSourceProject(absProjectRoot string) error { func ValidateSourceProjectForCompile(absProjectRoot string) error { return validateSourceDir(absProjectRoot, false) } + +// DetectLanguages returns the human-readable names of the supported languages +// detected under absProjectRoot (e.g. "Go", "Java/Kotlin"), found by searching for +// the registered build-system markers up to maxMarkerSearchDepth. It is the +// exported wrapper over the unexported detectLanguages, so callers (e.g. the scan +// wiring) can distinguish Go-only from polyglot projects without duplicating the +// detection logic. Returns an empty slice when the directory does not exist or +// cannot be walked. +func DetectLanguages(absProjectRoot string) []string { + return detectLanguages(absProjectRoot) +} + +// IsGoProject reports whether absProjectRoot is detected as a Go project, i.e. a +// go.mod marker is found within the marker search depth. It reuses the same +// (unexported) detection logic as source validation, so it stays in sync with +// the supportedLanguages registry. Returns false when the directory does not +// exist or cannot be walked. +func IsGoProject(absProjectRoot string) bool { + return slices.Contains(detectLanguages(absProjectRoot), "Go") +} diff --git a/cli/internal/validation/source_project_test.go b/cli/internal/validation/source_project_test.go index b9ca84b0c6..acde379af8 100644 --- a/cli/internal/validation/source_project_test.go +++ b/cli/internal/validation/source_project_test.go @@ -300,3 +300,82 @@ func TestValidateSourceProjectForCompile_IgnoresProjectYaml(t *testing.T) { t.Errorf("unexpected error when build marker exists alongside project.yaml, got: %v", err) } } + +// --- IsGoProject --- + +func TestIsGoProject_GoMod(t *testing.T) { + dir := t.TempDir() + createFile(t, filepath.Join(dir, "go.mod")) + + if !IsGoProject(dir) { + t.Errorf("IsGoProject() = false, want true for dir containing go.mod") + } +} + +func TestIsGoProject_JavaKotlinMarkersOnly(t *testing.T) { + dir := t.TempDir() + createFile(t, filepath.Join(dir, "pom.xml")) + + if IsGoProject(dir) { + t.Errorf("IsGoProject() = true, want false for dir with only Java/Kotlin markers (pom.xml)") + } +} + +func TestIsGoProject_EmptyDir(t *testing.T) { + dir := t.TempDir() + + if IsGoProject(dir) { + t.Errorf("IsGoProject() = true, want false for empty dir") + } +} + +func TestIsGoProject_NonExistentPath(t *testing.T) { + missing := filepath.Join(t.TempDir(), "does-not-exist") + + if IsGoProject(missing) { + t.Errorf("IsGoProject() = true, want false for non-existent path") + } +} + +func TestIsGoProject_NestedGoMod(t *testing.T) { + // go.mod one level down — within maxMarkerSearchDepth, so it is detected, + // mirroring TestDetectLanguages_NestedMarker for the Java/Kotlin case. + dir := t.TempDir() + createFile(t, filepath.Join(dir, "service", "go.mod")) + + if !IsGoProject(dir) { + t.Errorf("IsGoProject() = false, want true for nested go.mod one level down") + } +} + +// --- DetectLanguages (exported) --- + +func TestDetectLanguages_PolyglotGoAndJava(t *testing.T) { + dir := t.TempDir() + createFile(t, filepath.Join(dir, "go.mod")) + createFile(t, filepath.Join(dir, "pom.xml")) + + got := DetectLanguages(dir) + if !slices.Contains(got, "Go") { + t.Errorf("DetectLanguages() = %v, want to contain %q for polyglot project", got, "Go") + } + if !slices.Contains(got, "Java/Kotlin") { + t.Errorf("DetectLanguages() = %v, want to contain %q for polyglot project", got, "Java/Kotlin") + } + if len(got) <= 1 { + t.Errorf("DetectLanguages() = %v, want more than one language for polyglot project", got) + } +} + +func TestDetectLanguages_JavaOnlyHasNoGo(t *testing.T) { + dir := t.TempDir() + createFile(t, filepath.Join(dir, "pom.xml")) + + got := DetectLanguages(dir) + if slices.Contains(got, "Go") { + t.Errorf("DetectLanguages() = %v, want not to contain %q for Java-only project", got, "Go") + } + if !slices.Contains(got, "Java/Kotlin") { + t.Errorf("DetectLanguages() = %v, want to contain %q for Java-only project", got, "Java/Kotlin") + } +} diff --git a/cli/scripts/bundle-go-server.sh b/cli/scripts/bundle-go-server.sh new file mode 100755 index 0000000000..fd33a9e8c5 --- /dev/null +++ b/cli/scripts/bundle-go-server.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Bundle the platform-specific go-ssa-server binary into the release archives. +# The script runs after GoReleaser. It puts the binary into the lib/ directory +# of the "opentaint" and "opentaint-full" archives, next to the analyzer, the +# autobuilder, and the rules. This is the analogue of bundle-jre.sh, which +# injects the platform-specific JREs. +# +# Usage: ./scripts/bundle-go-server.sh [github-repo] +# +# Prerequisites: gh (authenticated), tar, unzip, zip + +DIST_DIR="${1:?Usage: $0 [github-repo]}" +REPO="${2:-seqra/opentaint}" +VERSIONS_FILE="internal/globals/versions.yaml" + +GO_SERVER_TAG=$(grep '^go-server:' "$VERSIONS_FILE" | awk '{print $2}' | tr -d '"') +echo "go-ssa-server version: $GO_SERVER_TAG" + +# The release assets are named "go-ssa-server__", with ".exe" on +# windows. The platform list matches the GoReleaser build matrix. +PLATFORMS=(linux_amd64 linux_arm64 darwin_amd64 darwin_arm64 windows_amd64 windows_arm64) + +inject_tar_gz() { + local archive="$1" + local binary="$2" + local tmp_dir + + tmp_dir=$(mktemp -d) + trap 'rm -rf "$tmp_dir"' RETURN + + echo " Extracting archive..." + tar -xzf "$archive" -C "$tmp_dir" + + mkdir -p "$tmp_dir/lib" + cp "$binary" "$tmp_dir/lib/$(basename "$binary")" + # The binary must be executable when the archive is unpacked on unix. + chmod 0755 "$tmp_dir/lib/$(basename "$binary")" + + echo " Recompressing archive..." + tar -czf "$archive" -C "$tmp_dir" . +} + +inject_zip() { + local archive + archive="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" + local binary="$2" + local tmp_dir + + tmp_dir=$(mktemp -d) + trap 'rm -rf "$tmp_dir"' RETURN + + echo " Extracting archive..." + unzip -q "$archive" -d "$tmp_dir" + + mkdir -p "$tmp_dir/lib" + cp "$binary" "$tmp_dir/lib/$(basename "$binary")" + + echo " Recompressing archive..." + (cd "$tmp_dir" && zip -qr "$archive" .) +} + +for platform in "${PLATFORMS[@]}"; do + asset="go-ssa-server_${platform}" + if [[ "$platform" == windows_* ]]; then + asset="${asset}.exe" + archive_ext="zip" + else + archive_ext="tar.gz" + fi + + echo "Processing platform: $platform" + + asset_dir=$(mktemp -d) + echo " Downloading $asset..." + gh release download "$GO_SERVER_TAG" \ + --repo "$REPO" \ + --pattern "$asset" \ + --dir "$asset_dir" + + for variant in opentaint opentaint-full; do + archive_file="$DIST_DIR/${variant}_${platform}.${archive_ext}" + if [ ! -f "$archive_file" ]; then + echo " Archive not found: $archive_file, skipping" + continue + fi + if [ "$archive_ext" = "zip" ]; then + inject_zip "$archive_file" "$asset_dir/$asset" + else + inject_tar_gz "$archive_file" "$asset_dir/$asset" + fi + echo " Done: $archive_file" + done + + rm -rf "$asset_dir" +done + +echo "go-ssa-server bundling complete." diff --git a/docs/README.md b/docs/README.md index 760695b752..b5cd5c9e13 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,7 +37,7 @@ The more AI writes code, the more you need formal methods underneath. The engine runs IFDS-with-abduction — formal inter-procedural dataflow analysis. It tracks untrusted data from HTTP inputs to dangerous APIs across endpoints, persistence layers, object fields, aliased references, and async code. That includes multi-hop attack paths — cross-endpoint flows, stored injections, data through object fields and aliases — at monorepo scale. 100+ rules across 20+ vulnerability classes. -Models Spring data flow and the full Boot ecosystem, analyzing Java and Kotlin at bytecode level. More languages and frameworks ahead. +Models Spring data flow and the full Boot ecosystem, analyzing Java and Kotlin at bytecode level and Go on SSA form. More languages and frameworks ahead. ### One finding becomes total coverage @@ -110,7 +110,7 @@ Each finding includes the HTTP endpoint, making it easy to map your application' | **Docker** | See [Docker docs](docker.md) | | **Binary** | [Download from releases](https://github.com/seqra/opentaint/releases/latest) | -Release archives come in three variants: **`opentaint-full`** (binary + JARs + rules + JRE), **`opentaint`** (binary + JARs + rules), and **`opentaint-cli`** (binary only). Homebrew and install scripts default to `full`. +Release archives come in three variants: **`opentaint-full`** (binary + JARs + rules + go-ssa-server + JRE), **`opentaint`** (binary + JARs + rules + go-ssa-server), and **`opentaint-cli`** (binary only). Homebrew and install scripts default to `full`. For detailed instructions, see [Installation Guide](installation.md). @@ -128,11 +128,11 @@ opentaint summary --show-findings --verbose-flow --show-code-snippets results.sa | Command | Description | |---------|-------------| -| `opentaint scan` | Analyze projects (auto-detects Maven/Gradle) | +| `opentaint scan` | Analyze projects (auto-detects the build system) | | `opentaint compile` | Build project model separately | | `opentaint project` | Create model from precompiled JARs | | `opentaint summary` | View SARIF results | -| `opentaint health` | Show resolved analyzer, autobuilder, rules, and runtime paths | +| `opentaint health` | Show dependency paths and report missing components | | `opentaint test rule` | Scaffold, test, and debug detection rules | | `opentaint test approximation` | Scaffold and test dataflow approximations | | `opentaint pull` | Download dependencies | diff --git a/docs/installation.md b/docs/installation.md index c1e400cc8f..3158303a29 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,6 +1,6 @@ # Installation -**Prerequisites:** Same build requirements as your Java/Kotlin project (Maven or Gradle, project dependencies). Java runtime is bundled with release archives. +**Prerequisites:** Same build requirements as your project (Maven or Gradle for Java/Kotlin, the Go toolchain for Go modules). Java runtime is bundled with release archives. ## Homebrew (Linux/macOS) @@ -104,8 +104,8 @@ Each release provides three archive variants: | Variant | Contents | Use case | |---------|----------|----------| -| **`opentaint-full`** | Binary + JARs + rules + JRE | Recommended — everything included, no additional downloads | -| **`opentaint`** | Binary + JARs + rules | Use your own JRE (Java 17+) | +| **`opentaint-full`** | Binary + JARs + rules + go-ssa-server + JRE | Recommended — everything included, no additional downloads | +| **`opentaint`** | Binary + JARs + rules + go-ssa-server | Use your own JRE (Java 17+) | | **`opentaint-cli`** | Binary only | Minimal install; run `opentaint pull` to download components | Download `opentaint-full` for your platform (recommended): @@ -119,7 +119,7 @@ Download `opentaint-full` for your platform (recommended): | Windows x64 | [opentaint-full_windows_amd64.zip](https://github.com/seqra/opentaint/releases/latest/download/opentaint-full_windows_amd64.zip) | | Windows ARM64 | [opentaint-full_windows_arm64.zip](https://github.com/seqra/opentaint/releases/latest/download/opentaint-full_windows_arm64.zip) | -The `opentaint-full` archives include bundled JARs, rules, and JRE — no additional downloads needed. Replace `opentaint-full` with `opentaint` or `opentaint-cli` in the URLs above for other variants. +The `opentaint-full` archives include bundled JARs, rules, the go-ssa-server, and a JRE — no additional downloads needed. Replace `opentaint-full` with `opentaint` or `opentaint-cli` in the URLs above for other variants. ### Linux/macOS Installation @@ -150,7 +150,7 @@ For package manager installations, `opentaint update` will show the appropriate ## Cleaning Up -Remove stale downloaded artifacts: +Remove old downloaded artifacts: ```bash opentaint prune # Interactive confirmation diff --git a/docs/usage.md b/docs/usage.md index eb2d776d2b..e8a3636572 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -78,21 +78,25 @@ Use [CodeChecker](https://github.com/Ericsson/codechecker) for advanced result m | Command | Description | |---------|-------------| -| `opentaint scan` | Analyze projects (auto-detects Maven/Gradle, builds, and scans) | +| `opentaint scan` | Analyze projects (auto-detects the build system, builds, and scans) | | `opentaint compile` | Build project model separately from scanning | | `opentaint project` | Create project model from precompiled JARs/classes | | `opentaint summary` | View SARIF analysis results | -| `opentaint health` | Show resolved paths for the analyzer, autobuilder, rules, and Java runtime | +| `opentaint health` | Show dependency paths and report missing components | | `opentaint test rule` | Create, run, and debug detection-rule tests | -| `opentaint test approximation` | Create and run dataflow-approximation tests | +| `opentaint test dataflow-model` | Create and run dataflow model tests | | `opentaint approximation` | Create dataflow approximation projects | -| `opentaint pull` | Download analyzer dependencies | + +`test approximation` remains accepted as a deprecated compatibility alias for `test dataflow-model`. +| `opentaint pull` | Download the analysis toolchain and Java runtime | | `opentaint update` | Update to latest version | -| `opentaint prune` | Remove stale downloaded artifacts and cached models | +| `opentaint prune` | Remove old downloaded artifacts and cached models | ### opentaint scan -Automatically detects Maven/Gradle projects, builds them, and performs security analysis. The source path defaults to the current directory when omitted. +Automatically detects the project's build system (Maven, Gradle, or go.mod), builds the project, and runs taint analysis over the result. The source path defaults to the current directory when omitted. + +Go projects need the `go` command on `PATH`. The first Go scan downloads the `go-ssa-server` component. A project can mix languages. If `go` is missing in a mixed project, the scan skips the Go code and shows a warning. On the first run, the compiled project model is cached in `~/.opentaint/cache/`. Subsequent scans of the same project reuse the cached model, skipping compilation entirely. @@ -101,24 +105,34 @@ On the first run, the compiled project model is cached in `~/.opentaint/cache/`. | `--output`, `-o` | Path to the SARIF report (default: `/sources/opentaint.sarif`) | | `--recompile` | Force recompilation even if a cached project model exists | | `--project-model` | Path to a pre-compiled project model (skips compilation) | -| `--timeout`, `-t` | Timeout for analysis (default: `15m`) | -| `--max-memory` | Maximum memory for the analyzer (default: `8G`) | -| `--severity` | Severity levels to report (default: `warning`, `error`) | -| `--ruleset` | YAML rules file or directory (default: `builtin`) | +| `--timeout`, `-t` | Maximum wall-clock time for analysis (default: `15m`) | +| `--max-memory` | Maximum analyzer heap size (default: `8G`) | +| `--severity` | Run only rules at these severity levels: `note`, `warning`, `error` (default: `warning`, `error`) | +| `--ruleset` | Rules to run: a YAML file, a directory of rules files, or `builtin` (default: `builtin`) | | `--dry-run` | Validate inputs and show what would run without compiling or scanning | | `--log-file` | Path to the log file (default: `/logs/.log`) | #### Rule-authoring flags -These flags are to work with custom approximations: +These flags are for working with custom models: | Flag | Description | |------|-------------| | `--track-external-methods` | Write external-method coverage files next to the SARIF report | -| `--passthrough-approximations` | Apply pass-through approximation YAML files or directories (repeatable) | -| `--dataflow-approximations` | Apply dataflow approximation projects, build outputs, or class directories (repeatable) | +| `--passthrough-models` | Apply pass-through model YAML files or directories (repeatable) | +| `--java-models` | Apply Java dataflow model projects, build outputs, or class directories (repeatable) | + +
+Deprecated flag names + +The old names remain accepted for compatibility and are deprecated: -Use external-method tracking when a scan may miss flows through library methods. The dropped-methods file shows where taint was killed because no model was available; the approximated-methods file shows methods already covered by built-in or custom models. +- `--passthrough-approximations` → `--passthrough-models` +- `--dataflow-approximations` → `--java-models` + +
+ +Use external-method tracking when a scan may miss flows through library methods. The dropped-methods file shows where taint was killed because no model was available. The modeled-methods file shows methods already covered by built-in or custom models. ### opentaint health @@ -130,18 +144,19 @@ opentaint health --rules opentaint health --analyzer ``` -With no flags, `health` shows the autobuilder, analyzer, built-in rules, and Java runtime. With a single component flag, it prints only the bare path, which is useful for scripts. +With no flags, `health` shows the autobuilder, analyzer, built-in rules, go-ssa-server, and Java runtime, and reports whether each is present. With a single component flag, it prints only the bare path, which is useful for scripts. The command exits non-zero when a selected component is missing. Fetch missing components with `opentaint pull`. | Flag | Description | |------|-------------| | `--autobuilder` | Print only the autobuilder JAR path | | `--analyzer` | Print only the analyzer JAR path | | `--rules` | Print only the built-in rules path, downloading rules if needed | +| `--go-server` | Print only the go-ssa-server binary path | | `--runtime` | Print only the Java runtime path | ### opentaint test -The `test` command group is tooling for rule and approximation development. +The `test` command group is tooling for rule and dataflow model development. #### Rule tests @@ -156,28 +171,32 @@ opentaint test rule reachability java/security/my-rule.yaml:my-rule --project-mo |---------|-------------| | `opentaint test rule init ` | Create source and sink test projects with annotated sample support | | `opentaint test rule run ` | Run detection-rule tests on a compiled project model | -| `opentaint test rule reachability [source-path]` | Trace why a rule can or cannot reach its facts | +| `opentaint test rule reachability [source-path]` | Show why a rule does or does not fire | + +Rule-test options include `--ruleset`, `--rule-id`, `--project-model`, `--entry-points`, `--output`, `--timeout`, `--max-memory`, and `--dry-run`. +Rule runs also accept `--java-models` and the deprecated `--dataflow-approximations` alias. Reachability accepts `--project-model` or a source path, but not both. -#### Approximation tests +#### Java dataflow model tests ```bash -opentaint test approximation init .opentaint/test-projects/my-approximation \ +opentaint test dataflow-model init .opentaint/test-projects/my-model \ --dependency "io.projectreactor:reactor-core:3.8.5" -opentaint compile .opentaint/test-projects/my-approximation -o .opentaint/test-compiled/my-approximation -opentaint test approximation run .opentaint/test-compiled/my-approximation \ - --dataflow-approximations .opentaint/dataflow/my-approximation +opentaint compile .opentaint/test-projects/my-model -o .opentaint/test-compiled/my-model +opentaint test dataflow-model run .opentaint/test-compiled/my-model \ + --java-models .opentaint/dataflow/my-model ``` | Command | Description | |---------|-------------| -| `opentaint test approximation init ` | Create a test project with a fixed `Taint.source()` to `Taint.sink(...)` harness | -| `opentaint test approximation run ` | Run dataflow approximation tests on a compiled project model | +| `opentaint test dataflow-model init ` | Create a test project with a fixed `Taint.source()` to `Taint.sink(...)` harness | +| `opentaint test dataflow-model run ` | Run dataflow model tests on a compiled project model | -Rule and approximation test runs write `test-result.json` and `test-results.sarif` to the selected output directory. +Dataflow-model-test options include `--java-models`, `--output`, `--timeout`, `--max-memory`, and `--dry-run`. The deprecated `test approximation` command and `--dataflow-approximations` flag remain accepted. +Rule and model test runs write `test-result.json` and `test-results.sarif` to the selected output directory. ### opentaint compile -Compiles Java and Kotlin projects and generates project models for analysis. Useful when you want to separate compilation from scanning or need to inspect the project model. +Compiles Java, Kotlin, and Go projects and generates project models for analysis. Useful when you want to separate compilation from scanning or need to inspect the project model. ```bash opentaint compile --output ./my-project-model /path/to/project @@ -186,7 +205,7 @@ opentaint scan --project-model ./my-project-model | Flag | Description | |------|-------------| -| `--output`, `-o` | Path to the result project model (required) | +| `--output`, `-o` | Path to the project model directory to create (required, must not exist) | | `--dry-run` | Validate inputs and show what would run without compiling | | `--log-file` | Path to the log file (default: `/logs/.log`) | @@ -200,12 +219,12 @@ opentaint approximation init .opentaint/dataflow/my-batch \ # write the @Approximate classes under src/main/java/, then apply them opentaint scan --project-model ./my-project-model \ - --dataflow-approximations .opentaint/dataflow/my-batch + --java-models .opentaint/dataflow/my-batch ``` Pin the versions the target application uses. Those pins are the models' compile environment: they, and nothing about the project under analysis, decide what the models compile against, so a model compiles identically wherever it is applied. -`--dataflow-approximations` accepts an approximation project, a build output, or a directory of compiled classes — and a directory holding any of those, so a tree with one project per batch can be passed as a single flag. A directory counts as compiled classes only when nothing below it still needs building; one that holds compiled classes of its own alongside a project is reported rather than guessed at. Paths may overlap, and the same models may be reached twice, but two of them may not offer the same class with different content — which copy an analysis would read is not defined. Projects are built on demand and rebuilt when their sources, their dependency pins, or the compiler change. +`--java-models` accepts an approximation project, a build output, or a directory of compiled classes — and a directory holding any of those, so a tree with one project per batch can be passed as a single flag. A directory counts as compiled classes only when nothing below it still needs building; one that holds compiled classes of its own alongside a project is reported rather than guessed at. Paths may overlap, and the same models may be reached twice, but two of them may not offer the same class with different content — which copy an analysis would read is not defined. Projects are built on demand and rebuilt when their sources, their dependency pins, or the compiler change. | Command | Description | |---------|-------------| @@ -221,14 +240,14 @@ Compiling ahead of time is optional — scanning and testing build the project t ### opentaint summary -View findings from a SARIF report. By default it prints the Scan Summary; add +View findings from a SARIF report. By default it prints the Scan Summary. Add `--show-findings` for the detailed listing. The filter flags below narrow the -whole summary (both the counts and the listing); `Rules executed` always +whole summary (both the counts and the listing). `Rules executed` always reflects the full set the tool ran. | Flag | Description | |------|-------------| -| `--show-findings` | Show all findings | +| `--show-findings` | Show every finding in the SARIF report | | `--show-code-snippets` | Show code snippets for each finding | | `--verbose-flow` | Show full code flow steps for each finding | | `--path` | Show only findings whose file path matches this glob (`**` supported, repeatable) | @@ -255,14 +274,26 @@ opentaint scan --project-model ./project-model | Flag | Description | |------|-------------| -| `--output`, `-o` | Output directory for project.yaml (required) | -| `--source-root` | Source root directory (required) | -| `--classpath` | Classpath entries — classes or JAR files (required) | -| `--package` | Project packages (required) | -| `--dependency` | Project dependencies — JAR files | -| `--dry-run` | Validate inputs and show what would run without generating project model | +| `--output`, `-o` | Directory to write the generated project model (required, must not exist) | +| `--source-root` | Path to the project source root (required) | +| `--classpath` | Classpath entries: compiled classes directories or JAR files (required, repeatable) | +| `--package` | Packages to include in the generated model (required, repeatable) | +| `--dependency` | Additional dependency JAR files on the compile classpath (repeatable) | +| `--dry-run` | Validate inputs and show what would run without generating the project model | | `--log-file` | Path to the log file (default: `/logs/.log`) | +### opentaint pull, update, and prune + +`opentaint pull` downloads the analyzer, autobuilder, built-in rules, and Java runtime. Use it before the first scan or to restore components removed by pruning. + +`opentaint update [version]` updates the OpenTaint binary. `--check` checks for an available update without installing it, and `--yes` skips confirmation. Homebrew and npm installations are updated with their package managers. + +`opentaint prune` removes old cached artifacts and models. By default it removes old artifacts, rules, JDK/JRE versions, and cached models. Use `--logs` or `--install` to include those categories, `--all` to select every category, `--dry-run` to preview deletion, and `--yes` to skip confirmation. `--all` cannot be combined with a category flag. + +### opentaint health + +`opentaint health` reports the autobuilder, analyzer, built-in rules, and Java runtime. Use `--autobuilder`, `--analyzer`, `--rules`, or `--runtime` to select one component. A selected missing component causes a non-zero exit status. + ## Model Caching When `opentaint scan` compiles a project, the resulting project model is cached in `~/.opentaint/cache/`. The cache directory name is derived from the project path (e.g. `my-project-a1b2c3d4`). @@ -298,6 +329,6 @@ These options apply to all commands: - `--java-version int` — Java version for analyzer (default: 21) - `--quiet` / `-q` — Suppress interactive output (spinners, progress bars, JAR streaming) - `--debug` / `-d` — Enable debug output (stream JAR subprocess output, show debug fields) -- `--color string` — Color mode (`auto`, `always`, `never`); defaults to `auto` (detects terminal) +- `--color string` — Color mode (`auto`, `always`, `never`), defaults to `auto` (detects terminal) For persistent configuration using files or environment variables, see the [Configuration](configuration.md) documentation. diff --git a/skills-templates/analyze-external-methods/references/go.md.j2 b/skills-templates/analyze-external-methods/references/go.md.j2 new file mode 100644 index 0000000000..ef7d272bab --- /dev/null +++ b/skills-templates/analyze-external-methods/references/go.md.j2 @@ -0,0 +1,45 @@ +# analyze-external-methods — Go + +## Workflow + +### 1. Classify propagation + +**Read the plan entry first.** A Go dropped record carries the qualified callee in `method` only; its `signature` is literally `args:` — the parameter count at the call site (it varies only for variadic calls) and holds no package, type, receiver, or name. Never read a package or type out of `signature`. + +`method` comes in two shapes: + +- plain function — `.`, e.g. `net/url.ParseQuery`, `net/http.Get`. Split at the last `.`: everything before is the import path (it may hold `/` and `.`, as in `github.com/foo/bar`), the trailing segment is the name; no receiver +- method — `().`, e.g. `(*net/http.Request).FormValue`. The parenthesized part is the receiver type; strip a leading `*`, then split at the last `.` for the import path and the type. The segment after `)` is the method name + +That decomposition is what the build stage's `function:` matcher consumes: + +```text +net/url.ParseQuery → package: net/url, name: ParseQuery, receiver: false +(*net/http.Request).FormValue → package: net/http, type: Request, name: FormValue, receiver: true +``` + +**Getting the source.** An application-internal function sits in the project's own sources under the project root — read it directly. For a dependency, the source is source, not bytecode: it sits in the module cache (`$(go env GOMODCACHE)/@/…`) or in `vendor//` when the project vendors. Standard-library code is under `$(go env GOROOT)/src//`. `go doc .` and `go doc -src .` resolve a name and print its declaration without hunting for the file. + +**Go has no dataflow approximation.** The `dataflow` bucket stays empty and `dependencies` stays `[]` — there is no dataflow builder, no test project, and no `build.test_project` entries for Go. A carrier whose data travels through a function value, a callback, or a goroutine/channel chain is still classified `passthrough`, modeled as the closest position-to-position copy, and only becomes an `engine_issues` entry the usual way: after it is built and the method still drops. Never classify it `dataflow` in the hope of a second approximation kind — there isn't one. + +passthrough examples — the data is copied from one position to another: + +- `strings.ToLower`/`#Join`/`#TrimSpace`/`#Replace`, `fmt.Sprintf`, `[]byte(…)` / `string(…)` conversions +- `strings.Builder#WriteString` and `bytes.Buffer#Write`/`#String`, `io.Copy`, `bufio.Scanner#Text` +- request accessors — `(*net/http.Request).FormValue`/`#PostFormValue`, `net/url.Values#Get`, `net/url.ParseQuery` +- encoding round trips that keep the data — `encoding/json.Marshal`/`Unmarshal`, `net/url.QueryEscape`/`#QueryUnescape`, `encoding/base64` encode/decode +- external key-value stores — a `Set` paired with a `Get` carries the value across the round trip; model both ends + +skipped examples — the method carries the data nowhere: + +- predicates and inspectors returning a bool or a number — `strings.Contains`/`#HasPrefix`/`#EqualFold`, `len`, `bytes.Equal` +- conversions that collapse the data — `strconv.Atoi`, `strconv.ParseInt`, one-way hashes +- logging and metrics that keep none of the data downstream — `log.Printf`, `log.Println` + +### 2. Classify sinks (deep run only) + +Unchanged in substance; the record's `method` is the Go qualified name above, and each sink entry's `signature` is the plan's `args:` string — Go has no descriptor, so overloads never need distinguishing (Go has none). Go sink shapes worth recognizing: `database/sql.DB#Query`/`#Exec` (SQL), `os/exec.Command`/`#CommandContext` (OS command), `os.Open`/`os.ReadFile`/`path/filepath.Join` (path traversal), `net/http.Get`/`#Post`/`Client#Do` (SSRF), `html/template` vs `text/template` `Execute` (XSS — `text/template` does not escape), `encoding/gob.Decoder#Decode` (deserialization). + +### 3–4. Verify coverage, re-verify the skips + +Unchanged — `check-coverage.py --batch ` is language-independent. diff --git a/skills-templates/appsec-agent/sections/setup.md b/skills-templates/appsec-agent/sections/setup.md index b884117c5b..b01e1e7fc3 100644 --- a/skills-templates/appsec-agent/sections/setup.md +++ b/skills-templates/appsec-agent/sections/setup.md @@ -15,6 +15,8 @@ This workflow requires two subagent levels: MAIN → stage orchestrator → leaf Read the project's build files to fix the target language — Maven/Gradle → java, `go.mod` → go, and so on. Record it at bootstrap; stage orchestrators pass it to language-coupled leaves. +For `go`, also confirm `go` is on PATH (`command -v go`) before bootstrapping — the analyzer drives the Go toolchain at both build and scan time, so a Go run hard-fails without it. Tell the user and stop if it's missing. Two pipeline branches are Java-only and simply don't run for Go: dataflow approximations (Go has only passThrough) and everything that depends on them. + ### 4. Choose the workflow Ask the user for both levels together: diff --git a/skills-templates/build-project/references/go.md.j2 b/skills-templates/build-project/references/go.md.j2 new file mode 100644 index 0000000000..97a960887a --- /dev/null +++ b/skills-templates/build-project/references/go.md.j2 @@ -0,0 +1,65 @@ +# build-project — Go specific instructions + +## Workflow + +### 2. Identify the build + +- `go.mod` at or under `` (within a few levels) → Go module +- the `go` toolchain must be on PATH (`command -v go`). The analyzer drives it through go-ssa-server at scan time, so a Go project cannot be analyzed without it — if `go` is missing, say so and stop rather than producing a half-model +- the installed `go` must satisfy the module's own `go`/`toolchain` directive: a project pinning a newer toolchain either auto-downloads it (needs network) or fails under `GOTOOLCHAIN=local` +- there is no classpath mode. `opentaint scan ` also compiles Go sources on the fly, so a model is a convenience for reusing a fixed project, not a precondition for scanning + +### 3. Enable all modules + +A Go repo has no disabled-module concept — nothing to re-enable. A multi-module repo yields one `goProjects` entry per module; confirm the autobuilder found them all. + +### 4. Autobuilder + +The same `opentaint compile` that builds JVM projects builds Go — it detects `go.mod` and emits a `goProjects` model. No `JAVA_HOME` and no toolchain hint apply: + +```bash +opentaint compile -o .opentaint/project +``` + +The autobuilder writes a portable model: it copies each module's sources into `go_/` inside the model directory and records a relative `projectDir` (no top-level `projectRoot`), so the copied tree travels with the model: + +```yaml +goProjects: +- projectDir: go_0 +``` + +The model is multi-language — one `project.yaml` can hold both `goProjects:` and `javaProjects:` for a mixed repository. + +On failure `opentaint compile` prints the log-file path ("For full details, check the log file:"); read it and fix the real build with the project's own commands before re-running: + +```bash +go build ./... +go mod download +``` + +### 5. Manual model — last resort + +A Go model is minimal, so the fallback is to write `.opentaint/project/project.yaml` by hand, one entry per module, each an absolute path (a relative `projectDir` resolves against the model directory, not the current directory): + +```yaml +goProjects: +- projectDir: +``` + +There is no `--package` scoping step: the Go model's scope is the module itself, and `opentaint project` is a JVM-only path. Alternatively skip the model and point the scan straight at the sources. + +### 6. Verify + +`project.yaml` exists, is non-empty, and lists one `goProjects.projectDir` per module the repo declares in a `go.mod`. + +## Constraints + +- `go` on PATH is required for both building and scanning; the bundled Java runtime does not cover it +- `opentaint pull` pre-provisions the analyzer, autobuilder, built-in rules, go-ssa-server, and bundled Java — it does not install `go` + +## Gotchas + +- `go` not on PATH → a Go project hard-fails at scan time; install the toolchain before building or scanning +- Unresolvable dependencies → analysis loads the module graph through the `go` toolchain (go-ssa-server calls `packages.Load` with imports and deps). A dependency that fails to resolve is logged as a warning and its types and flows go missing — findings through it are silently lost, and the scan only hard-fails when the `go` invocation itself errors. Run `go mod download` or vendor the deps, ensure network access, and set `GOPRIVATE` plus credentials for private modules +- A hand-written model does not repair a broken build — `opentaint scan` recompiles from source and hits the same error +- Missing dependencies from submodules → `git submodule update --init` diff --git a/skills-templates/create-dataflow-approximation/references/java.md.j2 b/skills-templates/create-dataflow-approximation/references/java.md.j2 index 125fabdef3..b42ff19338 100644 --- a/skills-templates/create-dataflow-approximation/references/java.md.j2 +++ b/skills-templates/create-dataflow-approximation/references/java.md.j2 @@ -74,7 +74,7 @@ Run `test approximation run` over the compiled test project applying this batch' ```bash opentaint test approximation run .opentaint/test-compiled/ \ -o .opentaint/test-results/ \ - --dataflow-approximations .opentaint/dataflow/ + --java-models .opentaint/dataflow/ ``` `test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI builds the approximation project first. if compilation fails it reports the errors and aborts before the tests. A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: diff --git a/skills-templates/create-pass-through-approximation/references/go.md.j2 b/skills-templates/create-pass-through-approximation/references/go.md.j2 new file mode 100644 index 0000000000..9cf1336dcb --- /dev/null +++ b/skills-templates/create-pass-through-approximation/references/go.md.j2 @@ -0,0 +1,140 @@ +# create-pass-through-approximation — Go + +Go has only this one approximation kind — there is no dataflow approximation to fall back on. + +## Workflow + +### 1. Read the method's source + +Go dependencies ship as source, not bytecode. An application-internal function sits in the project's own sources under the project root. A dependency's source is in the module cache (`$(go env GOMODCACHE)/@/…`) or in `vendor//` when the project vendors; standard-library code is under `$(go env GOROOT)/src//`. `go doc .` and `go doc -src .` resolve a name without hunting for the file. + +The plan's `method` names the callee — `.` for a plain function, `().` for a method — and its `signature` is only `args:`; the `function:` matcher below is built from the `method` string, never from `signature`. + +### 2. Write the config + +Each file opens with a `language: go` header line above `passThrough:` — without it, or with a mismatched one, the loader silently skips the whole file: no load error, and every function in it stays dropped. Below it, a config is one top-level `passThrough` list, one entry per `function` with its `copy` edges. + +Positions (the base of any `from`/`to`) +- `this` (the receiver, methods only), `result`, `arg(0)`, `arg(1)`, … +- `result(N)` — Go returns multiple values; address them by index, `result(0)`, `result(1)` +- `[*]` — slice/array element; as a list item it must be quoted (`- '[*]'`), since unquoted it parses as a YAML alias and fails to load + +Access-path modifiers (list form `[, , …]`) +- A list is ALWAYS `[, , …]` — element 0 is the base position, every later element is a modifier. It is never a list of destinations: to send one source to several destinations, write one `copy` edge per destination +- `..#` — a struct field or a virtual slot. The slot name is nominal — the engine never resolves it, so it need not be a real field +- `` — a container's element slot; `` and `` are accepted aliases and all three collapse to the same single accessor (key and value are not distinguished) +- `` — a pointer's pointee +- there is no `.*` all-fields wildcard — always name an explicit field or slot + +Function matching — the structured block only: + +```yaml +function: + package: net/http # the import path + type: Request # the receiver type, methods only, leading '*' stripped + name: FormValue + receiver: true # false for a package-level function +``` + +Go has no overloads, so there is no `signature` key and no overload selection. There are no `overrides` and **no `condition:` support** — the loader parses only `function`/`copy`/`from`/`to` and the position modifiers, so `condition`, `typeIs`, `isConstant` and friends do not load. + +Simplest form — a direct copy from one position to another: + +```yaml +language: go +passThrough: +- function: + package: net + name: JoinHostPort + receiver: false + copy: + - from: arg(0) + to: result + - from: arg(1) + to: result +- function: + package: net + type: IPConn + name: ReadFromIP + receiver: true + copy: + - from: this + to: arg(0) +``` + +Multiple return values — one edge per tainted return position: + +```yaml +language: go +passThrough: +- function: + package: net + name: SplitHostPort + receiver: false + copy: + - from: arg(0) + to: result(0) + - from: arg(0) + to: result(1) +``` + +Virtual slot — when data lives in a struct between calls, the writer and the reader route through an access path. Both must name the **identical** `package.Type#field` accessor, or the taint drops: + +```yaml +language: go +passThrough: +- function: + package: container/list + type: List + name: PushBack + receiver: true + copy: + - from: arg(0) + to: + - this + - .container/list.List# +- function: + package: container/list + type: List + name: Front + receiver: true + copy: + - from: + - this + - .container/list.List# + to: + - result + - .container/list.Element#Value +``` + +### 3. Re-check your configs + +Beyond the body's checks, confirm the file still opens with `language: go` and that each matcher's `package`/`type`/`name`/`receiver` matches the `method` string it came from. + +## Verification is the scan + +Only when this skill runs standalone — never under the orchestrator, whose scan phase already verifies the config — a compiled model on hand catches YAML load/parse errors early. It proves the file loads, nothing about propagation: + +```bash +opentaint scan --project-model \ + -o .opentaint/test-results//passthrough-loadcheck.sarif \ + --ruleset builtin \ + --passthrough-models +``` + +A config error aborts the scan with the parse/load message. + + +There is no test project for a passThrough. The project scan applies the config and the scan agent reports back; the failure shapes map to distinct causes: + +- *every* function in the file still dropped, with no load error → the whole file was silently skipped: check the `language: go` header +- a *single* function still dropped → the `function` matcher didn't match (check `package`, `type`, `name`, `receiver`), or `from`/`to` doesn't land on the tainted position +- the method is no longer dropped but the flow still doesn't surface → most often a broken channel: writer and reader naming different `package.Type#field` accessors +- a load / parse error → fix the YAML + +After ~2 fix rounds with matcher fields and accessors verified and the method no longer dropped, stop guessing: Go has no dataflow fallback, so a propagation a passThrough genuinely can't express is an engine issue — report non-convergence for escalation to debug-rule / report-analyzer-issue. + +## Gotchas + +- User passThroughs **EXTEND** (concatenate with) Go's built-in configs — both your rule and any built-in matching the same function apply; there is no override. Check `approximated-external-methods.yaml` before modeling so you don't redundantly re-model something already covered +- The engine is field-sensitive: route data field-to-field as the source does rather than tainting the whole struct diff --git a/skills-templates/create-rule/references/debugging.md.j2 b/skills-templates/create-rule/references/debugging.md.j2 index dd8fce4a05..bd27742644 100644 --- a/skills-templates/create-rule/references/debugging.md.j2 +++ b/skills-templates/create-rule/references/debugging.md.j2 @@ -8,7 +8,7 @@ When a positive won't pass and the suspicion is a library method on its flow dro opentaint scan --project-model .opentaint/test-compiled// \ -o .opentaint/test-results///diag.sarif \ --ruleset builtin --ruleset .opentaint/rules --ruleset .opentaint/test-projects///test-rules \ - --passthrough-approximations .opentaint/pass-through \ + --passthrough-models .opentaint/pass-through \ --track-external-methods ``` diff --git a/skills-templates/create-rule/references/go-semgrep-examples.md.j2 b/skills-templates/create-rule/references/go-semgrep-examples.md.j2 new file mode 100644 index 0000000000..93f19cc11f --- /dev/null +++ b/skills-templates/create-rule/references/go-semgrep-examples.md.j2 @@ -0,0 +1,238 @@ +# Go Semgrep rule examples + +A catalog of real, working Go rules trimmed from `rules/ruleset/go/`. Each entry shows the +shape to copy when authoring a new Go rule. Use these as the concrete starting point instead +of inventing patterns. Produced rules stay comment-free — the prose here is reference only. + +## Go pattern syntax cheat-sheet + +- Typed receiver / variable binding: `($R : *http.Request)` — binds `$R` to a value of the + given Go type (note the pointer `*`). Use it to pin a method/field to its receiver type. +- Metavariables: `$UNTRUSTED`, `$SQL`, `$NAME`, `$P`, `$K`, … — `$UPPER` names. +- Method / field access: `$R.Method($A)`, `$R.Field`, index `$R.Header[$K]`. +- Variadic tail: `...` matches the rest of the args, e.g. `exec.Command($NAME, ...)`. +- Lib source rules: `mode: taint`, `options: {lib: true}`, `pattern-sources` with + `label: "$UNTRUSTED"`. +- Lib sink rules: `mode: taint`, `options: {lib: true}`, `pattern-sinks`; use + `focus-metavariable:` to point at the tainted argument. +- Package guard: wrap the source/sink patterns in a `patterns:` block led by a + `pattern-inside: import "pkg"` guard so bare calls don't fire in unrelated files — the + built-in multi-pattern lib rules all do this; a single-statement rule can instead inline + the `import "pkg"` line into a bare `pattern: |`. +- Security rules: `mode: join`, `metadata.cwe` + `short-description`, `refs:`/`on:` wiring + `src.$UNTRUSTED -> sink.$UNTRUSTED`. +- Structural rules (no dataflow): `mode: search` + `pattern-either` — flags a call site + regardless of taint (weak crypto/hash/random). + +## Source lib rule — HTTP request data (`go/lib/http-sources.yaml`) + +```yaml +rules: + - id: http-sources + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted user input originates here + mode: taint + pattern-sources: + - label: "$UNTRUSTED" + patterns: + - pattern-inside: | + import "net/http" + import "os" + ... + - pattern-either: + - pattern: "($R : *http.Request).FormValue($K)" + - pattern: "($R : *http.Request).PostFormValue($K)" + - pattern: "($R : *http.Request).Cookie($K)" + - pattern: $R.URL.Query().Get($K) + - pattern: $R.URL.Path + - pattern: $R.Header.Get($K) + - pattern: $R.Body + - pattern: os.Getenv($K) + - pattern: os.Args +``` + +Notes: lib sources are `severity: NOTE` and bind the tainted value as the labelled +`$UNTRUSTED`. The `patterns:` block scopes matching to the imported packages via +`pattern-inside: import "..."`. Real file lists many more request accessors and framework +variants (beego); add the project-used ones, don't enumerate the whole API. + +## Sink lib rule — SQL execution (`go/lib/sql-sinks.yaml`) + +```yaml +rules: + - id: sql-sinks + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted data reaches a SQL execution API + mode: taint + pattern-sinks: + - patterns: + - pattern-inside: | + import "database/sql" + ... + - pattern-either: + - pattern: "($DB : *sql.DB).Query($SQL, ...)" + - pattern: "($DB : *sql.DB).QueryContext($CTX, $SQL, ...)" + - pattern: "($DB : *sql.DB).Exec($SQL, ...)" + - focus-metavariable: $SQL +``` + +Notes: the `patterns:` block pairs a `pattern-inside: import "database/sql"` guard with a +`pattern-either` of the call shapes, then `focus-metavariable: $SQL` selects the tainted +argument. The receiver type `*sql.DB` pins the method. Lib sinks are `severity: NOTE`. + +## Sink lib rule — OS command execution (`go/lib/cmdi-sinks.yaml`) + +```yaml +rules: + - id: cmdi-sinks + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted data reaches an OS command execution API + mode: taint + pattern-sinks: + - patterns: + - pattern-inside: | + import "os/exec" + import "os" + import "syscall" + ... + - pattern-either: + - pattern: exec.Command($NAME, ...) + - pattern: exec.CommandContext($CTX, $NAME, ...) + - pattern: syscall.Exec($NAME, ...) + - focus-metavariable: $NAME + - patterns: + - pattern-inside: | + import "os/exec" + ... + - pattern-either: + - pattern: "exec.Command(\"$NAME\", ..., $UNTRUSTED, ...)" + - metavariable-regex: + metavariable: $NAME + regex: sh + - focus-metavariable: $UNTRUSTED +``` + +Notes: each sink shape is a `patterns:` block guarded by `pattern-inside: import "os/exec"`. +The first focuses the command name (`$NAME`); the second matches a shell interpreter invoked +with a literal name and narrows it with `metavariable-regex` before focusing the tainted +argument (`$UNTRUSTED`). Point `focus-metavariable:` at the tainted position when it isn't +first. + +## Sink lib rule — filesystem path (`go/lib/path-sinks.yaml`) + +```yaml +rules: + - id: path-sinks + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted data reaches a filesystem path API + mode: taint + pattern-sinks: + - patterns: + - pattern-inside: | + import "os" + import "io/ioutil" + import "net/http" + ... + - pattern-either: + - pattern: os.Open($P) + - pattern: os.OpenFile($P, ...) + - pattern: os.ReadFile($P) + - pattern: os.WriteFile($P, ...) + - pattern: ioutil.ReadFile($P) + - pattern: http.ServeFile($W, $R, $P) +``` + +## Security join — SQL injection (`go/security/sql-injection.yaml`) + +```yaml +rules: + - id: sql-injection + languages: [go] + severity: ERROR + message: Tainted user input flows into SQL query (SQL injection) + metadata: + cwe: CWE-89 + short-description: SQL injection + mode: join + join: + refs: + - rule: go/lib/http-sources.yaml#http-sources + as: src + - rule: go/lib/http-sources-requesturi.yaml#http-sources-requesturi + as: extra_requesturi + - rule: go/lib/sql-sinks.yaml#sql-sinks + as: sink + on: + - 'src.$UNTRUSTED -> sink.$UNTRUSTED' + - 'extra_requesturi.$UNTRUSTED -> sink.$UNTRUSTED' +``` + +## Security join — others (same shape, swap the sink ref + CWE) + +| Class | id | CWE | sink ref | +|---|---|---|---| +| Command injection | `command-injection` | CWE-78 | `go/lib/cmdi-sinks.yaml#cmdi-sinks` | +| Path traversal | `path-traversal` | CWE-22 | `go/lib/path-sinks.yaml#path-sinks` | +| SSRF | `ssrf` | CWE-918 | `go/lib/ssrf-sinks.yaml#ssrf-sinks` | +| Reflected XSS | `reflected-xss` | CWE-79 | `go/lib/xss-sinks.yaml#xss-sinks` | +| SSTI | `ssti` | CWE-1336 | `go/lib/ssti-sinks.yaml#ssti-sinks` | + +Each is one `src.$UNTRUSTED -> sink.$UNTRUSTED` join refing `http-sources` (plus the +extra header/requesturi source variants where the class needs them). + +## Structural rule — weak hash / weak random (`mode: search`, no dataflow) + +```yaml +rules: + - id: weak-hash-cwe-328 + languages: [go] + severity: ERROR + message: Use of a weak or broken cryptographic hash + metadata: + cwe: CWE-328 + short-description: Weak cryptographic hash + mode: search + patterns: + - pattern-inside: | + import "crypto/md5" + import "crypto/sha1" + ... + - pattern-either: + - pattern: md5.New(...) + - pattern: md5.Sum(...) + - pattern: sha1.New(...) + - pattern: sha1.Sum(...) +``` + +```yaml +rules: + - id: weak-random-cwe-330 + languages: [go] + severity: ERROR + message: Use of a weak (non-cryptographic) random number generator + metadata: + cwe: CWE-330 + short-description: Insecure randomness + mode: search + patterns: + - pattern-inside: | + import "math/rand" + ... + - pattern-either: + - pattern: rand.Intn(...) + - pattern: rand.Int63() + - pattern: rand.Float64() +``` + +Notes: `mode: search` flags a call site structurally — no source/sink join, no taint flow. +The structural rule still guards on `pattern-inside: import "..."` so it only fires for the +intended package. Use it for "dangerous primitive used at all" rules; use `mode: join` when +the finding requires untrusted data to reach the sink. diff --git a/skills-templates/create-rule/references/go.md.j2 b/skills-templates/create-rule/references/go.md.j2 new file mode 100644 index 0000000000..33afacbf21 --- /dev/null +++ b/skills-templates/create-rule/references/go.md.j2 @@ -0,0 +1,120 @@ +# create-rule — Go + +Language reference for authoring Go source/sink library rules, keyed to the body's steps. Bind every tainted value to `$UNTRUSTED` so the joins connect on one name. + +Worked, repo-faithful examples for every vuln class live in `references/go-semgrep-examples.md` — read it before authoring and copy the closest shape. + +## Workflow + +### 1. Check existing coverage + +Built-in Go rules live under `go/lib/` (source and sink library rules) and `go/security/` (the joins and structural rules); mirror that layout for any custom rule you add — never the `java/` tree. Rule ids carry no `go-` prefix. + +### 2. Author the library rules + +Go lib rules are `severity: NOTE` with `options: {lib: true}` and `mode: taint`, carry `languages: [go]`, use the typed-receiver pattern syntax (`($R : *http.Request)` — the `*` only where the API is pointer-receiver), and bind the tainted value as `$UNTRUSTED`. + +Scope every rule to its package by wrapping the patterns in a `patterns:` block led by a `pattern-inside: import "pkg"` guard — the built-in multi-pattern lib rules all do this, so a bare call like `os.Getenv` or `exec.Command` can't fire in an unrelated file. A single-statement rule can instead inline the `import "pkg"` line into a bare `pattern: |`, as `go/lib/http-sources-requesturi.yaml` does. + +Custom source library rule (`.opentaint/rules/go/lib/my-source.yaml`): + +```yaml +rules: + - id: mylib-source + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted user input originates here + mode: taint + pattern-sources: + - label: "$UNTRUSTED" + patterns: + - pattern-inside: | + import mylib "github.com/acme/mylib" + ... + - pattern-either: + - pattern: "($C : *mylib.Ctx).UserInput($K)" +``` + +Custom sink library rule (`.opentaint/rules/go/lib/my-sink.yaml`) — `focus-metavariable` points at the tainted argument: + +```yaml +rules: + - id: mylib-sink + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted data reaches a dangerous operation + mode: taint + pattern-sinks: + - patterns: + - pattern-inside: | + import mylib "github.com/acme/mylib" + ... + - pattern-either: + - pattern: "($E : *mylib.Engine).Render($TPL, ...)" + - focus-metavariable: $TPL +``` + +### 3. Write the test joins + +**Go deviates here.** The Go test harness has no generic taint marker and no throwaway test-rules tree: a sample wires a real source into a real sink, so what the samples run against is the *real* security join, written once under `.opentaint/rules/go/security/` and named by the `rule-test.yaml` manifest as `go/security/.yaml#`. There is nothing test-only to write and nothing to keep out of the scanned rules tree. + +```yaml +rules: + - id: my-ssti-lib-ext + languages: [go] + severity: ERROR + message: Untrusted input reaches a template engine (SSTI) + metadata: + cwe: CWE-1336 + short-description: Server-side template injection + mode: join + join: + refs: + - rule: go/lib/http-sources.yaml#http-sources + as: src + - rule: go/lib/my-ssti-sink.yaml#mylib-sink + as: sink + on: + - 'src.$UNTRUSTED -> sink.$UNTRUSTED' +``` + +For a purely structural rule — a dangerous primitive used at all, such as weak hash or weak random — use `mode: search` + `pattern-either` instead of a join (see the catalog); it needs no source side. + +### 4. Test until success + +There is one shared Go test module and one compiled model, not per-unit `sinks/`/`sources/` sides, so the command names them directly: + +```bash +opentaint test rule run .opentaint/test-compiled/go \ + -o .opentaint/test-results/go \ + --ruleset .opentaint/rules \ + --passthrough-models .opentaint/pass-through +``` + +`test rule run` auto-loads the built-in rules, so pass only your own ruleset — a literal `builtin` here would be treated as a path. There is no `--java-models` for Go. Read the result with the bundled `check-test-result.py`, passing `go` as the unit. + +Pattern operators for each verdict: + +- `falseNegative` → broaden `pattern-either`, check the receiver type (and its `*`), and confirm `$UNTRUSTED` matches across the lib rules and the join's `on:` +- `falsePositive` → narrow with `pattern-not`, `pattern-not-inside`, `pattern-sanitizers`, or `metavariable-regex`, or focus a different sink argument. Inspect the flow that fired with `opentaint summary .opentaint/test-results/go/test-results.sarif --show-findings` +- `skipped` → the manifest entry's rule-id or sample full name is wrong; fix the `rule-test.yaml` entry + +### 5. Escalate when a positive won't converge + +Unchanged, with one Go difference: a dropped library function on the failing sample's path routes to a Go **passThrough** (there is no dataflow approximation), and a re-dispatch applies it with `--passthrough-models`. + +## Constraints + +- Lib rules (source and sink) are `severity: NOTE`, `options: {lib: true}`, `mode: taint`; `severity: ERROR` belongs to the security rules — joins and structural `mode: search` rules alike +- Security joins (`mode: join`) must carry `metadata.cwe` and `metadata.short-description` +- Rule ids are globally unique and carry no language prefix; a custom join id must not collide with a built-in (`sql-injection`, `ssrf`, …) or it is dropped silently +- Custom Go rules go under `.opentaint/rules/go/{lib,security}/`, never the `java/` tree + +## Gotchas + +- The typed-receiver form needs the pointer where the API is pointer-receiver (`($R : *http.Request)`); a value receiver omits the `*` +- Without the `pattern-inside: import "pkg"` guard a bare call pattern fires in unrelated files +- A wrong argument position in a sink focuses the wrong parameter — point `focus-metavariable` at the tainted one +- Don't unpack or grep the analyzer JAR or the go-ssa-server binary for built-in rules — their internals aren't a stable API; read the YAMLs from the path `opentaint health --rules` prints diff --git a/skills-templates/create-rule/sections/workflow.md b/skills-templates/create-rule/sections/workflow.md index 4c4a933ff9..b3ebffeb06 100644 --- a/skills-templates/create-rule/sections/workflow.md +++ b/skills-templates/create-rule/sections/workflow.md @@ -28,7 +28,7 @@ Run the rule tests directly as a foreground, blocking command and wait for exit opentaint test rule run .opentaint/test-compiled// \ -o .opentaint/test-results// \ --ruleset .opentaint/rules --ruleset .opentaint/test-projects///test-rules \ - --passthrough-approximations .opentaint/pass-through + --passthrough-models .opentaint/pass-through ``` `test rule run` auto-loads the built-in rules, so pass only your custom rulesets. Apply the passthrough approximations as-is, an empty one is harmless. Read the result with the bundled script — it prints the pass/fail counts and names the failing samples, so you never parse the JSON by hand: diff --git a/skills-templates/create-test-project/references/go-rule.md.j2 b/skills-templates/create-test-project/references/go-rule.md.j2 new file mode 100644 index 0000000000..74ecf6be60 --- /dev/null +++ b/skills-templates/create-test-project/references/go-rule.md.j2 @@ -0,0 +1,105 @@ +# create-test-project — Go rule test project + +For `type: rule-source` / `rule-sink` on a Go project. There is no Go dataflow approximation, so `type: dataflow` never reaches this language. + +**Go deviates from the body in three ways.** There is one shared test module for every Go rule test — not one project per unit, and no `sources/`/`sinks/` sub-projects. There is no scaffold command and no generic taint marker: a sample wires a *real* source into a *real* sink, and the manifest names the *real* rule under test. Everything else — one verdict per sample, extend rather than re-scaffold, compile as the deliverable — is unchanged. + +## Workflow + +### 1. Init or extend the module + +The module lives at `.opentaint/test-projects/go` and is an ordinary Go module. Extend it for each new unit rather than creating another: + +- `go.mod` — `module test`, a `go` line, and for any third-party API the samples import a `require ` plus a local `replace => ./stubs/`. Stub a heavy dependency to a minimal local package rather than vendoring the library. The stub directory must itself be a **nested Go module**: `stubs//go.mod` whose `module` line is the **exact real import path** being stubbed (e.g. `module github.com/beego/beego/v2`), holding only the API surface the samples use. A bare directory of `.go` files fails module resolution when `opentaint compile` runs `go` +- samples under a category package per unit, e.g. `security//sample.go`. The analyzer references a function by its **import path + name** — `test/security/.PositiveSourceRequestFormValue` +- small helper source/sink funcs so each sample stays one flow — `requestForSources()` returning `*http.Request`, `envSource()` returning a tainted string, `sqlSink(v)` calling `db.Query(fmt.Sprint(v))` + +### 2. Write the samples + +Read the real Go source/sink API from the dependency or stdlib first, so each sample is built on the signature the rule actually matches. One function, one verdict, named by convention: + +- **positive** (`PositiveXxx`) — a minimal flow that must flag, routing a real source into the real sink +- **negative** (`NegativeXxx`) — the same shape neutralized: a sanitizer between source and sink, or a safe/parameterized sink + +Prefix a sample whose behaviour the engine is known not to support yet with `Unsupported` so it isn't counted a strict pass/fail. + +```go +package allpatterns + +import ( + "database/sql" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" +) + +var db *sql.DB + +func requestForSources() *http.Request { + return &http.Request{ + URL: &url.URL{RawQuery: "q=v"}, + Form: url.Values{"q": []string{"form"}}, + Body: io.NopCloser(strings.NewReader("body")), + } +} + +func sqlSink(value interface{}) { + _, _ = db.Query(fmt.Sprint(value)) +} + +func envSource() string { + return os.Getenv("TAINTED") +} + +func PositiveSourceRequestFormValue() { + r := requestForSources() + sqlSink(r.FormValue("q")) +} + +func PositiveSourceURLQueryGet() { + r := requestForSources() + sqlSink(r.URL.Query().Get("q")) +} + +func NegativeSQLParameterizedArgument() { + _, _ = db.Query("select * from users where name = ?", envSource()) +} + +func NegativeSQLConstantQuery() { + _, _ = db.Query("select * from users where active = 1") +} +``` + +Expected verdicts go in `.opentaint/test-projects/go/rule-test.yaml`, keyed by the **real** rule-id (`go/security/.yaml#`, the file and id create-rule gave the join) and listing each sample by its `import-path.FuncName` full name — not the Go `package` name: + +```yaml +tests: + - rule-id: go/security/sql-injection.yaml#sql-injection + positive: + - test/security/all-patterns.PositiveSourceRequestFormValue + - test/security/all-patterns.PositiveSourceURLQueryGet + negative: + - test/security/all-patterns.NegativeSQLParameterizedArgument + - test/security/all-patterns.NegativeSQLConstantQuery +``` + +### 3. Compile to the model + +One module, one model — no per-side sub-model: + +```bash +opentaint compile .opentaint/test-projects/go -o .opentaint/test-compiled/go +``` + +The model is portable: the compile copies the module into `.opentaint/test-compiled/go/go_0` and records the relative `projectDir: go_0`, so `test rule run` resolves `rule-test.yaml` and the sample functions from that copy, not from the live sources. Recompile after every sample or manifest edit; `-o` must not already exist, so delete the old model or compile to a fresh directory. `go` must be on PATH. + +## Gotchas + +- The full name in `rule-test.yaml` is `import-path.FuncName` (the directory path, e.g. `test/security/all-patterns.Foo`), not the Go `package` name (`allpatterns`) +- A name that resolves to no function is fatal to the **whole run**, not just that sample: resolution is an exact `fullName` match with no fuzzy fallback, and one unresolved entrypoint aborts every test with an opaque analyzer exception and no per-sample results. After `compile`, verify every name matches a compiled function before `test rule run` +- Keep the `go.mod` `go` directive (and each stub's) at or below the installed toolchain. Under the default `GOTOOLCHAIN=auto` a newer directive makes `go` try to **download** a toolchain, which fails offline; the shared module tracks `go 1.22` +- One module serves every Go rule test, so parallel agents share `go.mod`, the sample packages, and `rule-test.yaml`. Give each unit its own category dir/package, keep `rule-test.yaml` edits append-only, and don't concurrently edit a shared helper file or the same `require`/`replace` lines +- A positive must route the helper source into the real sink — a sink fed a constant or a bare parameter with no in-sample source can't be flagged by a taint rule diff --git a/skills-templates/debug-rule/sections/workflow.md b/skills-templates/debug-rule/sections/workflow.md index 3a490974f6..f000540111 100644 --- a/skills-templates/debug-rule/sections/workflow.md +++ b/skills-templates/debug-rule/sections/workflow.md @@ -7,22 +7,24 @@ opentaint test rule reachability \ --project-model \ -o /report.sarif \ --ruleset builtin --ruleset .opentaint/rules \ - --passthrough-approximations .opentaint/pass-through \ - --dataflow-approximations .opentaint/dataflow + --passthrough-models .opentaint/pass-through \ + --java-models .opentaint/dataflow ``` +`--java-models` is Java/JVM only — a Go run passes `--passthrough-models` alone. + `` is `.opentaint/test-results/` for a test model, `.opentaint/results` for the main scan. The per-instruction facts are in the sibling `/debug-ifds-fact-reachability.sarif`, not the `-o` file — the `-o` SARIF only shows whether the rule fired. Read that sibling to find the kill: - a missed detection (a positive that won't pass, or a flow absent from a scan) — confirm a fact exists at the source; if none, the gap is in `pattern-sources`, not the flow. Otherwise walk the facts to the last instruction still carrying it and the first where it's gone — that gap is the kill - a spurious detection (a negative that fires) — the reverse: find where a fact appears with no tainted input reaching it -Trace the exact run that misbehaved — a different `model` or ruleset traces something else; taint dying at an approximated call means that approximation isn't propagating. When the flow is missed and the entry method may never be analyzed, rerun with `--entry-points ""`: a finding that appears only then is an entry-point-discovery problem, not dataflow. On Spring the flag is additive — auto-discovered endpoints stay and your method is added, so use it to force-include an endpoint the analyzer never starts from, not to narrow to one method. +Trace the exact run that misbehaved — a different `model` or ruleset traces something else; taint dying at an approximated call means that approximation isn't propagating. When the flow is missed and the entry method may never be analyzed, rerun with `--entry-points ""`: a finding that appears only then is an entry-point-discovery problem, not dataflow. On Spring the flag is additive — auto-discovered endpoints stay and your method is added, so use it to force-include an endpoint the analyzer never starts from, not to narrow to one method. On Go, `--entry-points` takes a function full name in `package.FunctionName` form, not the JVM `Class#method` form. ### 2. Classify the cause The killing instruction decides who owns the fix. An engine bug is by far the least likely — assume it last, only once the other two are ruled out; nearly every kill is a missing or wrong library model or a rule defect, both tedious to exclude but far more probable, and the tedium is no reason to jump to "engine". Three outcomes: -- the kill is at an external library method → a model issue. Cross-check `dropped-external-methods.yaml` from that run (a `--track-external-methods` scan regenerates it if absent): listed there means the method is unmodeled — the missing model is the cause, for the approximation stage to model. Not listed but a built-in claims to model it, yet taint dies here → that model is wrong for this case: a passThrough override applies at the rule level, so prefer one for the method; a dataflow override conflicts with built-ins at load, so fall back to a passThrough, or call it an engine issue when only a dataflow shape can express the propagation +- the kill is at an external library method → a model issue. Cross-check `dropped-external-methods.yaml` from that run (a `--track-external-methods` scan regenerates it if absent): listed there means the method is unmodeled — the missing model is the cause, for the approximation stage to model. Not listed but a built-in claims to model it, yet taint dies here → that model is wrong for this case: a passThrough override applies at the rule level, so prefer one for the method; a dataflow override conflicts with built-ins at load, so fall back to a passThrough, or call it an engine issue when only a dataflow shape can express the propagation. On Go there is no dataflow approximation at all: the fix is a passThrough, or an engine issue when no passThrough can express the propagation — never a dataflow fallback - the kill is where the rule should have matched — a sanitizer misfires, a sink or source variant went unmatched → a rule defect, for rule authoring to fix - the kill is a plain instruction the engine must propagate through (assignment, cast, field read, an already-modeled call), with the rule correct and the model complete → an engine issue diff --git a/skills-templates/discover-attack-surface/references/go.md.j2 b/skills-templates/discover-attack-surface/references/go.md.j2 new file mode 100644 index 0000000000..5be895c49a --- /dev/null +++ b/skills-templates/discover-attack-surface/references/go.md.j2 @@ -0,0 +1,36 @@ +# discover-attack-surface — Go + +`go` must be on PATH — the member extraction and every signature lookup drive the Go toolchain. + +## Workflow + +### 1. Settle built-in coverage + +Built-in Go source and sink rules live under `go/lib/` within the `opentaint health --rules` root — grep there for the member's package or method name (a rule is a semgrep-like pattern, not indexed by import path), then read the matched rule file to confirm it covers the member. The project's own custom rules are under `.opentaint/rules/go`. + +Go untrusted-data sources are declared by those `go/lib/*-sources.yaml` rules, not by framework endpoint auto-discovery: an inbound source the built-in rules already match needs no new source, so focus on the boundaries they miss and on the sinks. + +### 2. Classify the plan's members + +Each plan member is `{ method, signature }` where `method` is `.` and `signature` is empty — Go has no descriptor and no overloads, so a member is identified by its name alone. + +Go dependencies are source on disk, not resolved jars. Locate a module's source to read signatures: + +- `go mod download -json ` prints the extracted source dir as its `Dir` field +- or `$(go env GOMODCACHE)/@/` — uppercase letters in a module path are case-escaped as `!`, so `github.com/Azure` is `github.com/!azure` +- or `vendor//` when the project vendors + +Run `go` commands from a module directory (a `goProjects.projectDir` from `project.yaml`). `go doc .` prints the exported signature and receiver; `go doc -src` prints its body. + +**The plan's scope is an entry scope, not the whole surface.** The extractor sees only package-qualified selectors (`gin.New`, `jwt.Parse`); it is structurally blind to method calls on receiver values (`c.Query(…)`, `db.QueryContext(…)`) — the dominant Go idiom and the exact shape of most accessor sources and sinks — and to members reached through interface dispatch, reflection, struct-embedding promotion, router/handler registration, config strings, or generated code absent from the source tree. Treat the plan as *which packages, constructors, and package-level functions the project touches*, then read the app source and the dependency's source to add the methods called on those values and the indirectly reached members, and classify those too. Do not enumerate the whole module API. + +Inspect only the project's own source and the dependency's source — never the analyzer JAR or the go-ssa-server binary. + +Classifying: + +- **sources** — where untrusted data first enters from a boundary: a method that *returns* attacker-controlled data (`(*http.Request).FormValue`, a framework context accessor, a message-broker payload). Not a method that merely passes along data it was handed — that is a propagator the engine already handles +- **sinks** — dangerous operations (query construction, command/file/path ops, deserialization, template/SSTI, SSRF request building, reflection), each tagged with its vuln class + +### 3. Write the source units + +`dependencies` is the Go module identity, `@` (e.g. `github.com/gin-gonic/gin@v1.10.0`) — read the version from the project's `go.mod`, not from the import path. diff --git a/skills-templates/generate-poc/references/go.md.j2 b/skills-templates/generate-poc/references/go.md.j2 new file mode 100644 index 0000000000..9ebb519dff --- /dev/null +++ b/skills-templates/generate-poc/references/go.md.j2 @@ -0,0 +1,17 @@ +# generate-poc — Go + +Language reference for building and starting a Go app to reproduce against, keyed to the body's step. + +## Workflow + +### 1. Start the app + +Build and start the app the way the project expects, then wait until the port is listening: + +- from source — `go run ./...`, or `go run ./cmd/` when the module has several commands +- a built binary — `go build -o app . && ./app` +- a containerized app — `docker compose up` + +`go` must be on PATH, and the module's `go`/`toolchain` directive must be satisfiable — under the default `GOTOOLCHAIN=auto` a newer directive makes `go` try to download a toolchain, which fails offline. + +Loopback bind (per the body's Constraints): a Go server's listen address is its own flag or env var — pass `127.0.0.1:` to whatever the app hands `http.ListenAndServe` / `net.Listen` rather than a bare `:`, which binds the wildcard. For a container, `docker run -p 127.0.0.1:8080:8080 …` or a compose override on the port mapping. diff --git a/skills-templates/orchestrate-stage/references/approx-round.md b/skills-templates/orchestrate-stage/references/approx-round.md index 95c1b0375a..8bcbcad8df 100644 --- a/skills-templates/orchestrate-stage/references/approx-round.md +++ b/skills-templates/orchestrate-stage/references/approx-round.md @@ -37,7 +37,10 @@ It merges every batch's `skipped` / `engine_issues` into `approximations/skipped - `language` - `batch` - `methods` (optional) — only an explicitly assigned subset to redo -- dataflow step 1 → dispatch create-test-project + +On Go the dataflow branch never runs: there is no dataflow approximation, so a Go batch's `dataflow` bucket stays empty, no test project is scaffolded for it, and every carrier is a passThrough. A Go carrier that won't converge goes straight to escalation. + +- dataflow step 1 (Java/JVM only) → dispatch create-test-project - `language` - `type: dataflow` - `batch` diff --git a/skills-templates/report-analyzer-issue/sections/workflow.md b/skills-templates/report-analyzer-issue/sections/workflow.md index 5a813cdf36..c268adaa2e 100644 --- a/skills-templates/report-analyzer-issue/sections/workflow.md +++ b/skills-templates/report-analyzer-issue/sections/workflow.md @@ -15,7 +15,7 @@ Header (both kinds): - Type — `analysis` or `resource` - Setup — the exact command that reproduces it: the `opentaint test rule …` / `test approximation run` for `analysis`, or the failed `opentaint scan` for `resource`, each with its model, rulesets, and approximation dirs (plus final memory and timeout/backstop outcome for `resource`) - Run logs — the run's log files, by path: for `analysis`, any debug log or fact-reachability SARIF the caller cited, if present; for `resource`, the scan's output log -- Minimal repro — the smallest set of files/folders that demonstrates it: for `analysis`, the test project `.opentaint/test-projects/` if one was named; for `resource`, the project model, rulesets, and approximation dirs in play (with their rough size) +- Minimal repro — the smallest set of files/folders that demonstrates it: for `analysis`, the test project `.opentaint/test-projects/` if one was named; for `resource`, the project model, rulesets, and approximation dirs in play (with their rough size). A Go passThrough has no test project — cite the scan evidence instead: the config under test, the modeled method now absent from `dropped-external-methods.yaml` (and present in `approximated-external-methods.yaml`), the flow still missing from the scan SARIF, and a minimal standalone Go module wiring the modeled call between a real source and sink - TL;DR — 2–3 sentences: the symptom, and for `analysis` where taint appears to die and what the engine should do instead Below the header, free-form — whatever makes the case, at least: diff --git a/skills-templates/run-scan/sections/constraints.md b/skills-templates/run-scan/sections/constraints.md index 14a45bb492..2e28051b5b 100644 --- a/skills-templates/run-scan/sections/constraints.md +++ b/skills-templates/run-scan/sections/constraints.md @@ -1 +1,2 @@ - Never hand-edit the project model to change scan results — this skill only reads it. If the model is wrong, rebuild it at the model stage rather than patching `project.yaml` or anything under it +- Go coverage can go missing without the scan failing. `go` absent from PATH hard-fails a Go-only scan, but in a polyglot repo it only warns and silently skips the Go analysis — the other languages scan normally and the report simply carries no Go findings. The same silence follows a model whose Go source root no longer resolves: a `--project-model` scan re-derives that root from `project.yaml` and re-detects Go against it, so a hand-written model recording an absolute `projectDir` loses Go coverage once that tree moves or loses its `go.mod` (an autobuilder model is portable — the sources are copied into `/go_0` under a model-relative `projectDir`). On a Go run, check the report actually holds Go results before treating it as complete diff --git a/skills-templates/run-scan/sections/workflow.md b/skills-templates/run-scan/sections/workflow.md index d351480681..bdcd65b0fa 100644 --- a/skills-templates/run-scan/sections/workflow.md +++ b/skills-templates/run-scan/sections/workflow.md @@ -10,11 +10,13 @@ opentaint scan --project-model .opentaint/project \ ``` - `--rule-id ` — restrict to specific executable rules (repeatable, one per input rule ID); referenced library rules remain available without listing their IDs separately. Omit to run all loaded rules -- `--passthrough-approximations .opentaint/pass-through` — add when that directory exists: passThrough configs override built-ins at the rule level, a provided rule overriding a built-in only when it matches one -- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations, one project per batch +- `--passthrough-models .opentaint/pass-through` — add when that directory exists: passThrough configs override built-ins at the rule level, a provided rule overriding a built-in only when it matches one +- `--java-models .opentaint/dataflow` — Java/JVM only; add when that directory exists: code-based approximations, one project per batch. A Go run has no dataflow directory and never passes this flag — `--passthrough-models` is its only model flag Both approximation-dir flags walk their trees recursively; pass each parent directory once, not every package or batch separately. +A Go project also scans straight from sources — `opentaint scan ` compiles on the fly — but prefer the pre-built model so every scan in a run sees the same code. `go` must be on PATH either way. + The scan is long — run it in the background and wait for it to finish. Leave `--timeout` at the engine default (900s); the CLI ends the analysis itself and writes whatever SARIF it has. ### 2. Retry once on out-of-memory diff --git a/skills-templates/shared/debugging.md b/skills-templates/shared/debugging.md index 8b7ce31714..8d80bd6ce8 100644 --- a/skills-templates/shared/debugging.md +++ b/skills-templates/shared/debugging.md @@ -15,7 +15,7 @@ opentaint test rule reachability \ - `` — the one rule whose sample routes taint through the code under test (`.yaml:`). One rule per run — across many rules the trace is unusably huge. Its library `refs` are collected automatically - read the sibling `/debug-ifds-fact-reachability.sarif`, not the `-o` file. The `-o` SARIF only shows default scan output. The sibling holds the per-instruction facts that show where taint dies -- apply the approximations the failing run used so the trace matches it: `--passthrough-approximations ` and/or `--dataflow-approximations `. Taint dying at an approximated call then means that approximation isn't propagating +- apply the approximations the failing run used so the trace matches it: `--passthrough-models ` and/or `--java-models `. Taint dying at an approximated call then means that approximation isn't propagating - debug the exact run that showed the problem — same model, rulesets, approximation dirs — or you're debugging something else ## Reading the trace diff --git a/skills-templates/shared/scripts/generate.py b/skills-templates/shared/scripts/generate.py index 53e6b4488d..aba5c73647 100644 --- a/skills-templates/shared/scripts/generate.py +++ b/skills-templates/shared/scripts/generate.py @@ -118,8 +118,9 @@ def root_of(fqn, depth=ROOT_DEPTH): def in_packages(cls, prefixes): - # dotted-boundary match: `a.b.collect` never matches a sibling `a.b.collectX` - return any(cls == p or cls.startswith(p + ".") for p in prefixes) + # dotted-boundary match: `a.b.collect` never matches a sibling `a.b.collectX`. `/` is a + # boundary too, so a Go module prefix also covers its subpackages (`mod/sub.Member`) + return any(cls == p or cls.startswith(p + ".") or cls.startswith(p + "/") for p in prefixes) def atomize(fqns, cap): @@ -283,6 +284,86 @@ def is_project_class(cls, packages): def extract_usages(): + # (fqn, signature) pairs for the members the project's own code calls; the model decides + # which extractor applies — a Go model has no bytecode to disassemble + doc = load_yaml(MODEL / "project.yaml", {}) or {} + if doc.get("goProjects") and not doc.get("javaProjects"): + return extract_usages_go(doc) + return extract_usages_java() + + +GO_IMPORT_RE = re.compile(r'^\s*(?:import\s+)?([\w.]+)?\s*"([^"]+)"') + + +def go_module_dirs(doc): + dirs = [] + for entry in doc.get("goProjects") or []: + d = Path(str((entry or {}).get("projectDir") or "")) + if not str(d): + continue + dirs.append(d if d.is_absolute() and d.is_dir() else MODEL / d) + return [d for d in dirs if d.is_dir()] or [MODEL] + + +def go_package_name(mod_dir, import_path, cache): + # the local identifier an import binds when the file does not alias it: the package's own + # name, which need not be the last path segment (and is never a `vN` version suffix) + if import_path in cache: + return cache[import_path] + name = "" + try: + out = subprocess.run(["go", "list", "-f", "{{.Name}}", import_path], cwd=str(mod_dir), + capture_output=True, text=True) + name = out.stdout.strip() + except OSError: + pass + if not name: + segs = import_path.split("/") + name = segs[-2] if len(segs) > 1 and re.fullmatch(r"v\d+", segs[-1]) else segs[-1] + cache[import_path] = name + return name + + +def extract_usages_go(doc): + # scan the project's own .go files for package-qualified selectors on a pending module and + # return them as `.`; Go has no descriptor, so the signature is empty. + # Structurally blind to method calls on receiver values, interface dispatch, and reflection — + # the frontier agent adds those from the source (per its language reference). + packages = pending_packages() + fqns, cache = set(), {} + for mod_dir in go_module_dirs(doc): + for f in mod_dir.rglob("*.go"): + try: + text = f.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + imports, block = [], False + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("import ("): + block = True + continue + if block and stripped.startswith(")"): + block = False + continue + if not block and not stripped.startswith("import"): + continue + m = GO_IMPORT_RE.match(line) + if not m: + continue + alias, path = m.group(1), m.group(2) + if not in_packages(path, packages): + continue + if alias in (".", "_"): + continue + imports.append((path, alias or go_package_name(mod_dir, path, cache))) + for path, ident in imports: + for member in re.findall(rf"\b{re.escape(ident)}\.([A-Z]\w*)", text): + fqns.add((f"{path}.{member}", "")) + return fqns + + +def extract_usages_java(): # disassemble project classes, collect // Method / // InterfaceMethod call sites with their # JVM descriptor; returns (fqn, signature) pairs so an overloaded member stays disambiguated fqns = set() diff --git a/skills-templates/triage-dependencies/sections/workflow.md b/skills-templates/triage-dependencies/sections/workflow.md index 89bfc69d2f..911cb70f8e 100644 --- a/skills-templates/triage-dependencies/sections/workflow.md +++ b/skills-templates/triage-dependencies/sections/workflow.md @@ -2,10 +2,14 @@ Read `.opentaint/project/project.yaml` — the `dependencies` list under each per-language projects entry is every third-party dependency the model resolved. Resolve each to the library it is. Most of a large project's dependencies are transitive infrastructure +A `goProjects:` entry carries no such list — read the module's own `go.mod` (`require` blocks) and `go.sum` for its dependency modules instead, and identify each by module path rather than by a coordinate + ### 2. Mark each library For each library decide: could it introduce an attacker-controlled source — a method returning untrusted data (HTTP/RPC request data, message-broker payloads, deserialized untrusted input and so on)? Judge by the library's identity itself, read sources to get overviews, docs +Dismiss the obvious infrastructure by identity — logging (logback/slf4j; Go `go.uber.org/zap`, `log/slog`), build plugins, annotations, bytecode tooling (ASM, byte-buddy) or codegen runtimes (`google.golang.org/protobuf`), test libraries (Go `github.com/stretchr/testify`), pure data structures. When unsure, peek: grep the project's sources for the library's imports or call sites — for Go that grep on the import path is the only peek available, there is no jar to inspect + ### 3. Write the flag list Write in `.opentaint/tracking/coverage.yaml` (per Tracking) a flat list of the flagged libraries' packages diff --git a/skills/analyze-external-methods/references/go.md b/skills/analyze-external-methods/references/go.md new file mode 100644 index 0000000000..ef7d272bab --- /dev/null +++ b/skills/analyze-external-methods/references/go.md @@ -0,0 +1,45 @@ +# analyze-external-methods — Go + +## Workflow + +### 1. Classify propagation + +**Read the plan entry first.** A Go dropped record carries the qualified callee in `method` only; its `signature` is literally `args:` — the parameter count at the call site (it varies only for variadic calls) and holds no package, type, receiver, or name. Never read a package or type out of `signature`. + +`method` comes in two shapes: + +- plain function — `.`, e.g. `net/url.ParseQuery`, `net/http.Get`. Split at the last `.`: everything before is the import path (it may hold `/` and `.`, as in `github.com/foo/bar`), the trailing segment is the name; no receiver +- method — `().`, e.g. `(*net/http.Request).FormValue`. The parenthesized part is the receiver type; strip a leading `*`, then split at the last `.` for the import path and the type. The segment after `)` is the method name + +That decomposition is what the build stage's `function:` matcher consumes: + +```text +net/url.ParseQuery → package: net/url, name: ParseQuery, receiver: false +(*net/http.Request).FormValue → package: net/http, type: Request, name: FormValue, receiver: true +``` + +**Getting the source.** An application-internal function sits in the project's own sources under the project root — read it directly. For a dependency, the source is source, not bytecode: it sits in the module cache (`$(go env GOMODCACHE)/@/…`) or in `vendor//` when the project vendors. Standard-library code is under `$(go env GOROOT)/src//`. `go doc .` and `go doc -src .` resolve a name and print its declaration without hunting for the file. + +**Go has no dataflow approximation.** The `dataflow` bucket stays empty and `dependencies` stays `[]` — there is no dataflow builder, no test project, and no `build.test_project` entries for Go. A carrier whose data travels through a function value, a callback, or a goroutine/channel chain is still classified `passthrough`, modeled as the closest position-to-position copy, and only becomes an `engine_issues` entry the usual way: after it is built and the method still drops. Never classify it `dataflow` in the hope of a second approximation kind — there isn't one. + +passthrough examples — the data is copied from one position to another: + +- `strings.ToLower`/`#Join`/`#TrimSpace`/`#Replace`, `fmt.Sprintf`, `[]byte(…)` / `string(…)` conversions +- `strings.Builder#WriteString` and `bytes.Buffer#Write`/`#String`, `io.Copy`, `bufio.Scanner#Text` +- request accessors — `(*net/http.Request).FormValue`/`#PostFormValue`, `net/url.Values#Get`, `net/url.ParseQuery` +- encoding round trips that keep the data — `encoding/json.Marshal`/`Unmarshal`, `net/url.QueryEscape`/`#QueryUnescape`, `encoding/base64` encode/decode +- external key-value stores — a `Set` paired with a `Get` carries the value across the round trip; model both ends + +skipped examples — the method carries the data nowhere: + +- predicates and inspectors returning a bool or a number — `strings.Contains`/`#HasPrefix`/`#EqualFold`, `len`, `bytes.Equal` +- conversions that collapse the data — `strconv.Atoi`, `strconv.ParseInt`, one-way hashes +- logging and metrics that keep none of the data downstream — `log.Printf`, `log.Println` + +### 2. Classify sinks (deep run only) + +Unchanged in substance; the record's `method` is the Go qualified name above, and each sink entry's `signature` is the plan's `args:` string — Go has no descriptor, so overloads never need distinguishing (Go has none). Go sink shapes worth recognizing: `database/sql.DB#Query`/`#Exec` (SQL), `os/exec.Command`/`#CommandContext` (OS command), `os.Open`/`os.ReadFile`/`path/filepath.Join` (path traversal), `net/http.Get`/`#Post`/`Client#Do` (SSRF), `html/template` vs `text/template` `Execute` (XSS — `text/template` does not escape), `encoding/gob.Decoder#Decode` (deserialization). + +### 3–4. Verify coverage, re-verify the skips + +Unchanged — `check-coverage.py --batch ` is language-independent. diff --git a/skills/appsec-agent/SKILL.md b/skills/appsec-agent/SKILL.md index 35129654d4..4acd037f90 100644 --- a/skills/appsec-agent/SKILL.md +++ b/skills/appsec-agent/SKILL.md @@ -32,6 +32,8 @@ This workflow requires two subagent levels: MAIN → stage orchestrator → leaf Read the project's build files to fix the target language — Maven/Gradle → java, `go.mod` → go, and so on. Record it at bootstrap; stage orchestrators pass it to language-coupled leaves. +For `go`, also confirm `go` is on PATH (`command -v go`) before bootstrapping — the analyzer drives the Go toolchain at both build and scan time, so a Go run hard-fails without it. Tell the user and stop if it's missing. Two pipeline branches are Java-only and simply don't run for Go: dataflow approximations (Go has only passThrough) and everything that depends on them. + ### 4. Choose the workflow Ask the user for both levels together: diff --git a/skills/appsec-agent/scripts/generate.py b/skills/appsec-agent/scripts/generate.py index 53e6b4488d..aba5c73647 100644 --- a/skills/appsec-agent/scripts/generate.py +++ b/skills/appsec-agent/scripts/generate.py @@ -118,8 +118,9 @@ def root_of(fqn, depth=ROOT_DEPTH): def in_packages(cls, prefixes): - # dotted-boundary match: `a.b.collect` never matches a sibling `a.b.collectX` - return any(cls == p or cls.startswith(p + ".") for p in prefixes) + # dotted-boundary match: `a.b.collect` never matches a sibling `a.b.collectX`. `/` is a + # boundary too, so a Go module prefix also covers its subpackages (`mod/sub.Member`) + return any(cls == p or cls.startswith(p + ".") or cls.startswith(p + "/") for p in prefixes) def atomize(fqns, cap): @@ -283,6 +284,86 @@ def is_project_class(cls, packages): def extract_usages(): + # (fqn, signature) pairs for the members the project's own code calls; the model decides + # which extractor applies — a Go model has no bytecode to disassemble + doc = load_yaml(MODEL / "project.yaml", {}) or {} + if doc.get("goProjects") and not doc.get("javaProjects"): + return extract_usages_go(doc) + return extract_usages_java() + + +GO_IMPORT_RE = re.compile(r'^\s*(?:import\s+)?([\w.]+)?\s*"([^"]+)"') + + +def go_module_dirs(doc): + dirs = [] + for entry in doc.get("goProjects") or []: + d = Path(str((entry or {}).get("projectDir") or "")) + if not str(d): + continue + dirs.append(d if d.is_absolute() and d.is_dir() else MODEL / d) + return [d for d in dirs if d.is_dir()] or [MODEL] + + +def go_package_name(mod_dir, import_path, cache): + # the local identifier an import binds when the file does not alias it: the package's own + # name, which need not be the last path segment (and is never a `vN` version suffix) + if import_path in cache: + return cache[import_path] + name = "" + try: + out = subprocess.run(["go", "list", "-f", "{{.Name}}", import_path], cwd=str(mod_dir), + capture_output=True, text=True) + name = out.stdout.strip() + except OSError: + pass + if not name: + segs = import_path.split("/") + name = segs[-2] if len(segs) > 1 and re.fullmatch(r"v\d+", segs[-1]) else segs[-1] + cache[import_path] = name + return name + + +def extract_usages_go(doc): + # scan the project's own .go files for package-qualified selectors on a pending module and + # return them as `.`; Go has no descriptor, so the signature is empty. + # Structurally blind to method calls on receiver values, interface dispatch, and reflection — + # the frontier agent adds those from the source (per its language reference). + packages = pending_packages() + fqns, cache = set(), {} + for mod_dir in go_module_dirs(doc): + for f in mod_dir.rglob("*.go"): + try: + text = f.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + imports, block = [], False + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("import ("): + block = True + continue + if block and stripped.startswith(")"): + block = False + continue + if not block and not stripped.startswith("import"): + continue + m = GO_IMPORT_RE.match(line) + if not m: + continue + alias, path = m.group(1), m.group(2) + if not in_packages(path, packages): + continue + if alias in (".", "_"): + continue + imports.append((path, alias or go_package_name(mod_dir, path, cache))) + for path, ident in imports: + for member in re.findall(rf"\b{re.escape(ident)}\.([A-Z]\w*)", text): + fqns.add((f"{path}.{member}", "")) + return fqns + + +def extract_usages_java(): # disassemble project classes, collect // Method / // InterfaceMethod call sites with their # JVM descriptor; returns (fqn, signature) pairs so an overloaded member stays disambiguated fqns = set() diff --git a/skills/build-project/references/go.md b/skills/build-project/references/go.md new file mode 100644 index 0000000000..97a960887a --- /dev/null +++ b/skills/build-project/references/go.md @@ -0,0 +1,65 @@ +# build-project — Go specific instructions + +## Workflow + +### 2. Identify the build + +- `go.mod` at or under `` (within a few levels) → Go module +- the `go` toolchain must be on PATH (`command -v go`). The analyzer drives it through go-ssa-server at scan time, so a Go project cannot be analyzed without it — if `go` is missing, say so and stop rather than producing a half-model +- the installed `go` must satisfy the module's own `go`/`toolchain` directive: a project pinning a newer toolchain either auto-downloads it (needs network) or fails under `GOTOOLCHAIN=local` +- there is no classpath mode. `opentaint scan ` also compiles Go sources on the fly, so a model is a convenience for reusing a fixed project, not a precondition for scanning + +### 3. Enable all modules + +A Go repo has no disabled-module concept — nothing to re-enable. A multi-module repo yields one `goProjects` entry per module; confirm the autobuilder found them all. + +### 4. Autobuilder + +The same `opentaint compile` that builds JVM projects builds Go — it detects `go.mod` and emits a `goProjects` model. No `JAVA_HOME` and no toolchain hint apply: + +```bash +opentaint compile -o .opentaint/project +``` + +The autobuilder writes a portable model: it copies each module's sources into `go_/` inside the model directory and records a relative `projectDir` (no top-level `projectRoot`), so the copied tree travels with the model: + +```yaml +goProjects: +- projectDir: go_0 +``` + +The model is multi-language — one `project.yaml` can hold both `goProjects:` and `javaProjects:` for a mixed repository. + +On failure `opentaint compile` prints the log-file path ("For full details, check the log file:"); read it and fix the real build with the project's own commands before re-running: + +```bash +go build ./... +go mod download +``` + +### 5. Manual model — last resort + +A Go model is minimal, so the fallback is to write `.opentaint/project/project.yaml` by hand, one entry per module, each an absolute path (a relative `projectDir` resolves against the model directory, not the current directory): + +```yaml +goProjects: +- projectDir: +``` + +There is no `--package` scoping step: the Go model's scope is the module itself, and `opentaint project` is a JVM-only path. Alternatively skip the model and point the scan straight at the sources. + +### 6. Verify + +`project.yaml` exists, is non-empty, and lists one `goProjects.projectDir` per module the repo declares in a `go.mod`. + +## Constraints + +- `go` on PATH is required for both building and scanning; the bundled Java runtime does not cover it +- `opentaint pull` pre-provisions the analyzer, autobuilder, built-in rules, go-ssa-server, and bundled Java — it does not install `go` + +## Gotchas + +- `go` not on PATH → a Go project hard-fails at scan time; install the toolchain before building or scanning +- Unresolvable dependencies → analysis loads the module graph through the `go` toolchain (go-ssa-server calls `packages.Load` with imports and deps). A dependency that fails to resolve is logged as a warning and its types and flows go missing — findings through it are silently lost, and the scan only hard-fails when the `go` invocation itself errors. Run `go mod download` or vendor the deps, ensure network access, and set `GOPRIVATE` plus credentials for private modules +- A hand-written model does not repair a broken build — `opentaint scan` recompiles from source and hits the same error +- Missing dependencies from submodules → `git submodule update --init` diff --git a/skills/create-dataflow-approximation/references/debugging.md b/skills/create-dataflow-approximation/references/debugging.md index 8b7ce31714..8d80bd6ce8 100644 --- a/skills/create-dataflow-approximation/references/debugging.md +++ b/skills/create-dataflow-approximation/references/debugging.md @@ -15,7 +15,7 @@ opentaint test rule reachability \ - `` — the one rule whose sample routes taint through the code under test (`.yaml:`). One rule per run — across many rules the trace is unusably huge. Its library `refs` are collected automatically - read the sibling `/debug-ifds-fact-reachability.sarif`, not the `-o` file. The `-o` SARIF only shows default scan output. The sibling holds the per-instruction facts that show where taint dies -- apply the approximations the failing run used so the trace matches it: `--passthrough-approximations ` and/or `--dataflow-approximations `. Taint dying at an approximated call then means that approximation isn't propagating +- apply the approximations the failing run used so the trace matches it: `--passthrough-models ` and/or `--java-models `. Taint dying at an approximated call then means that approximation isn't propagating - debug the exact run that showed the problem — same model, rulesets, approximation dirs — or you're debugging something else ## Reading the trace diff --git a/skills/create-dataflow-approximation/references/java.md b/skills/create-dataflow-approximation/references/java.md index 125fabdef3..b42ff19338 100644 --- a/skills/create-dataflow-approximation/references/java.md +++ b/skills/create-dataflow-approximation/references/java.md @@ -74,7 +74,7 @@ Run `test approximation run` over the compiled test project applying this batch' ```bash opentaint test approximation run .opentaint/test-compiled/ \ -o .opentaint/test-results/ \ - --dataflow-approximations .opentaint/dataflow/ + --java-models .opentaint/dataflow/ ``` `test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI builds the approximation project first. if compilation fails it reports the errors and aborts before the tests. A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: diff --git a/skills/create-pass-through-approximation/references/go.md b/skills/create-pass-through-approximation/references/go.md new file mode 100644 index 0000000000..9cf1336dcb --- /dev/null +++ b/skills/create-pass-through-approximation/references/go.md @@ -0,0 +1,140 @@ +# create-pass-through-approximation — Go + +Go has only this one approximation kind — there is no dataflow approximation to fall back on. + +## Workflow + +### 1. Read the method's source + +Go dependencies ship as source, not bytecode. An application-internal function sits in the project's own sources under the project root. A dependency's source is in the module cache (`$(go env GOMODCACHE)/@/…`) or in `vendor//` when the project vendors; standard-library code is under `$(go env GOROOT)/src//`. `go doc .` and `go doc -src .` resolve a name without hunting for the file. + +The plan's `method` names the callee — `.` for a plain function, `().` for a method — and its `signature` is only `args:`; the `function:` matcher below is built from the `method` string, never from `signature`. + +### 2. Write the config + +Each file opens with a `language: go` header line above `passThrough:` — without it, or with a mismatched one, the loader silently skips the whole file: no load error, and every function in it stays dropped. Below it, a config is one top-level `passThrough` list, one entry per `function` with its `copy` edges. + +Positions (the base of any `from`/`to`) +- `this` (the receiver, methods only), `result`, `arg(0)`, `arg(1)`, … +- `result(N)` — Go returns multiple values; address them by index, `result(0)`, `result(1)` +- `[*]` — slice/array element; as a list item it must be quoted (`- '[*]'`), since unquoted it parses as a YAML alias and fails to load + +Access-path modifiers (list form `[, , …]`) +- A list is ALWAYS `[, , …]` — element 0 is the base position, every later element is a modifier. It is never a list of destinations: to send one source to several destinations, write one `copy` edge per destination +- `..#` — a struct field or a virtual slot. The slot name is nominal — the engine never resolves it, so it need not be a real field +- `` — a container's element slot; `` and `` are accepted aliases and all three collapse to the same single accessor (key and value are not distinguished) +- `` — a pointer's pointee +- there is no `.*` all-fields wildcard — always name an explicit field or slot + +Function matching — the structured block only: + +```yaml +function: + package: net/http # the import path + type: Request # the receiver type, methods only, leading '*' stripped + name: FormValue + receiver: true # false for a package-level function +``` + +Go has no overloads, so there is no `signature` key and no overload selection. There are no `overrides` and **no `condition:` support** — the loader parses only `function`/`copy`/`from`/`to` and the position modifiers, so `condition`, `typeIs`, `isConstant` and friends do not load. + +Simplest form — a direct copy from one position to another: + +```yaml +language: go +passThrough: +- function: + package: net + name: JoinHostPort + receiver: false + copy: + - from: arg(0) + to: result + - from: arg(1) + to: result +- function: + package: net + type: IPConn + name: ReadFromIP + receiver: true + copy: + - from: this + to: arg(0) +``` + +Multiple return values — one edge per tainted return position: + +```yaml +language: go +passThrough: +- function: + package: net + name: SplitHostPort + receiver: false + copy: + - from: arg(0) + to: result(0) + - from: arg(0) + to: result(1) +``` + +Virtual slot — when data lives in a struct between calls, the writer and the reader route through an access path. Both must name the **identical** `package.Type#field` accessor, or the taint drops: + +```yaml +language: go +passThrough: +- function: + package: container/list + type: List + name: PushBack + receiver: true + copy: + - from: arg(0) + to: + - this + - .container/list.List# +- function: + package: container/list + type: List + name: Front + receiver: true + copy: + - from: + - this + - .container/list.List# + to: + - result + - .container/list.Element#Value +``` + +### 3. Re-check your configs + +Beyond the body's checks, confirm the file still opens with `language: go` and that each matcher's `package`/`type`/`name`/`receiver` matches the `method` string it came from. + +## Verification is the scan + +Only when this skill runs standalone — never under the orchestrator, whose scan phase already verifies the config — a compiled model on hand catches YAML load/parse errors early. It proves the file loads, nothing about propagation: + +```bash +opentaint scan --project-model \ + -o .opentaint/test-results//passthrough-loadcheck.sarif \ + --ruleset builtin \ + --passthrough-models +``` + +A config error aborts the scan with the parse/load message. + + +There is no test project for a passThrough. The project scan applies the config and the scan agent reports back; the failure shapes map to distinct causes: + +- *every* function in the file still dropped, with no load error → the whole file was silently skipped: check the `language: go` header +- a *single* function still dropped → the `function` matcher didn't match (check `package`, `type`, `name`, `receiver`), or `from`/`to` doesn't land on the tainted position +- the method is no longer dropped but the flow still doesn't surface → most often a broken channel: writer and reader naming different `package.Type#field` accessors +- a load / parse error → fix the YAML + +After ~2 fix rounds with matcher fields and accessors verified and the method no longer dropped, stop guessing: Go has no dataflow fallback, so a propagation a passThrough genuinely can't express is an engine issue — report non-convergence for escalation to debug-rule / report-analyzer-issue. + +## Gotchas + +- User passThroughs **EXTEND** (concatenate with) Go's built-in configs — both your rule and any built-in matching the same function apply; there is no override. Check `approximated-external-methods.yaml` before modeling so you don't redundantly re-model something already covered +- The engine is field-sensitive: route data field-to-field as the source does rather than tainting the whole struct diff --git a/skills/create-rule/SKILL.md b/skills/create-rule/SKILL.md index 9f8ae12700..1e20b877b1 100644 --- a/skills/create-rule/SKILL.md +++ b/skills/create-rule/SKILL.md @@ -55,7 +55,7 @@ Run the rule tests directly as a foreground, blocking command and wait for exit opentaint test rule run .opentaint/test-compiled// \ -o .opentaint/test-results// \ --ruleset .opentaint/rules --ruleset .opentaint/test-projects///test-rules \ - --passthrough-approximations .opentaint/pass-through + --passthrough-models .opentaint/pass-through ``` `test rule run` auto-loads the built-in rules, so pass only your custom rulesets. Apply the passthrough approximations as-is, an empty one is harmless. Read the result with the bundled script — it prints the pass/fail counts and names the failing samples, so you never parse the JSON by hand: diff --git a/skills/create-rule/references/debugging.md b/skills/create-rule/references/debugging.md index e6d8f8ad17..b280d46ea7 100644 --- a/skills/create-rule/references/debugging.md +++ b/skills/create-rule/references/debugging.md @@ -15,7 +15,7 @@ opentaint test rule reachability \ - `` — the one rule whose sample routes taint through the code under test (`.yaml:`). One rule per run — across many rules the trace is unusably huge. Its library `refs` are collected automatically - read the sibling `/debug-ifds-fact-reachability.sarif`, not the `-o` file. The `-o` SARIF only shows default scan output. The sibling holds the per-instruction facts that show where taint dies -- apply the approximations the failing run used so the trace matches it: `--passthrough-approximations ` and/or `--dataflow-approximations `. Taint dying at an approximated call then means that approximation isn't propagating +- apply the approximations the failing run used so the trace matches it: `--passthrough-models ` and/or `--java-models `. Taint dying at an approximated call then means that approximation isn't propagating - debug the exact run that showed the problem — same model, rulesets, approximation dirs — or you're debugging something else ## Reading the trace @@ -35,7 +35,7 @@ When a positive won't pass and the suspicion is a library method on its flow dro opentaint scan --project-model .opentaint/test-compiled// \ -o .opentaint/test-results///diag.sarif \ --ruleset builtin --ruleset .opentaint/rules --ruleset .opentaint/test-projects///test-rules \ - --passthrough-approximations .opentaint/pass-through \ + --passthrough-models .opentaint/pass-through \ --track-external-methods ``` diff --git a/skills/create-rule/references/go-semgrep-examples.md b/skills/create-rule/references/go-semgrep-examples.md new file mode 100644 index 0000000000..93f19cc11f --- /dev/null +++ b/skills/create-rule/references/go-semgrep-examples.md @@ -0,0 +1,238 @@ +# Go Semgrep rule examples + +A catalog of real, working Go rules trimmed from `rules/ruleset/go/`. Each entry shows the +shape to copy when authoring a new Go rule. Use these as the concrete starting point instead +of inventing patterns. Produced rules stay comment-free — the prose here is reference only. + +## Go pattern syntax cheat-sheet + +- Typed receiver / variable binding: `($R : *http.Request)` — binds `$R` to a value of the + given Go type (note the pointer `*`). Use it to pin a method/field to its receiver type. +- Metavariables: `$UNTRUSTED`, `$SQL`, `$NAME`, `$P`, `$K`, … — `$UPPER` names. +- Method / field access: `$R.Method($A)`, `$R.Field`, index `$R.Header[$K]`. +- Variadic tail: `...` matches the rest of the args, e.g. `exec.Command($NAME, ...)`. +- Lib source rules: `mode: taint`, `options: {lib: true}`, `pattern-sources` with + `label: "$UNTRUSTED"`. +- Lib sink rules: `mode: taint`, `options: {lib: true}`, `pattern-sinks`; use + `focus-metavariable:` to point at the tainted argument. +- Package guard: wrap the source/sink patterns in a `patterns:` block led by a + `pattern-inside: import "pkg"` guard so bare calls don't fire in unrelated files — the + built-in multi-pattern lib rules all do this; a single-statement rule can instead inline + the `import "pkg"` line into a bare `pattern: |`. +- Security rules: `mode: join`, `metadata.cwe` + `short-description`, `refs:`/`on:` wiring + `src.$UNTRUSTED -> sink.$UNTRUSTED`. +- Structural rules (no dataflow): `mode: search` + `pattern-either` — flags a call site + regardless of taint (weak crypto/hash/random). + +## Source lib rule — HTTP request data (`go/lib/http-sources.yaml`) + +```yaml +rules: + - id: http-sources + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted user input originates here + mode: taint + pattern-sources: + - label: "$UNTRUSTED" + patterns: + - pattern-inside: | + import "net/http" + import "os" + ... + - pattern-either: + - pattern: "($R : *http.Request).FormValue($K)" + - pattern: "($R : *http.Request).PostFormValue($K)" + - pattern: "($R : *http.Request).Cookie($K)" + - pattern: $R.URL.Query().Get($K) + - pattern: $R.URL.Path + - pattern: $R.Header.Get($K) + - pattern: $R.Body + - pattern: os.Getenv($K) + - pattern: os.Args +``` + +Notes: lib sources are `severity: NOTE` and bind the tainted value as the labelled +`$UNTRUSTED`. The `patterns:` block scopes matching to the imported packages via +`pattern-inside: import "..."`. Real file lists many more request accessors and framework +variants (beego); add the project-used ones, don't enumerate the whole API. + +## Sink lib rule — SQL execution (`go/lib/sql-sinks.yaml`) + +```yaml +rules: + - id: sql-sinks + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted data reaches a SQL execution API + mode: taint + pattern-sinks: + - patterns: + - pattern-inside: | + import "database/sql" + ... + - pattern-either: + - pattern: "($DB : *sql.DB).Query($SQL, ...)" + - pattern: "($DB : *sql.DB).QueryContext($CTX, $SQL, ...)" + - pattern: "($DB : *sql.DB).Exec($SQL, ...)" + - focus-metavariable: $SQL +``` + +Notes: the `patterns:` block pairs a `pattern-inside: import "database/sql"` guard with a +`pattern-either` of the call shapes, then `focus-metavariable: $SQL` selects the tainted +argument. The receiver type `*sql.DB` pins the method. Lib sinks are `severity: NOTE`. + +## Sink lib rule — OS command execution (`go/lib/cmdi-sinks.yaml`) + +```yaml +rules: + - id: cmdi-sinks + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted data reaches an OS command execution API + mode: taint + pattern-sinks: + - patterns: + - pattern-inside: | + import "os/exec" + import "os" + import "syscall" + ... + - pattern-either: + - pattern: exec.Command($NAME, ...) + - pattern: exec.CommandContext($CTX, $NAME, ...) + - pattern: syscall.Exec($NAME, ...) + - focus-metavariable: $NAME + - patterns: + - pattern-inside: | + import "os/exec" + ... + - pattern-either: + - pattern: "exec.Command(\"$NAME\", ..., $UNTRUSTED, ...)" + - metavariable-regex: + metavariable: $NAME + regex: sh + - focus-metavariable: $UNTRUSTED +``` + +Notes: each sink shape is a `patterns:` block guarded by `pattern-inside: import "os/exec"`. +The first focuses the command name (`$NAME`); the second matches a shell interpreter invoked +with a literal name and narrows it with `metavariable-regex` before focusing the tainted +argument (`$UNTRUSTED`). Point `focus-metavariable:` at the tainted position when it isn't +first. + +## Sink lib rule — filesystem path (`go/lib/path-sinks.yaml`) + +```yaml +rules: + - id: path-sinks + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted data reaches a filesystem path API + mode: taint + pattern-sinks: + - patterns: + - pattern-inside: | + import "os" + import "io/ioutil" + import "net/http" + ... + - pattern-either: + - pattern: os.Open($P) + - pattern: os.OpenFile($P, ...) + - pattern: os.ReadFile($P) + - pattern: os.WriteFile($P, ...) + - pattern: ioutil.ReadFile($P) + - pattern: http.ServeFile($W, $R, $P) +``` + +## Security join — SQL injection (`go/security/sql-injection.yaml`) + +```yaml +rules: + - id: sql-injection + languages: [go] + severity: ERROR + message: Tainted user input flows into SQL query (SQL injection) + metadata: + cwe: CWE-89 + short-description: SQL injection + mode: join + join: + refs: + - rule: go/lib/http-sources.yaml#http-sources + as: src + - rule: go/lib/http-sources-requesturi.yaml#http-sources-requesturi + as: extra_requesturi + - rule: go/lib/sql-sinks.yaml#sql-sinks + as: sink + on: + - 'src.$UNTRUSTED -> sink.$UNTRUSTED' + - 'extra_requesturi.$UNTRUSTED -> sink.$UNTRUSTED' +``` + +## Security join — others (same shape, swap the sink ref + CWE) + +| Class | id | CWE | sink ref | +|---|---|---|---| +| Command injection | `command-injection` | CWE-78 | `go/lib/cmdi-sinks.yaml#cmdi-sinks` | +| Path traversal | `path-traversal` | CWE-22 | `go/lib/path-sinks.yaml#path-sinks` | +| SSRF | `ssrf` | CWE-918 | `go/lib/ssrf-sinks.yaml#ssrf-sinks` | +| Reflected XSS | `reflected-xss` | CWE-79 | `go/lib/xss-sinks.yaml#xss-sinks` | +| SSTI | `ssti` | CWE-1336 | `go/lib/ssti-sinks.yaml#ssti-sinks` | + +Each is one `src.$UNTRUSTED -> sink.$UNTRUSTED` join refing `http-sources` (plus the +extra header/requesturi source variants where the class needs them). + +## Structural rule — weak hash / weak random (`mode: search`, no dataflow) + +```yaml +rules: + - id: weak-hash-cwe-328 + languages: [go] + severity: ERROR + message: Use of a weak or broken cryptographic hash + metadata: + cwe: CWE-328 + short-description: Weak cryptographic hash + mode: search + patterns: + - pattern-inside: | + import "crypto/md5" + import "crypto/sha1" + ... + - pattern-either: + - pattern: md5.New(...) + - pattern: md5.Sum(...) + - pattern: sha1.New(...) + - pattern: sha1.Sum(...) +``` + +```yaml +rules: + - id: weak-random-cwe-330 + languages: [go] + severity: ERROR + message: Use of a weak (non-cryptographic) random number generator + metadata: + cwe: CWE-330 + short-description: Insecure randomness + mode: search + patterns: + - pattern-inside: | + import "math/rand" + ... + - pattern-either: + - pattern: rand.Intn(...) + - pattern: rand.Int63() + - pattern: rand.Float64() +``` + +Notes: `mode: search` flags a call site structurally — no source/sink join, no taint flow. +The structural rule still guards on `pattern-inside: import "..."` so it only fires for the +intended package. Use it for "dangerous primitive used at all" rules; use `mode: join` when +the finding requires untrusted data to reach the sink. diff --git a/skills/create-rule/references/go.md b/skills/create-rule/references/go.md new file mode 100644 index 0000000000..33afacbf21 --- /dev/null +++ b/skills/create-rule/references/go.md @@ -0,0 +1,120 @@ +# create-rule — Go + +Language reference for authoring Go source/sink library rules, keyed to the body's steps. Bind every tainted value to `$UNTRUSTED` so the joins connect on one name. + +Worked, repo-faithful examples for every vuln class live in `references/go-semgrep-examples.md` — read it before authoring and copy the closest shape. + +## Workflow + +### 1. Check existing coverage + +Built-in Go rules live under `go/lib/` (source and sink library rules) and `go/security/` (the joins and structural rules); mirror that layout for any custom rule you add — never the `java/` tree. Rule ids carry no `go-` prefix. + +### 2. Author the library rules + +Go lib rules are `severity: NOTE` with `options: {lib: true}` and `mode: taint`, carry `languages: [go]`, use the typed-receiver pattern syntax (`($R : *http.Request)` — the `*` only where the API is pointer-receiver), and bind the tainted value as `$UNTRUSTED`. + +Scope every rule to its package by wrapping the patterns in a `patterns:` block led by a `pattern-inside: import "pkg"` guard — the built-in multi-pattern lib rules all do this, so a bare call like `os.Getenv` or `exec.Command` can't fire in an unrelated file. A single-statement rule can instead inline the `import "pkg"` line into a bare `pattern: |`, as `go/lib/http-sources-requesturi.yaml` does. + +Custom source library rule (`.opentaint/rules/go/lib/my-source.yaml`): + +```yaml +rules: + - id: mylib-source + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted user input originates here + mode: taint + pattern-sources: + - label: "$UNTRUSTED" + patterns: + - pattern-inside: | + import mylib "github.com/acme/mylib" + ... + - pattern-either: + - pattern: "($C : *mylib.Ctx).UserInput($K)" +``` + +Custom sink library rule (`.opentaint/rules/go/lib/my-sink.yaml`) — `focus-metavariable` points at the tainted argument: + +```yaml +rules: + - id: mylib-sink + options: {lib: true} + languages: [go] + severity: NOTE + message: Untrusted data reaches a dangerous operation + mode: taint + pattern-sinks: + - patterns: + - pattern-inside: | + import mylib "github.com/acme/mylib" + ... + - pattern-either: + - pattern: "($E : *mylib.Engine).Render($TPL, ...)" + - focus-metavariable: $TPL +``` + +### 3. Write the test joins + +**Go deviates here.** The Go test harness has no generic taint marker and no throwaway test-rules tree: a sample wires a real source into a real sink, so what the samples run against is the *real* security join, written once under `.opentaint/rules/go/security/` and named by the `rule-test.yaml` manifest as `go/security/.yaml#`. There is nothing test-only to write and nothing to keep out of the scanned rules tree. + +```yaml +rules: + - id: my-ssti-lib-ext + languages: [go] + severity: ERROR + message: Untrusted input reaches a template engine (SSTI) + metadata: + cwe: CWE-1336 + short-description: Server-side template injection + mode: join + join: + refs: + - rule: go/lib/http-sources.yaml#http-sources + as: src + - rule: go/lib/my-ssti-sink.yaml#mylib-sink + as: sink + on: + - 'src.$UNTRUSTED -> sink.$UNTRUSTED' +``` + +For a purely structural rule — a dangerous primitive used at all, such as weak hash or weak random — use `mode: search` + `pattern-either` instead of a join (see the catalog); it needs no source side. + +### 4. Test until success + +There is one shared Go test module and one compiled model, not per-unit `sinks/`/`sources/` sides, so the command names them directly: + +```bash +opentaint test rule run .opentaint/test-compiled/go \ + -o .opentaint/test-results/go \ + --ruleset .opentaint/rules \ + --passthrough-models .opentaint/pass-through +``` + +`test rule run` auto-loads the built-in rules, so pass only your own ruleset — a literal `builtin` here would be treated as a path. There is no `--java-models` for Go. Read the result with the bundled `check-test-result.py`, passing `go` as the unit. + +Pattern operators for each verdict: + +- `falseNegative` → broaden `pattern-either`, check the receiver type (and its `*`), and confirm `$UNTRUSTED` matches across the lib rules and the join's `on:` +- `falsePositive` → narrow with `pattern-not`, `pattern-not-inside`, `pattern-sanitizers`, or `metavariable-regex`, or focus a different sink argument. Inspect the flow that fired with `opentaint summary .opentaint/test-results/go/test-results.sarif --show-findings` +- `skipped` → the manifest entry's rule-id or sample full name is wrong; fix the `rule-test.yaml` entry + +### 5. Escalate when a positive won't converge + +Unchanged, with one Go difference: a dropped library function on the failing sample's path routes to a Go **passThrough** (there is no dataflow approximation), and a re-dispatch applies it with `--passthrough-models`. + +## Constraints + +- Lib rules (source and sink) are `severity: NOTE`, `options: {lib: true}`, `mode: taint`; `severity: ERROR` belongs to the security rules — joins and structural `mode: search` rules alike +- Security joins (`mode: join`) must carry `metadata.cwe` and `metadata.short-description` +- Rule ids are globally unique and carry no language prefix; a custom join id must not collide with a built-in (`sql-injection`, `ssrf`, …) or it is dropped silently +- Custom Go rules go under `.opentaint/rules/go/{lib,security}/`, never the `java/` tree + +## Gotchas + +- The typed-receiver form needs the pointer where the API is pointer-receiver (`($R : *http.Request)`); a value receiver omits the `*` +- Without the `pattern-inside: import "pkg"` guard a bare call pattern fires in unrelated files +- A wrong argument position in a sink focuses the wrong parameter — point `focus-metavariable` at the tainted one +- Don't unpack or grep the analyzer JAR or the go-ssa-server binary for built-in rules — their internals aren't a stable API; read the YAMLs from the path `opentaint health --rules` prints diff --git a/skills/create-test-project/references/go-rule.md b/skills/create-test-project/references/go-rule.md new file mode 100644 index 0000000000..74ecf6be60 --- /dev/null +++ b/skills/create-test-project/references/go-rule.md @@ -0,0 +1,105 @@ +# create-test-project — Go rule test project + +For `type: rule-source` / `rule-sink` on a Go project. There is no Go dataflow approximation, so `type: dataflow` never reaches this language. + +**Go deviates from the body in three ways.** There is one shared test module for every Go rule test — not one project per unit, and no `sources/`/`sinks/` sub-projects. There is no scaffold command and no generic taint marker: a sample wires a *real* source into a *real* sink, and the manifest names the *real* rule under test. Everything else — one verdict per sample, extend rather than re-scaffold, compile as the deliverable — is unchanged. + +## Workflow + +### 1. Init or extend the module + +The module lives at `.opentaint/test-projects/go` and is an ordinary Go module. Extend it for each new unit rather than creating another: + +- `go.mod` — `module test`, a `go` line, and for any third-party API the samples import a `require ` plus a local `replace => ./stubs/`. Stub a heavy dependency to a minimal local package rather than vendoring the library. The stub directory must itself be a **nested Go module**: `stubs//go.mod` whose `module` line is the **exact real import path** being stubbed (e.g. `module github.com/beego/beego/v2`), holding only the API surface the samples use. A bare directory of `.go` files fails module resolution when `opentaint compile` runs `go` +- samples under a category package per unit, e.g. `security//sample.go`. The analyzer references a function by its **import path + name** — `test/security/.PositiveSourceRequestFormValue` +- small helper source/sink funcs so each sample stays one flow — `requestForSources()` returning `*http.Request`, `envSource()` returning a tainted string, `sqlSink(v)` calling `db.Query(fmt.Sprint(v))` + +### 2. Write the samples + +Read the real Go source/sink API from the dependency or stdlib first, so each sample is built on the signature the rule actually matches. One function, one verdict, named by convention: + +- **positive** (`PositiveXxx`) — a minimal flow that must flag, routing a real source into the real sink +- **negative** (`NegativeXxx`) — the same shape neutralized: a sanitizer between source and sink, or a safe/parameterized sink + +Prefix a sample whose behaviour the engine is known not to support yet with `Unsupported` so it isn't counted a strict pass/fail. + +```go +package allpatterns + +import ( + "database/sql" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" +) + +var db *sql.DB + +func requestForSources() *http.Request { + return &http.Request{ + URL: &url.URL{RawQuery: "q=v"}, + Form: url.Values{"q": []string{"form"}}, + Body: io.NopCloser(strings.NewReader("body")), + } +} + +func sqlSink(value interface{}) { + _, _ = db.Query(fmt.Sprint(value)) +} + +func envSource() string { + return os.Getenv("TAINTED") +} + +func PositiveSourceRequestFormValue() { + r := requestForSources() + sqlSink(r.FormValue("q")) +} + +func PositiveSourceURLQueryGet() { + r := requestForSources() + sqlSink(r.URL.Query().Get("q")) +} + +func NegativeSQLParameterizedArgument() { + _, _ = db.Query("select * from users where name = ?", envSource()) +} + +func NegativeSQLConstantQuery() { + _, _ = db.Query("select * from users where active = 1") +} +``` + +Expected verdicts go in `.opentaint/test-projects/go/rule-test.yaml`, keyed by the **real** rule-id (`go/security/.yaml#`, the file and id create-rule gave the join) and listing each sample by its `import-path.FuncName` full name — not the Go `package` name: + +```yaml +tests: + - rule-id: go/security/sql-injection.yaml#sql-injection + positive: + - test/security/all-patterns.PositiveSourceRequestFormValue + - test/security/all-patterns.PositiveSourceURLQueryGet + negative: + - test/security/all-patterns.NegativeSQLParameterizedArgument + - test/security/all-patterns.NegativeSQLConstantQuery +``` + +### 3. Compile to the model + +One module, one model — no per-side sub-model: + +```bash +opentaint compile .opentaint/test-projects/go -o .opentaint/test-compiled/go +``` + +The model is portable: the compile copies the module into `.opentaint/test-compiled/go/go_0` and records the relative `projectDir: go_0`, so `test rule run` resolves `rule-test.yaml` and the sample functions from that copy, not from the live sources. Recompile after every sample or manifest edit; `-o` must not already exist, so delete the old model or compile to a fresh directory. `go` must be on PATH. + +## Gotchas + +- The full name in `rule-test.yaml` is `import-path.FuncName` (the directory path, e.g. `test/security/all-patterns.Foo`), not the Go `package` name (`allpatterns`) +- A name that resolves to no function is fatal to the **whole run**, not just that sample: resolution is an exact `fullName` match with no fuzzy fallback, and one unresolved entrypoint aborts every test with an opaque analyzer exception and no per-sample results. After `compile`, verify every name matches a compiled function before `test rule run` +- Keep the `go.mod` `go` directive (and each stub's) at or below the installed toolchain. Under the default `GOTOOLCHAIN=auto` a newer directive makes `go` try to **download** a toolchain, which fails offline; the shared module tracks `go 1.22` +- One module serves every Go rule test, so parallel agents share `go.mod`, the sample packages, and `rule-test.yaml`. Give each unit its own category dir/package, keep `rule-test.yaml` edits append-only, and don't concurrently edit a shared helper file or the same `require`/`replace` lines +- A positive must route the helper source into the real sink — a sink fed a constant or a bare parameter with no in-sample source can't be flagged by a taint rule diff --git a/skills/debug-rule/SKILL.md b/skills/debug-rule/SKILL.md index 0907d496d4..e31fb67e37 100644 --- a/skills/debug-rule/SKILL.md +++ b/skills/debug-rule/SKILL.md @@ -30,22 +30,24 @@ opentaint test rule reachability \ --project-model \ -o /report.sarif \ --ruleset builtin --ruleset .opentaint/rules \ - --passthrough-approximations .opentaint/pass-through \ - --dataflow-approximations .opentaint/dataflow + --passthrough-models .opentaint/pass-through \ + --java-models .opentaint/dataflow ``` +`--java-models` is Java/JVM only — a Go run passes `--passthrough-models` alone. + `` is `.opentaint/test-results/` for a test model, `.opentaint/results` for the main scan. The per-instruction facts are in the sibling `/debug-ifds-fact-reachability.sarif`, not the `-o` file — the `-o` SARIF only shows whether the rule fired. Read that sibling to find the kill: - a missed detection (a positive that won't pass, or a flow absent from a scan) — confirm a fact exists at the source; if none, the gap is in `pattern-sources`, not the flow. Otherwise walk the facts to the last instruction still carrying it and the first where it's gone — that gap is the kill - a spurious detection (a negative that fires) — the reverse: find where a fact appears with no tainted input reaching it -Trace the exact run that misbehaved — a different `model` or ruleset traces something else; taint dying at an approximated call means that approximation isn't propagating. When the flow is missed and the entry method may never be analyzed, rerun with `--entry-points ""`: a finding that appears only then is an entry-point-discovery problem, not dataflow. On Spring the flag is additive — auto-discovered endpoints stay and your method is added, so use it to force-include an endpoint the analyzer never starts from, not to narrow to one method. +Trace the exact run that misbehaved — a different `model` or ruleset traces something else; taint dying at an approximated call means that approximation isn't propagating. When the flow is missed and the entry method may never be analyzed, rerun with `--entry-points ""`: a finding that appears only then is an entry-point-discovery problem, not dataflow. On Spring the flag is additive — auto-discovered endpoints stay and your method is added, so use it to force-include an endpoint the analyzer never starts from, not to narrow to one method. On Go, `--entry-points` takes a function full name in `package.FunctionName` form, not the JVM `Class#method` form. ### 2. Classify the cause The killing instruction decides who owns the fix. An engine bug is by far the least likely — assume it last, only once the other two are ruled out; nearly every kill is a missing or wrong library model or a rule defect, both tedious to exclude but far more probable, and the tedium is no reason to jump to "engine". Three outcomes: -- the kill is at an external library method → a model issue. Cross-check `dropped-external-methods.yaml` from that run (a `--track-external-methods` scan regenerates it if absent): listed there means the method is unmodeled — the missing model is the cause, for the approximation stage to model. Not listed but a built-in claims to model it, yet taint dies here → that model is wrong for this case: a passThrough override applies at the rule level, so prefer one for the method; a dataflow override conflicts with built-ins at load, so fall back to a passThrough, or call it an engine issue when only a dataflow shape can express the propagation +- the kill is at an external library method → a model issue. Cross-check `dropped-external-methods.yaml` from that run (a `--track-external-methods` scan regenerates it if absent): listed there means the method is unmodeled — the missing model is the cause, for the approximation stage to model. Not listed but a built-in claims to model it, yet taint dies here → that model is wrong for this case: a passThrough override applies at the rule level, so prefer one for the method; a dataflow override conflicts with built-ins at load, so fall back to a passThrough, or call it an engine issue when only a dataflow shape can express the propagation. On Go there is no dataflow approximation at all: the fix is a passThrough, or an engine issue when no passThrough can express the propagation — never a dataflow fallback - the kill is where the rule should have matched — a sanitizer misfires, a sink or source variant went unmatched → a rule defect, for rule authoring to fix - the kill is a plain instruction the engine must propagate through (assignment, cast, field read, an already-modeled call), with the rule correct and the model complete → an engine issue diff --git a/skills/discover-attack-surface/references/go.md b/skills/discover-attack-surface/references/go.md new file mode 100644 index 0000000000..5be895c49a --- /dev/null +++ b/skills/discover-attack-surface/references/go.md @@ -0,0 +1,36 @@ +# discover-attack-surface — Go + +`go` must be on PATH — the member extraction and every signature lookup drive the Go toolchain. + +## Workflow + +### 1. Settle built-in coverage + +Built-in Go source and sink rules live under `go/lib/` within the `opentaint health --rules` root — grep there for the member's package or method name (a rule is a semgrep-like pattern, not indexed by import path), then read the matched rule file to confirm it covers the member. The project's own custom rules are under `.opentaint/rules/go`. + +Go untrusted-data sources are declared by those `go/lib/*-sources.yaml` rules, not by framework endpoint auto-discovery: an inbound source the built-in rules already match needs no new source, so focus on the boundaries they miss and on the sinks. + +### 2. Classify the plan's members + +Each plan member is `{ method, signature }` where `method` is `.` and `signature` is empty — Go has no descriptor and no overloads, so a member is identified by its name alone. + +Go dependencies are source on disk, not resolved jars. Locate a module's source to read signatures: + +- `go mod download -json ` prints the extracted source dir as its `Dir` field +- or `$(go env GOMODCACHE)/@/` — uppercase letters in a module path are case-escaped as `!`, so `github.com/Azure` is `github.com/!azure` +- or `vendor//` when the project vendors + +Run `go` commands from a module directory (a `goProjects.projectDir` from `project.yaml`). `go doc .` prints the exported signature and receiver; `go doc -src` prints its body. + +**The plan's scope is an entry scope, not the whole surface.** The extractor sees only package-qualified selectors (`gin.New`, `jwt.Parse`); it is structurally blind to method calls on receiver values (`c.Query(…)`, `db.QueryContext(…)`) — the dominant Go idiom and the exact shape of most accessor sources and sinks — and to members reached through interface dispatch, reflection, struct-embedding promotion, router/handler registration, config strings, or generated code absent from the source tree. Treat the plan as *which packages, constructors, and package-level functions the project touches*, then read the app source and the dependency's source to add the methods called on those values and the indirectly reached members, and classify those too. Do not enumerate the whole module API. + +Inspect only the project's own source and the dependency's source — never the analyzer JAR or the go-ssa-server binary. + +Classifying: + +- **sources** — where untrusted data first enters from a boundary: a method that *returns* attacker-controlled data (`(*http.Request).FormValue`, a framework context accessor, a message-broker payload). Not a method that merely passes along data it was handed — that is a propagator the engine already handles +- **sinks** — dangerous operations (query construction, command/file/path ops, deserialization, template/SSTI, SSRF request building, reflection), each tagged with its vuln class + +### 3. Write the source units + +`dependencies` is the Go module identity, `@` (e.g. `github.com/gin-gonic/gin@v1.10.0`) — read the version from the project's `go.mod`, not from the import path. diff --git a/skills/generate-poc/references/go.md b/skills/generate-poc/references/go.md new file mode 100644 index 0000000000..9ebb519dff --- /dev/null +++ b/skills/generate-poc/references/go.md @@ -0,0 +1,17 @@ +# generate-poc — Go + +Language reference for building and starting a Go app to reproduce against, keyed to the body's step. + +## Workflow + +### 1. Start the app + +Build and start the app the way the project expects, then wait until the port is listening: + +- from source — `go run ./...`, or `go run ./cmd/` when the module has several commands +- a built binary — `go build -o app . && ./app` +- a containerized app — `docker compose up` + +`go` must be on PATH, and the module's `go`/`toolchain` directive must be satisfiable — under the default `GOTOOLCHAIN=auto` a newer directive makes `go` try to download a toolchain, which fails offline. + +Loopback bind (per the body's Constraints): a Go server's listen address is its own flag or env var — pass `127.0.0.1:` to whatever the app hands `http.ListenAndServe` / `net.Listen` rather than a bare `:`, which binds the wildcard. For a container, `docker run -p 127.0.0.1:8080:8080 …` or a compose override on the port mapping. diff --git a/skills/orchestrate-stage/references/approx-round.md b/skills/orchestrate-stage/references/approx-round.md index 95c1b0375a..8bcbcad8df 100644 --- a/skills/orchestrate-stage/references/approx-round.md +++ b/skills/orchestrate-stage/references/approx-round.md @@ -37,7 +37,10 @@ It merges every batch's `skipped` / `engine_issues` into `approximations/skipped - `language` - `batch` - `methods` (optional) — only an explicitly assigned subset to redo -- dataflow step 1 → dispatch create-test-project + +On Go the dataflow branch never runs: there is no dataflow approximation, so a Go batch's `dataflow` bucket stays empty, no test project is scaffolded for it, and every carrier is a passThrough. A Go carrier that won't converge goes straight to escalation. + +- dataflow step 1 (Java/JVM only) → dispatch create-test-project - `language` - `type: dataflow` - `batch` diff --git a/skills/orchestrate-stage/scripts/generate.py b/skills/orchestrate-stage/scripts/generate.py index 53e6b4488d..aba5c73647 100644 --- a/skills/orchestrate-stage/scripts/generate.py +++ b/skills/orchestrate-stage/scripts/generate.py @@ -118,8 +118,9 @@ def root_of(fqn, depth=ROOT_DEPTH): def in_packages(cls, prefixes): - # dotted-boundary match: `a.b.collect` never matches a sibling `a.b.collectX` - return any(cls == p or cls.startswith(p + ".") for p in prefixes) + # dotted-boundary match: `a.b.collect` never matches a sibling `a.b.collectX`. `/` is a + # boundary too, so a Go module prefix also covers its subpackages (`mod/sub.Member`) + return any(cls == p or cls.startswith(p + ".") or cls.startswith(p + "/") for p in prefixes) def atomize(fqns, cap): @@ -283,6 +284,86 @@ def is_project_class(cls, packages): def extract_usages(): + # (fqn, signature) pairs for the members the project's own code calls; the model decides + # which extractor applies — a Go model has no bytecode to disassemble + doc = load_yaml(MODEL / "project.yaml", {}) or {} + if doc.get("goProjects") and not doc.get("javaProjects"): + return extract_usages_go(doc) + return extract_usages_java() + + +GO_IMPORT_RE = re.compile(r'^\s*(?:import\s+)?([\w.]+)?\s*"([^"]+)"') + + +def go_module_dirs(doc): + dirs = [] + for entry in doc.get("goProjects") or []: + d = Path(str((entry or {}).get("projectDir") or "")) + if not str(d): + continue + dirs.append(d if d.is_absolute() and d.is_dir() else MODEL / d) + return [d for d in dirs if d.is_dir()] or [MODEL] + + +def go_package_name(mod_dir, import_path, cache): + # the local identifier an import binds when the file does not alias it: the package's own + # name, which need not be the last path segment (and is never a `vN` version suffix) + if import_path in cache: + return cache[import_path] + name = "" + try: + out = subprocess.run(["go", "list", "-f", "{{.Name}}", import_path], cwd=str(mod_dir), + capture_output=True, text=True) + name = out.stdout.strip() + except OSError: + pass + if not name: + segs = import_path.split("/") + name = segs[-2] if len(segs) > 1 and re.fullmatch(r"v\d+", segs[-1]) else segs[-1] + cache[import_path] = name + return name + + +def extract_usages_go(doc): + # scan the project's own .go files for package-qualified selectors on a pending module and + # return them as `.`; Go has no descriptor, so the signature is empty. + # Structurally blind to method calls on receiver values, interface dispatch, and reflection — + # the frontier agent adds those from the source (per its language reference). + packages = pending_packages() + fqns, cache = set(), {} + for mod_dir in go_module_dirs(doc): + for f in mod_dir.rglob("*.go"): + try: + text = f.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + imports, block = [], False + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("import ("): + block = True + continue + if block and stripped.startswith(")"): + block = False + continue + if not block and not stripped.startswith("import"): + continue + m = GO_IMPORT_RE.match(line) + if not m: + continue + alias, path = m.group(1), m.group(2) + if not in_packages(path, packages): + continue + if alias in (".", "_"): + continue + imports.append((path, alias or go_package_name(mod_dir, path, cache))) + for path, ident in imports: + for member in re.findall(rf"\b{re.escape(ident)}\.([A-Z]\w*)", text): + fqns.add((f"{path}.{member}", "")) + return fqns + + +def extract_usages_java(): # disassemble project classes, collect // Method / // InterfaceMethod call sites with their # JVM descriptor; returns (fqn, signature) pairs so an overloaded member stays disambiguated fqns = set() diff --git a/skills/report-analyzer-issue/SKILL.md b/skills/report-analyzer-issue/SKILL.md index 862a9c05a5..2c670a2336 100644 --- a/skills/report-analyzer-issue/SKILL.md +++ b/skills/report-analyzer-issue/SKILL.md @@ -43,7 +43,7 @@ Header (both kinds): - Type — `analysis` or `resource` - Setup — the exact command that reproduces it: the `opentaint test rule …` / `test approximation run` for `analysis`, or the failed `opentaint scan` for `resource`, each with its model, rulesets, and approximation dirs (plus final memory and timeout/backstop outcome for `resource`) - Run logs — the run's log files, by path: for `analysis`, any debug log or fact-reachability SARIF the caller cited, if present; for `resource`, the scan's output log -- Minimal repro — the smallest set of files/folders that demonstrates it: for `analysis`, the test project `.opentaint/test-projects/` if one was named; for `resource`, the project model, rulesets, and approximation dirs in play (with their rough size) +- Minimal repro — the smallest set of files/folders that demonstrates it: for `analysis`, the test project `.opentaint/test-projects/` if one was named; for `resource`, the project model, rulesets, and approximation dirs in play (with their rough size). A Go passThrough has no test project — cite the scan evidence instead: the config under test, the modeled method now absent from `dropped-external-methods.yaml` (and present in `approximated-external-methods.yaml`), the flow still missing from the scan SARIF, and a minimal standalone Go module wiring the modeled call between a real source and sink - TL;DR — 2–3 sentences: the symptom, and for `analysis` where taint appears to die and what the engine should do instead Below the header, free-form — whatever makes the case, at least: diff --git a/skills/run-scan/SKILL.md b/skills/run-scan/SKILL.md index 221f4a67df..a37f2cde20 100644 --- a/skills/run-scan/SKILL.md +++ b/skills/run-scan/SKILL.md @@ -33,11 +33,13 @@ opentaint scan --project-model .opentaint/project \ ``` - `--rule-id ` — restrict to specific executable rules (repeatable, one per input rule ID); referenced library rules remain available without listing their IDs separately. Omit to run all loaded rules -- `--passthrough-approximations .opentaint/pass-through` — add when that directory exists: passThrough configs override built-ins at the rule level, a provided rule overriding a built-in only when it matches one -- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations, one project per batch +- `--passthrough-models .opentaint/pass-through` — add when that directory exists: passThrough configs override built-ins at the rule level, a provided rule overriding a built-in only when it matches one +- `--java-models .opentaint/dataflow` — Java/JVM only; add when that directory exists: code-based approximations, one project per batch. A Go run has no dataflow directory and never passes this flag — `--passthrough-models` is its only model flag Both approximation-dir flags walk their trees recursively; pass each parent directory once, not every package or batch separately. +A Go project also scans straight from sources — `opentaint scan ` compiles on the fly — but prefer the pre-built model so every scan in a run sees the same code. `go` must be on PATH either way. + The scan is long — run it in the background and wait for it to finish. Leave `--timeout` at the engine default (900s); the CLI ends the analysis itself and writes whatever SARIF it has. ### 2. Retry once on out-of-memory @@ -70,3 +72,4 @@ All three sit next to the report under `.opentaint/results/`: ## Constraints - Never hand-edit the project model to change scan results — this skill only reads it. If the model is wrong, rebuild it at the model stage rather than patching `project.yaml` or anything under it +- Go coverage can go missing without the scan failing. `go` absent from PATH hard-fails a Go-only scan, but in a polyglot repo it only warns and silently skips the Go analysis — the other languages scan normally and the report simply carries no Go findings. The same silence follows a model whose Go source root no longer resolves: a `--project-model` scan re-derives that root from `project.yaml` and re-detects Go against it, so a hand-written model recording an absolute `projectDir` loses Go coverage once that tree moves or loses its `go.mod` (an autobuilder model is portable — the sources are copied into `/go_0` under a model-relative `projectDir`). On a Go run, check the report actually holds Go results before treating it as complete diff --git a/skills/triage-dependencies/SKILL.md b/skills/triage-dependencies/SKILL.md index 2564ce731d..f4389e9906 100644 --- a/skills/triage-dependencies/SKILL.md +++ b/skills/triage-dependencies/SKILL.md @@ -23,10 +23,14 @@ Provided by the caller, fall back to the default value when omitted. Ask back on Read `.opentaint/project/project.yaml` — the `dependencies` list under each per-language projects entry is every third-party dependency the model resolved. Resolve each to the library it is. Most of a large project's dependencies are transitive infrastructure +A `goProjects:` entry carries no such list — read the module's own `go.mod` (`require` blocks) and `go.sum` for its dependency modules instead, and identify each by module path rather than by a coordinate + ### 2. Mark each library For each library decide: could it introduce an attacker-controlled source — a method returning untrusted data (HTTP/RPC request data, message-broker payloads, deserialized untrusted input and so on)? Judge by the library's identity itself, read sources to get overviews, docs +Dismiss the obvious infrastructure by identity — logging (logback/slf4j; Go `go.uber.org/zap`, `log/slog`), build plugins, annotations, bytecode tooling (ASM, byte-buddy) or codegen runtimes (`google.golang.org/protobuf`), test libraries (Go `github.com/stretchr/testify`), pure data structures. When unsure, peek: grep the project's sources for the library's imports or call sites — for Go that grep on the import path is the only peek available, there is no jar to inspect + ### 3. Write the flag list Write in `.opentaint/tracking/coverage.yaml` (per Tracking) a flat list of the flagged libraries' packages