From 1917358385b7edbdceea63c4f12796ec4316b450 Mon Sep 17 00:00:00 2001 From: Aleksandr Misonizhnik Date: Wed, 26 Aug 2026 19:40:52 +0200 Subject: [PATCH 1/4] feat(cli): build self-contained approximation projects Add project scaffolding, pinned dependency builds, on-demand rebuilds, compiled-output resolution, diagnostics, and documentation for custom dataflow approximations. --- cli/cmd/analyzer_inputs.go | 14 +- cli/cmd/approximation.go | 74 ++++ cli/cmd/command_builder.go | 10 +- cli/cmd/compile.go | 14 +- cli/cmd/compile_approximations.go | 184 +++------- cli/cmd/compile_approximations_cmd.go | 65 ++++ cli/cmd/scan.go | 5 +- cli/cmd/test.go | 2 +- cli/cmd/test_rule_run.go | 2 +- .../approximation/approximation_test.go | 323 ++++++++++++++++++ cli/internal/approximation/build.go | 204 +++++++++++ cli/internal/approximation/descriptor.go | 34 ++ cli/internal/approximation/project.go | 116 +++++++ cli/internal/approximation/resolve.go | 85 +++++ cli/internal/approximation/scaffold.go | 67 ++++ cli/internal/approximation/stamp.go | 90 +++++ cli/internal/autobuilder/buildlog.go | 47 +++ cli/internal/autobuilder/buildlog_test.go | 81 +++++ cli/internal/testproject/testproject.go | 12 +- cli/internal/utils/extract.go | 52 --- cli/internal/utils/java/detection.go | 13 - docs/usage.md | 35 +- .../references/java.md.j2 | 20 +- .../sections/output.md | 2 +- .../sections/workflow.md | 2 +- .../references/java-approximation.md.j2 | 2 +- .../run-scan/sections/workflow.md | 2 +- skills/create-dataflow-approximation/SKILL.md | 4 +- .../references/java.md | 20 +- .../references/java-approximation.md | 2 +- skills/run-scan/SKILL.md | 2 +- 31 files changed, 1348 insertions(+), 237 deletions(-) create mode 100644 cli/cmd/approximation.go create mode 100644 cli/cmd/compile_approximations_cmd.go create mode 100644 cli/internal/approximation/approximation_test.go create mode 100644 cli/internal/approximation/build.go create mode 100644 cli/internal/approximation/descriptor.go create mode 100644 cli/internal/approximation/project.go create mode 100644 cli/internal/approximation/resolve.go create mode 100644 cli/internal/approximation/scaffold.go create mode 100644 cli/internal/approximation/stamp.go create mode 100644 cli/internal/autobuilder/buildlog.go create mode 100644 cli/internal/autobuilder/buildlog_test.go diff --git a/cli/cmd/analyzer_inputs.go b/cli/cmd/analyzer_inputs.go index 3290a995fa..e13a7d5680 100644 --- a/cli/cmd/analyzer_inputs.go +++ b/cli/cmd/analyzer_inputs.go @@ -1,17 +1,23 @@ package cmd import ( + "github.com/seqra/opentaint/internal/approximation" "github.com/seqra/opentaint/internal/utils/log" ) -func addDataflowApproximations(b *AnalyzerBuilder, paths []string, analyzerJarPath, projectModelDir string) { +// addDataflowApproximations resolves each path to compiled model directories. +// It rebuilds a model project when its inputs change. +func addDataflowApproximations(b *AnalyzerBuilder, paths []string, analyzerJarPath string) { + builder := newApproximationBuilder(analyzerJarPath) for _, approxPath := range paths { absApproxPath := log.AbsPathOrExit(approxPath, "dataflow-approximations") - compiledPath, err := compileApproximationsIfNeeded(absApproxPath, analyzerJarPath, projectModelDir) + classDirs, err := approximation.Resolve(absApproxPath, builder) if err != nil { - out.Fatalf("Approximation compilation failed: %s", err) + out.Fatalf("Failed to apply dataflow approximations: %s", err) + } + for _, classDir := range classDirs { + b.AddDataflowApproximations(classDir) } - b.AddDataflowApproximations(compiledPath) } } diff --git a/cli/cmd/approximation.go b/cli/cmd/approximation.go new file mode 100644 index 0000000000..61d5050c88 --- /dev/null +++ b/cli/cmd/approximation.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "fmt" + + "github.com/seqra/opentaint/internal/approximation" + "github.com/seqra/opentaint/internal/utils/log" + "github.com/spf13/cobra" +) + +var initApproximationDeps []string + +var approximationCmd = &cobra.Command{ + Use: "approximation", + Short: "Create dataflow approximation projects", + Long: `A dataflow approximation project holds code-based models of how taint moves through +methods the analyzer cannot see into, together with the exact dependency versions those +models are written against. + +Compile one with 'opentaint compile approximations', and apply it with the +--dataflow-approximations flag of 'opentaint scan' or 'opentaint test approximation run'.`, +} + +var approximationInitCmd = &cobra.Command{ + Use: "init ", + Short: "Create a dataflow approximation project", + Long: `Create a Gradle project for OpenTaint dataflow approximation models. + +The project includes: + - build.gradle.kts with compile-only dependencies, settings.gradle.kts + - libs/opentaint-approximations-api.jar, the @Approximate annotations and the + OpentaintNdUtil / ArgumentTypeContext support types + - src/main/java/ for the model sources, one @Approximate class per target class + +Use --dependency to pin the libraries the models are written against, at the exact versions +the target application uses. Those pins are the models' compile environment: they, and +nothing about the project under analysis, decide what the models compile against.`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + projectPath := log.AbsPathOrExit(args[0], "output-dir") + + analyzerJarPath, err := ensureAnalyzerAvailable() + if err != nil { + out.Fatalf("Failed to resolve analyzer: %s", err) + } + + // Init replaces the build file but keeps the model sources. + // Warn the user because this removes dependency pins that they added manually. + existing := approximation.IsProject(projectPath) + if existing { + out.Warnf("Replacing the build file of the existing approximation project at %s; "+ + "pass every dependency the models need, not only the new ones", projectPath) + } + + if err := approximation.Scaffold(projectPath, initApproximationDeps, analyzerJarPath); err != nil { + out.Fatalf("Failed to create approximation project: %s", err) + } + + if existing { + fmt.Printf("Approximation project updated at %s\n", projectPath) + } else { + fmt.Printf("Approximation project initialized at %s\n", projectPath) + } + suggest("To compile the models run", fmt.Sprintf("opentaint compile approximations %s", args[0])) + }, +} + +func init() { + rootCmd.AddCommand(approximationCmd) + approximationCmd.AddCommand(approximationInitCmd) + + approximationInitCmd.Flags().StringArrayVar(&initApproximationDeps, "dependency", nil, + "Compile-only Maven dependency coordinates the models are written against (repeatable)") +} diff --git a/cli/cmd/command_builder.go b/cli/cmd/command_builder.go index 1f38f2ca69..263c45e58c 100644 --- a/cli/cmd/command_builder.go +++ b/cli/cmd/command_builder.go @@ -12,9 +12,13 @@ type BaseCommandBuilder struct { debug bool } -// appendVerbosityFlag passes the JAR's own --verbosity flag based on opentaint's -// debug bool. The Java tool's CLI surface is independent of opentaint's; we just -// translate. info ↔ false, debug ↔ true. +// ForceDebugVerbosity sets the tool log level to debug. +// The CLI can then read detailed output from the log file. +func (b *BaseCommandBuilder) ForceDebugVerbosity() { + b.debug = true +} + +// appendVerbosityFlag maps the CLI debug setting to the tool log level. func (b *BaseCommandBuilder) appendVerbosityFlag(flags []string) []string { if b.debug { return append(flags, "--verbosity=debug") diff --git a/cli/cmd/compile.go b/cli/cmd/compile.go index 3d33c8d337..3d04172990 100644 --- a/cli/cmd/compile.go +++ b/cli/cmd/compile.go @@ -143,12 +143,18 @@ func printCompileSummary(absOutputProjectModelPath string) { } func compileProject(absOutputProjectModelPath, absProjectRoot, autobuilderJarPath string, javaRunner java.JavaRunner) error { - var err error tempLogsDir, err := os.MkdirTemp("", "opentaint-*") if err != nil { return fmt.Errorf("failed to create temporary directory: %w", err) } - tempLogsFile := filepath.Join(tempLogsDir, "autobuild.log") + return compileProjectWithLogs(absOutputProjectModelPath, absProjectRoot, autobuilderJarPath, + filepath.Join(tempLogsDir, "autobuild.log"), false, javaRunner) +} + +// compileProjectWithLogs writes the autobuilder log to logsFile. +// captureBuildOutput adds the output from the build tool to this log. +func compileProjectWithLogs(absOutputProjectModelPath, absProjectRoot, autobuilderJarPath, tempLogsFile string, captureBuildOutput bool, javaRunner java.JavaRunner) error { + var err error builder := NewAutobuilderBuilder(). SetProjectRootDir(absProjectRoot). @@ -157,6 +163,10 @@ func compileProject(absOutputProjectModelPath, absProjectRoot, autobuilderJarPat SetLogsFile(tempLogsFile). SetJarPath(autobuilderJarPath) + if captureBuildOutput { + builder.ForceDebugVerbosity() + } + autobuilderCommand := builder.BuildNativeCommand() commandSucceeded := func(_ error) bool { diff --git a/cli/cmd/compile_approximations.go b/cli/cmd/compile_approximations.go index ed61db7b1f..b91f058688 100644 --- a/cli/cmd/compile_approximations.go +++ b/cli/cmd/compile_approximations.go @@ -3,168 +3,68 @@ package cmd import ( "fmt" "os" - "os/exec" "path/filepath" - "strings" - "github.com/seqra/opentaint/internal/globals" + "github.com/seqra/opentaint/internal/approximation" + "github.com/seqra/opentaint/internal/autobuilder" "github.com/seqra/opentaint/internal/output" - "github.com/seqra/opentaint/internal/utils" - "github.com/seqra/opentaint/internal/utils/java" - "github.com/seqra/opentaint/internal/utils/project" ) -// approxClassesJarPrefix is the path prefix under which the analyzer fat JAR -// bundles approximation support sources (OpentaintNdUtil, ArgumentTypeContext). -const approxClassesJarPrefix = "opentaint-dataflow-approximations/" - -// compileApproximationsIfNeeded checks whether a --dataflow-approximations directory -// contains .java source files. If so, it compiles them using javac (with the -// analyzer JAR + project dependencies on the classpath) and returns the path to -// the compiled .class output directory. If the directory already contains only -// .class files (or no .java files at all), it is returned as-is. -// -// projectModelDir is the directory containing project.yaml — used to resolve -// project dependencies for the javac classpath (approximation code may reference -// library types like org.apache.pdfbox.pdmodel.PDDocument). -func compileApproximationsIfNeeded(approxPath string, analyzerJarPath string, projectModelDir string) (string, error) { - info, err := os.Stat(approxPath) - if err != nil { - return "", fmt.Errorf("approximation path does not exist: %w", err) - } - if !info.IsDir() { - return approxPath, nil - } - - javaFiles, err := collectJavaSources(approxPath) - if err != nil { - return "", err - } - if len(javaFiles) == 0 { - return approxPath, nil - } - - output.LogInfof("Found %d .java file(s) in approximations directory, compiling...", len(javaFiles)) - - javacPath, err := resolveJavacPath() - if err != nil { - return "", err - } - - extractedDir, err := os.MkdirTemp("", "opentaint-approx-deps-*") - if err != nil { - return "", fmt.Errorf("failed to create temp directory for approximation deps: %w", err) - } - defer func() { _ = os.RemoveAll(extractedDir) }() - - if err := utils.ExtractZipPrefix(analyzerJarPath, approxClassesJarPrefix, extractedDir); err != nil { - return "", fmt.Errorf("failed to extract approximation classes from analyzer JAR: %w", err) - } - - outputDir, err := os.MkdirTemp("", "opentaint-approx-compiled-*") - if err != nil { - return "", fmt.Errorf("failed to create temp directory for compiled approximations: %w", err) - } - - classpath := buildApproxClasspath(analyzerJarPath, extractedDir, projectModelDir) - if err := runJavac(javacPath, classpath, outputDir, javaFiles); err != nil { - _ = os.RemoveAll(outputDir) - return "", err - } +// approximationBuilder compiles a model project with the autobuilder. +// The model project supplies the compile dependencies. +type approximationBuilder struct { + analyzerJarPath string + apiJar []byte +} - output.LogInfof("Approximation compilation succeeded, output: %s", outputDir) - return outputDir, nil +func newApproximationBuilder(analyzerJarPath string) approximation.Builder { + return &approximationBuilder{analyzerJarPath: analyzerJarPath} } -func collectJavaSources(root string) ([]string, error) { - var javaFiles []string - err := filepath.Walk(root, func(path string, fi os.FileInfo, walkErr error) error { - if walkErr != nil { - return walkErr +// Prepare updates the approximation API jar in the model project. +// The builder reads the jar from the analyzer only once. +func (b *approximationBuilder) Prepare(projectDir string) error { + if b.apiJar == nil { + apiJar, err := approximation.ExtractApiJar(b.analyzerJarPath) + if err != nil { + return err } - if !fi.IsDir() && strings.HasSuffix(fi.Name(), ".java") { - javaFiles = append(javaFiles, path) - } - return nil - }) - if err != nil { - return nil, fmt.Errorf("failed to walk approximations directory: %w", err) + b.apiJar = apiJar } - return javaFiles, nil + return approximation.WriteApiJar(projectDir, b.apiJar) } -func resolveJavacPath() (string, error) { - javacRunner := java.NewJavaRunner(). - WithSkipVerify(globals.Config.SkipVerify). - WithImageType(java.AdoptiumImageJDK). - TrySystem(). - TrySpecificVersion(globals.DefaultJavaVersion) - - javaPath, err := javacRunner.EnsureJava() +func (b *approximationBuilder) Compile(projectDir, projectModelDir string) error { + autobuilderJarPath, err := ensureAutobuilderAvailable() if err != nil { - return "", fmt.Errorf("failed to resolve Java for approximation compilation: %w", err) - } - - javacPath := java.DeriveJavacPath(javaPath) - if _, err := os.Stat(javacPath); err != nil { - return "", fmt.Errorf("javac not found at %s (resolved from java at %s). A JDK (not JRE) is required to compile approximation sources", javacPath, javaPath) + return fmt.Errorf("failed to resolve autobuilder: %w", err) } - return javacPath, nil -} - -// buildApproxClasspath assembles the javac classpath for approximation compilation: -// 1. Analyzer JAR — contains @Approximate, @ApproximateByName annotations -// 2. Extracted approximation utilities — OpentaintNdUtil, ArgumentTypeContext -// 3. Project dependencies — library JARs that approximation code may reference -func buildApproxClasspath(analyzerJarPath, extractedDir, projectModelDir string) string { - parts := []string{analyzerJarPath, extractedDir} - parts = append(parts, resolveProjectDependencies(projectModelDir)...) - return strings.Join(parts, string(os.PathListSeparator)) -} -func runJavac(javacPath, classpath, outputDir string, javaFiles []string) error { - args := []string{ - "-source", "8", - "-target", "8", - "-cp", classpath, - "-d", outputDir, + javaRunner := newAutobuilderJavaRunner() + if _, err := javaRunner.EnsureJava(); err != nil { + return fmt.Errorf("failed to resolve Java for approximation compilation: %w", err) } - args = append(args, javaFiles...) - - output.LogDebugf("Running javac: %s %s", javacPath, strings.Join(args, " ")) - cmd := exec.Command(javacPath, args...) - cmdOutput, cmdErr := cmd.CombinedOutput() - if cmdErr != nil { - return fmt.Errorf( - "approximation compilation failed:\n%s\njavac exited with: %w", - string(cmdOutput), cmdErr, - ) + logsDir, err := os.MkdirTemp("", "opentaint-approximation-logs-*") + if err != nil { + return fmt.Errorf("failed to create temp directory for approximation build logs: %w", err) } - return nil -} + logsFile := filepath.Join(logsDir, "approximation-build.log") -// resolveProjectDependencies reads project.yaml from the project model directory -// and returns absolute paths to the dependency JARs listed there. -func resolveProjectDependencies(projectModelDir string) []string { - if projectModelDir == "" { - return nil + // Show the spinner for direct and automatic builds. + output.LogInfof("Compiling approximation project %s", projectDir) + compile := func() error { + return compileProjectWithLogs(projectModelDir, projectDir, autobuilderJarPath, logsFile, true, javaRunner) } - config, err := project.LoadConfig(projectModelDir) - if err != nil { - output.LogDebugf("Could not read project config for approximation compilation: %v", err) - return nil - } - var absDeps []string - for _, dep := range config.AllDependencies() { - absPath := dep - if !filepath.IsAbs(dep) { - absPath = filepath.Join(projectModelDir, dep) - } - if _, err := os.Stat(absPath); err == nil { - absDeps = append(absDeps, absPath) + if err := out.RunWithSpinner("Compiling approximation project "+filepath.Base(projectDir), compile); err != nil { + // Show compiler errors when they are available. + // Keep the full log if no compiler error is available. + if diagnostics := autobuilder.CompilerDiagnostics(logsFile); diagnostics != "" { + return fmt.Errorf("approximation project %s failed to compile:\n%s", projectDir, diagnostics) } + return fmt.Errorf("failed to compile approximation project %s (build log: %s): %w", projectDir, logsFile, err) } - output.LogDebugf("Resolved %d project dependencies for approximation classpath", len(absDeps)) - return absDeps + + _ = os.RemoveAll(logsDir) + return nil } diff --git a/cli/cmd/compile_approximations_cmd.go b/cli/cmd/compile_approximations_cmd.go new file mode 100644 index 0000000000..d9ab54bcaf --- /dev/null +++ b/cli/cmd/compile_approximations_cmd.go @@ -0,0 +1,65 @@ +package cmd + +import ( + "fmt" + + "github.com/seqra/opentaint/internal/approximation" + "github.com/seqra/opentaint/internal/utils/log" + "github.com/spf13/cobra" +) + +var compileApproximationsOutput string + +var compileApproximationsCmd = &cobra.Command{ + Use: "approximations ", + Short: "Compile a dataflow approximation project", + Long: `Compile the dataflow approximation models in against the +dependencies that project pins in its own build file. + +This is the model project, not the project to scan: its build.gradle.kts lists the exact +library versions the models are written against, so the same models compile identically +wherever they are applied. + +Scanning and testing build an approximation project on demand, so running this is only +needed to compile ahead of time or to see compilation errors on their own.`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + projectPath := log.AbsPathOrExit(args[0], "approximation-project") + + if !approximation.IsProject(projectPath) { + out.Fatalf("%s is not an approximation project: it has no build.gradle.kts declaring the "+ + "dependencies its models are written against.\nCreate one with 'opentaint approximation init %s "+ + "--dependency '", projectPath, args[0]) + } + + outputPath := compileApproximationsOutput + if outputPath == "" { + outputPath = approximation.BuildDir(projectPath) + } else { + outputPath = log.AbsPathOrExit(outputPath, "output") + } + + analyzerJarPath, err := ensureAnalyzerAvailable() + if err != nil { + out.Fatalf("Failed to resolve analyzer: %s", err) + } + + if err := approximation.Build(projectPath, outputPath, newApproximationBuilder(analyzerJarPath)); err != nil { + out.Fatalf("Approximation compilation failed: %s", err) + } + + out.Section("Approximation Compile Summary"). + Field("Approximation project", projectPath). + Field("Compiled models written to", approximation.ClassesDir(outputPath)). + Render() + suggest("To apply these models in a scan run", + fmt.Sprintf("opentaint scan --dataflow-approximations %s", args[0])) + }, +} + +func init() { + compileCmd.AddCommand(compileApproximationsCmd) + + compileApproximationsCmd.Flags().StringVarP(&compileApproximationsOutput, "output", "o", "", + "Path to the compiled models (default: /.opentaint/build)") +} diff --git a/cli/cmd/scan.go b/cli/cmd/scan.go index 8ef75c3739..30f51ed2fa 100644 --- a/cli/cmd/scan.go +++ b/cli/cmd/scan.go @@ -144,7 +144,7 @@ func addScanFlags(cmd *cobra.Command) { cmd.Flags().StringArrayVar(&scanFlags.PassthroughApproximations, "passthrough-approximations", nil, "Pass-through approximation YAML file or directory (repeatable)") - cmd.Flags().StringArrayVar(&scanFlags.DataflowApproximations, "dataflow-approximations", nil, "Dataflow approximation class directory or Java source directory (repeatable)") + cmd.Flags().StringArrayVar(&scanFlags.DataflowApproximations, "dataflow-approximations", nil, "Dataflow approximation project, build output, or class directory (repeatable)") cmd.Flags().BoolVar(&scanFlags.TrackExternalMethods, "track-external-methods", false, "Write external-method coverage files next to the SARIF report") @@ -408,8 +408,7 @@ func runScan(cmd *cobra.Command, cfg ScanConfig) { } nativeBuilder.SetJarPath(analyzerJarPath) - // Process --dataflow-approximations: auto-compile .java sources if needed - addDataflowApproximations(nativeBuilder, cfg.DataflowApproximations, analyzerJarPath, absProjectModelPath) + addDataflowApproximations(nativeBuilder, cfg.DataflowApproximations, analyzerJarPath) analyzerJavaRunner := newAnalyzerJavaRunner() if _, err := analyzerJavaRunner.EnsureJava(); err != nil { diff --git a/cli/cmd/test.go b/cli/cmd/test.go index 409240606c..a31a3cc73b 100644 --- a/cli/cmd/test.go +++ b/cli/cmd/test.go @@ -43,5 +43,5 @@ func addTestRunFlags(cmd *cobra.Command, outputDir *string, timeout *time.Durati cmd.Flags().StringVarP(outputDir, "output", "o", "", "Directory for test-result.json and test-results.sarif") cmd.Flags().DurationVar(timeout, "timeout", 600*time.Second, "Analysis timeout") cmd.Flags().StringVar(maxMemory, "max-memory", "8G", "Maximum analyzer heap size (e.g., 8G)") - cmd.Flags().StringArrayVar(dataflow, "dataflow-approximations", nil, "Dataflow approximation class directory or Java source directory (repeatable)") + cmd.Flags().StringArrayVar(dataflow, "dataflow-approximations", nil, "Dataflow approximation project, build output, or class directory (repeatable)") } diff --git a/cli/cmd/test_rule_run.go b/cli/cmd/test_rule_run.go index 4b380c617b..6c7261e0fa 100644 --- a/cli/cmd/test_rule_run.go +++ b/cli/cmd/test_rule_run.go @@ -128,7 +128,7 @@ func runTestProject(projectModelArg string, opts testProjectOptions) { } builder.SetJarPath(analyzerJarPath) - addDataflowApproximations(builder, opts.dataflowApprox, analyzerJarPath, projectPath) + addDataflowApproximations(builder, opts.dataflowApprox, analyzerJarPath) addPassthroughApproximations(builder, opts.passthroughApprox) javaRunner := newAnalyzerJavaRunner() diff --git a/cli/internal/approximation/approximation_test.go b/cli/internal/approximation/approximation_test.go new file mode 100644 index 0000000000..86ae019cb2 --- /dev/null +++ b/cli/internal/approximation/approximation_test.go @@ -0,0 +1,323 @@ +package approximation + +import ( + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v2" +) + +// fakeBuilder records builds and writes a test project model. +type fakeBuilder struct { + t *testing.T + compiled []string +} + +func newFakeBuilder(t *testing.T) *fakeBuilder { + return &fakeBuilder{t: t} +} + +// Prepare writes the same API content for each call. +func (b *fakeBuilder) Prepare(projectDir string) error { + writeFile(b.t, apiJarPath(projectDir), "api-jar-bytes") + return nil +} + +func (b *fakeBuilder) Compile(projectDir, projectModelDir string) error { + b.compiled = append(b.compiled, projectDir) + writeModuleClasses(b.t, projectModelDir, map[string][]string{"c0_main": {"Model"}}) + return nil +} + +// writeModuleClasses writes classes and a project descriptor for tests. +func writeModuleClasses(t *testing.T, projectModelDir string, modules map[string][]string) { + t.Helper() + moduleDirs := make([]string, 0, len(modules)) + for module, classes := range modules { + for _, class := range classes { + writeFile(t, filepath.Join(projectModelDir, "classes", module, "com", "example", class+".class"), "bytecode") + } + moduleDirs = append(moduleDirs, module) + } + sort.Strings(moduleDirs) + + config := "javaProjects:\n - sourceRoot: /project\n modules:\n - moduleSourceRoot: /project\n moduleClasses:\n" + for _, module := range moduleDirs { + config += " - classes/" + module + "\n" + } + writeFile(t, filepath.Join(projectModelDir, "project.yaml"), config) +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// newProject writes a model project for tests. +func newProject(t *testing.T, dir string) string { + t.Helper() + writeFile(t, filepath.Join(dir, "build.gradle.kts"), "dependencies {\n compileOnly(\"com.foo:bar:1.0\")\n}\n") + writeFile(t, filepath.Join(dir, "src", "main", "java", "com", "example", "Model.java"), "package com.example;\n") + return dir +} + +func TestResolveBuildsProject(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := newFakeBuilder(t) + + classDirs, err := Resolve(project, builder) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + + want := ClassesDir(BuildDir(project)) + if len(classDirs) != 1 || classDirs[0] != want { + t.Fatalf("Resolve() = %v, want [%s]", classDirs, want) + } + if len(builder.compiled) != 1 { + t.Fatalf("expected one build, got %v", builder.compiled) + } + if _, err := os.Stat(filepath.Join(want, "com", "example", "Model.class")); err != nil { + t.Errorf("compiled model not collected: %v", err) + } + if _, err := os.Stat(filepath.Join(BuildDir(project), descriptorName)); err != nil { + t.Errorf("descriptor not written: %v", err) + } +} + +func TestResolveReusesBuildUntilSourcesChange(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := newFakeBuilder(t) + + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("first Resolve: %v", err) + } + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("second Resolve: %v", err) + } + if len(builder.compiled) != 1 { + t.Fatalf("unchanged sources rebuilt: %v", builder.compiled) + } + + writeFile(t, filepath.Join(project, "src", "main", "java", "com", "example", "Model.java"), "package com.example;\n// edited\n") + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("third Resolve: %v", err) + } + if len(builder.compiled) != 2 { + t.Fatalf("edited model not rebuilt: %v", builder.compiled) + } +} + +func TestResolveRebuildsWhenDependencyPinChanges(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := newFakeBuilder(t) + + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("first Resolve: %v", err) + } + writeFile(t, filepath.Join(project, "build.gradle.kts"), "dependencies {\n compileOnly(\"com.foo:bar:2.0\")\n}\n") + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("second Resolve: %v", err) + } + if len(builder.compiled) != 2 { + t.Fatalf("changed dependency pin not rebuilt: %v", builder.compiled) + } +} + +func TestResolveBuiltOutput(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, descriptorName), "sourceProject: \"/somewhere\"\n") + writeFile(t, filepath.Join(dir, classesDirName, "com", "example", "Model.class"), "bytecode") + + builder := newFakeBuilder(t) + classDirs, err := Resolve(dir, builder) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if len(classDirs) != 1 || classDirs[0] != ClassesDir(dir) { + t.Fatalf("Resolve() = %v, want [%s]", classDirs, ClassesDir(dir)) + } + if len(builder.compiled) != 0 { + t.Errorf("built output should not be rebuilt, got %v", builder.compiled) + } +} + +func TestResolvePrecompiledClassDirectory(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "com", "example", "Model.class"), "bytecode") + + classDirs, err := Resolve(dir, newFakeBuilder(t)) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if len(classDirs) != 1 || classDirs[0] != dir { + t.Fatalf("Resolve() = %v, want [%s]", classDirs, dir) + } +} + +func TestResolveTreeOfProjects(t *testing.T) { + root := t.TempDir() + newProject(t, filepath.Join(root, "batch-a")) + newProject(t, filepath.Join(root, "batch-b")) + + builder := newFakeBuilder(t) + classDirs, err := Resolve(root, builder) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if len(classDirs) != 2 { + t.Fatalf("Resolve() = %v, want both batches", classDirs) + } + if len(builder.compiled) != 2 { + t.Fatalf("expected both projects built, got %v", builder.compiled) + } + + // The second call must ignore the nested build outputs. + again, err := Resolve(root, builder) + if err != nil { + t.Fatalf("second Resolve: %v", err) + } + if len(again) != 2 { + t.Fatalf("second Resolve() = %v, want both batches", again) + } + if len(builder.compiled) != 2 { + t.Fatalf("unchanged projects rebuilt: %v", builder.compiled) + } +} + +func TestResolveLooseSourcesReportsMigration(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "com", "example", "Model.java"), "package com.example;\n") + + _, err := Resolve(dir, newFakeBuilder(t)) + if err == nil { + t.Fatal("expected loose .java sources to be rejected") + } + if !strings.Contains(err.Error(), "approximation init") { + t.Errorf("error should point at the migration command, got: %v", err) + } +} + +func TestResolveEmptyDirectory(t *testing.T) { + _, err := Resolve(t.TempDir(), newFakeBuilder(t)) + if err == nil { + t.Fatal("expected an empty directory to be rejected") + } +} + +func TestFailedBuildLeavesTheProjectBuildable(t *testing.T) { + project := newProject(t, t.TempDir()) + + // A failed build must not block the next build. + if err := Build(project, BuildDir(project), &collidingBuilder{fakeBuilder{t: t}}); err == nil { + t.Fatal("expected the colliding build to fail") + } + if err := Build(project, BuildDir(project), newFakeBuilder(t)); err != nil { + t.Fatalf("rebuild after a failed build: %v", err) + } +} + +func TestFailedRebuildKeepsThePreviousBuild(t *testing.T) { + project := newProject(t, t.TempDir()) + output := BuildDir(project) + + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("first build: %v", err) + } + if err := Build(project, output, &collidingBuilder{fakeBuilder{t: t}}); err == nil { + t.Fatal("expected the colliding build to fail") + } + + // The previous models must remain available. + if _, err := os.Stat(filepath.Join(ClassesDir(output), "com", "example", "Model.class")); err != nil { + t.Errorf("a failed rebuild destroyed the previous build: %v", err) + } + if !isBuiltOutput(output) { + t.Error("a failed rebuild left the output unusable") + } +} + +func TestBuildRefusesToOverwriteUnrelatedDirectory(t *testing.T) { + project := newProject(t, t.TempDir()) + output := t.TempDir() + precious := filepath.Join(output, "notes.txt") + writeFile(t, precious, "important user data") + + // The build must not replace an unrelated directory. + err := Build(project, output, newFakeBuilder(t)) + if err == nil { + t.Fatal("expected a build into an unrelated directory to be refused") + } + if !strings.Contains(err.Error(), "not an approximation build output") { + t.Errorf("unexpected error: %v", err) + } + if content, readErr := os.ReadFile(precious); readErr != nil || string(content) != "important user data" { + t.Errorf("existing content was destroyed: content=%q err=%v", content, readErr) + } +} + +func TestBuildAcceptsEmptyAndPreviousOutputDirectories(t *testing.T) { + project := newProject(t, t.TempDir()) + output := t.TempDir() // exists and is empty + + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("build into an empty directory: %v", err) + } + // A new build can replace the first build. + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("rebuild over a previous build output: %v", err) + } +} + +func TestBuildWritesReadableDescriptor(t *testing.T) { + project := newProject(t, t.TempDir()) + output := filepath.Join(t.TempDir(), "out") + + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("Build: %v", err) + } + + raw, err := os.ReadFile(filepath.Join(output, descriptorName)) + if err != nil { + t.Fatal(err) + } + var parsed descriptor + if err := yaml.Unmarshal(raw, &parsed); err != nil { + t.Fatalf("descriptor is not valid YAML: %v\n%s", err, raw) + } + if parsed.SourceProject != project { + t.Errorf("sourceProject = %q, want %q", parsed.SourceProject, project) + } + if parsed.Dependencies == nil { + t.Error("dependencies should be an empty list, not null") + } +} + +func TestBuildRejectsCollidingModelClasses(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := &collidingBuilder{fakeBuilder{t: t}} + + err := Build(project, BuildDir(project), builder) + if err == nil { + t.Fatal("expected colliding approximation classes to fail the build") + } + if !strings.Contains(err.Error(), "exactly one target class") { + t.Errorf("unexpected error: %v", err) + } +} + +// collidingBuilder writes the same class from two modules. +type collidingBuilder struct{ fakeBuilder } + +func (b *collidingBuilder) Compile(projectDir, projectModelDir string) error { + writeModuleClasses(b.t, projectModelDir, map[string][]string{"c0_main": {"Model"}, "c1_main": {"Model"}}) + return nil +} diff --git a/cli/internal/approximation/build.go b/cli/internal/approximation/build.go new file mode 100644 index 0000000000..e0dbd29218 --- /dev/null +++ b/cli/internal/approximation/build.go @@ -0,0 +1,204 @@ +package approximation + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/seqra/opentaint/internal/utils" + "github.com/seqra/opentaint/internal/utils/project" +) + +// Builder compiles a model project. +type Builder interface { + // Prepare updates local build inputs before the source stamp is calculated. + Prepare(projectDir string) error + // Compile writes a project model to projectModelDir. + Compile(projectDir, projectModelDir string) error +} + +// ensureBuilt returns current compiled models for a project. +func ensureBuilt(projectDir string, builder Builder) (string, error) { + sources, err := prepare(projectDir, builder) + if err != nil { + return "", err + } + + outputDir := BuildDir(projectDir) + if upToDate(outputDir, sources) { + return ClassesDir(outputDir), nil + } + if err := build(projectDir, outputDir, sources, builder); err != nil { + return "", err + } + return ClassesDir(outputDir), nil +} + +// Build compiles a model project into outputDir. +func Build(projectDir, outputDir string, builder Builder) error { + sources, err := prepare(projectDir, builder) + if err != nil { + return err + } + return build(projectDir, outputDir, sources, builder) +} + +// prepare updates build inputs and returns their source stamp. +func prepare(projectDir string, builder Builder) (string, error) { + if err := builder.Prepare(projectDir); err != nil { + return "", err + } + return computeStamp(projectDir) +} + +// stagingSuffix identifies a build that is not complete. +const stagingSuffix = ".incomplete" + +func build(projectDir, outputDir, sources string, builder Builder) error { + if err := checkOutputDir(outputDir); err != nil { + return err + } + + buildTempDir, err := os.MkdirTemp("", "opentaint-approximation-build-*") + if err != nil { + return fmt.Errorf("failed to create temp directory for approximation build: %w", err) + } + defer func() { _ = os.RemoveAll(buildTempDir) }() + + // A missing output shows that the compiler did not produce a model. + projectModelDir := filepath.Join(buildTempDir, "model") + + if err := builder.Compile(projectDir, projectModelDir); err != nil { + return err + } + + // Assemble the new output before it replaces the current output. + staging := outputDir + stagingSuffix + defer func() { _ = os.RemoveAll(staging) }() + if err := os.RemoveAll(staging); err != nil { + return fmt.Errorf("failed to clear %s: %w", staging, err) + } + if err := os.MkdirAll(ClassesDir(staging), 0o755); err != nil { + return fmt.Errorf("failed to create approximation build output %s: %w", staging, err) + } + + dependencies, err := collectClasses(projectModelDir, ClassesDir(staging)) + if err != nil { + return err + } + if !containsClassFiles(ClassesDir(staging)) { + return fmt.Errorf("approximation project %s compiled to no classes; it has no models under src/main/java", projectDir) + } + + if err := writeDescriptor(staging, descriptor{SourceProject: projectDir, Dependencies: dependencies}); err != nil { + return err + } + if err := writeStamp(staging, sources); err != nil { + return err + } + + return replaceDir(staging, outputDir) +} + +// checkOutputDir prevents replacement of an unrelated directory. +func checkOutputDir(outputDir string) error { + info, err := os.Stat(outputDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("failed to access output directory %s: %w", outputDir, err) + } + if !info.IsDir() { + return fmt.Errorf("output path %s is not a directory", outputDir) + } + if isBuiltOutput(outputDir) { + return nil + } + + empty, err := isEmptyDir(outputDir) + if err != nil { + return err + } + if !empty { + return fmt.Errorf( + "output directory %s already exists and is not an approximation build output; "+ + "remove it or choose another --output path", outputDir) + } + return nil +} + +// replaceDir replaces the current output with a complete build. +func replaceDir(staging, outputDir string) error { + if err := os.MkdirAll(filepath.Dir(outputDir), 0o755); err != nil { + return fmt.Errorf("failed to create %s: %w", filepath.Dir(outputDir), err) + } + if err := os.RemoveAll(outputDir); err != nil { + return fmt.Errorf("failed to clear approximation build output %s: %w", outputDir, err) + } + if err := os.Rename(staging, outputDir); err != nil { + return fmt.Errorf("failed to move the compiled models into %s: %w", outputDir, err) + } + return nil +} + +func isEmptyDir(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return false, fmt.Errorf("failed to read output directory %s: %w", dir, err) + } + return len(entries) == 0, nil +} + +// collectClasses merges the compiled classes from all modules. +// It returns an error if two modules contain the same class. +func collectClasses(projectModelDir, destDir string) ([]string, error) { + config, err := project.LoadConfig(projectModelDir) + if err != nil { + return nil, fmt.Errorf("approximation build produced no readable project model: %w", err) + } + + for _, module := range config.AllModules() { + for _, classes := range module.ModuleClasses { + if err := copyTree(absoluteUnder(projectModelDir, classes), destDir); err != nil { + return nil, err + } + } + } + + dependencies := make([]string, 0, len(config.AllDependencies())) + for _, dep := range config.AllDependencies() { + dependencies = append(dependencies, filepath.Base(dep)) + } + return dependencies, nil +} + +func absoluteUnder(baseDir, path string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(baseDir, path) +} + +func copyTree(srcDir, destDir string) error { + return filepath.WalkDir(srcDir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + target := filepath.Join(destDir, rel) + if entry.IsDir() { + return os.MkdirAll(target, 0o755) + } + if _, err := os.Stat(target); err == nil { + return fmt.Errorf( + "two approximation modules produce %s; an approximation class maps to exactly one target class", + filepath.ToSlash(rel), + ) + } + return utils.CopyFile(path, target) + }) +} diff --git a/cli/internal/approximation/descriptor.go b/cli/internal/approximation/descriptor.go new file mode 100644 index 0000000000..1a088fada4 --- /dev/null +++ b/cli/internal/approximation/descriptor.go @@ -0,0 +1,34 @@ +package approximation + +import ( + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v2" +) + +const descriptorHeader = "# Generated by 'opentaint compile approximations'. Do not edit.\n" + +// descriptor identifies the source and dependencies of a model build. +type descriptor struct { + // SourceProject is the project that supplied the models. + SourceProject string `yaml:"sourceProject"` + // Dependencies are the libraries used to compile the models. + Dependencies []string `yaml:"dependencies"` +} + +func writeDescriptor(outputDir string, d descriptor) error { + if d.Dependencies == nil { + d.Dependencies = []string{} + } + body, err := yaml.Marshal(d) + if err != nil { + return fmt.Errorf("failed to render the approximation descriptor: %w", err) + } + path := filepath.Join(outputDir, descriptorName) + if err := os.WriteFile(path, append([]byte(descriptorHeader), body...), 0o644); err != nil { + return fmt.Errorf("failed to write %s: %w", path, err) + } + return nil +} diff --git a/cli/internal/approximation/project.go b/cli/internal/approximation/project.go new file mode 100644 index 0000000000..4e1fe57fdc --- /dev/null +++ b/cli/internal/approximation/project.go @@ -0,0 +1,116 @@ +// Package approximation manages Gradle projects for dataflow models. +// +// Each project specifies the library versions that its models use. +// The scanned project does not supply these compile dependencies. +package approximation + +import ( + "os" + "path/filepath" + "strings" +) + +const ( + // apiJarName contains the annotations and support types for model projects. + apiJarName = "opentaint-approximations-api.jar" + // apiJarResourcePrefix must match API_JAR_RESOURCE_DIR in the core build. + apiJarResourcePrefix = "opentaint-approximations-api/" + + libsDirName = "libs" + classesDirName = "classes" + descriptorName = "approximation.yaml" + + stampFileName = "stamp.json" + gradleBuildFile = "build.gradle.kts" +) + +// opentaintDirName contains the local build output. +const opentaintDirName = ".opentaint" + +// skippedDirs contains generated and version-control directories. +var skippedDirs = map[string]bool{ + opentaintDirName: true, + "build": true, + ".gradle": true, + ".git": true, +} + +// IsProject reports whether dir is a model project root. +func IsProject(dir string) bool { + return isFile(filepath.Join(dir, gradleBuildFile)) +} + +// isBuiltOutput reports whether dir contains a model build. +func isBuiltOutput(dir string) bool { + return isFile(filepath.Join(dir, descriptorName)) && isDir(filepath.Join(dir, classesDirName)) +} + +// BuildDir returns the path of the local build output. +func BuildDir(projectDir string) string { + return filepath.Join(projectDir, opentaintDirName, "build") +} + +// ClassesDir returns the class directory in a build output. +func ClassesDir(outputDir string) string { + return filepath.Join(outputDir, classesDirName) +} + +// apiJarPath returns the path of the project API jar. +func apiJarPath(projectDir string) string { + return filepath.Join(projectDir, libsDirName, apiJarName) +} + +// containsClassFiles reports whether dir contains compiled classes. +// It does not search generated directories. +func containsClassFiles(dir string) bool { + found := false + _ = filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if path != dir && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + if strings.EqualFold(filepath.Ext(entry.Name()), ".class") { + found = true + return filepath.SkipAll + } + return nil + }) + return found +} + +// firstJavaSource returns one Java source file under dir. +func firstJavaSource(dir string) (string, bool) { + var found string + _ = filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if path != dir && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + if strings.EqualFold(filepath.Ext(entry.Name()), ".java") { + found = path + return filepath.SkipAll + } + return nil + }) + return found, found != "" +} + +func isFile(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} + +func isDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} diff --git a/cli/internal/approximation/resolve.go b/cli/internal/approximation/resolve.go new file mode 100644 index 0000000000..e3be9197ce --- /dev/null +++ b/cli/internal/approximation/resolve.go @@ -0,0 +1,85 @@ +package approximation + +import ( + "fmt" + "os" + "path/filepath" + "sort" +) + +// Resolve returns the class directories for one model path. +// The path can have one of these forms: +// +// 1. A model project. +// 2. A model build output. +// 3. A compiled class directory. +// 4. A directory tree that contains these items. +func Resolve(path string, builder Builder) ([]string, error) { + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("approximation path does not exist: %w", err) + } + if !info.IsDir() { + return nil, fmt.Errorf("approximation path %s is not a directory", path) + } + + classDirs, err := resolveDir(path, builder) + if err != nil { + return nil, err + } + if len(classDirs) == 0 { + return nil, emptyPathError(path) + } + return classDirs, nil +} + +func resolveDir(dir string, builder Builder) ([]string, error) { + if IsProject(dir) { + classes, err := ensureBuilt(dir, builder) + if err != nil { + return nil, err + } + return []string{classes}, nil + } + + if isBuiltOutput(dir) { + return []string{ClassesDir(dir)}, nil + } + + if containsClassFiles(dir) { + return []string{dir}, nil + } + + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("failed to read approximation directory %s: %w", dir, err) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + + var classDirs []string + for _, entry := range entries { + if !entry.IsDir() || skippedDirs[entry.Name()] { + continue + } + nested, err := resolveDir(filepath.Join(dir, entry.Name()), builder) + if err != nil { + return nil, err + } + classDirs = append(classDirs, nested...) + } + return classDirs, nil +} + +// emptyPathError reports why a path does not contain usable models. +func emptyPathError(path string) error { + if source, found := firstJavaSource(path); found { + return fmt.Errorf( + "%s holds approximation sources (e.g. %s) but no approximation project: nothing declares "+ + "the dependencies to compile the models against.\n"+ + "Create a project per model directory with 'opentaint approximation init "+ + "--dependency ' and move its sources under /src/main/java", + path, source, + ) + } + return fmt.Errorf("%s contains no approximation project, build output or compiled classes", path) +} diff --git a/cli/internal/approximation/scaffold.go b/cli/internal/approximation/scaffold.go new file mode 100644 index 0000000000..357b7f5ed8 --- /dev/null +++ b/cli/internal/approximation/scaffold.go @@ -0,0 +1,67 @@ +package approximation + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/seqra/opentaint/internal/testproject" +) + +const gradleProjectName = "approximation-models" + +// Scaffold creates a Gradle project for dataflow models. +func Scaffold(projectDir string, dependencies []string, analyzerJarPath string) error { + apiJar, err := ExtractApiJar(analyzerJarPath) + if err != nil { + return err + } + + localJar := filepath.ToSlash(filepath.Join(libsDirName, apiJarName)) + if err := testproject.BootstrapWithLocalJars(projectDir, gradleProjectName, dependencies, []string{localJar}); err != nil { + return err + } + if err := WriteApiJar(projectDir, apiJar); err != nil { + return err + } + return os.MkdirAll(filepath.Join(projectDir, "src", "main", "java"), 0o755) +} + +// ExtractApiJar reads the approximation API jar from the analyzer jar. +func ExtractApiJar(analyzerJarPath string) ([]byte, error) { + reader, err := zip.OpenReader(analyzerJarPath) + if err != nil { + return nil, fmt.Errorf("failed to open analyzer jar %s: %w", analyzerJarPath, err) + } + defer func() { _ = reader.Close() }() + + entryName := apiJarResourcePrefix + apiJarName + for _, file := range reader.File { + if file.Name != entryName { + continue + } + content, err := file.Open() + if err != nil { + return nil, fmt.Errorf("failed to read %s from the analyzer jar: %w", apiJarName, err) + } + defer func() { _ = content.Close() }() + return io.ReadAll(content) + } + return nil, fmt.Errorf("analyzer jar %s bundles no %s", analyzerJarPath, apiJarName) +} + +// WriteApiJar writes the approximation API jar to a model project. +// It does not write the file when the content is current. +func WriteApiJar(projectDir string, apiJar []byte) error { + path := apiJarPath(projectDir) + if current, err := os.ReadFile(path); err == nil && bytes.Equal(current, apiJar) { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("failed to create %s: %w", filepath.Dir(path), err) + } + return os.WriteFile(path, apiJar, 0o644) +} diff --git a/cli/internal/approximation/stamp.go b/cli/internal/approximation/stamp.go new file mode 100644 index 0000000000..99b25a9126 --- /dev/null +++ b/cli/internal/approximation/stamp.go @@ -0,0 +1,90 @@ +package approximation + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" +) + +// stamp identifies the inputs of a model build. +type stamp struct { + Sources string `json:"sources"` +} + +// computeStamp hashes all source files and build inputs. +// It does not hash generated files. +func computeStamp(projectDir string) (string, error) { + var paths []string + err := filepath.WalkDir(projectDir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if path != projectDir && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + rel, err := filepath.Rel(projectDir, path) + if err != nil { + return err + } + paths = append(paths, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return "", fmt.Errorf("failed to read approximation project %s: %w", projectDir, err) + } + sort.Strings(paths) + + digest := sha256.New() + for _, rel := range paths { + if _, err := fmt.Fprintf(digest, "%s\x00", rel); err != nil { + return "", fmt.Errorf("failed to hash path %s: %w", rel, err) + } + file, err := os.Open(filepath.Join(projectDir, filepath.FromSlash(rel))) + if err != nil { + return "", fmt.Errorf("failed to read %s: %w", rel, err) + } + _, copyErr := io.Copy(digest, file) + _ = file.Close() + if copyErr != nil { + return "", fmt.Errorf("failed to read %s: %w", rel, copyErr) + } + } + return hex.EncodeToString(digest.Sum(nil)), nil +} + +func readStamp(outputDir string) (string, bool) { + data, err := os.ReadFile(filepath.Join(outputDir, stampFileName)) + if err != nil { + return "", false + } + var s stamp + if err := json.Unmarshal(data, &s); err != nil { + return "", false + } + return s.Sources, s.Sources != "" +} + +func writeStamp(outputDir, sources string) error { + data, err := json.MarshalIndent(stamp{Sources: sources}, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(outputDir, stampFileName), append(data, '\n'), 0o644) +} + +// upToDate reports whether outputDir matches the source stamp. +func upToDate(outputDir, sources string) bool { + previous, ok := readStamp(outputDir) + if !ok || previous != sources { + return false + } + return containsClassFiles(ClassesDir(outputDir)) +} diff --git a/cli/internal/autobuilder/buildlog.go b/cli/internal/autobuilder/buildlog.go new file mode 100644 index 0000000000..848e69498a --- /dev/null +++ b/cli/internal/autobuilder/buildlog.go @@ -0,0 +1,47 @@ +// Package autobuilder reads autobuilder output. +package autobuilder + +import ( + "os" + "regexp" + "strings" +) + +// logLineSeparator precedes the build-tool output in an autobuilder log line. +const logLineSeparator = " - " + +// compilerErrorCount matches a javac error summary. +var compilerErrorCount = regexp.MustCompile(`^\d+ errors?$`) + +// CompilerDiagnostics returns compiler errors from an autobuilder log. +// It returns an empty string if the log has no compiler error. +func CompilerDiagnostics(logPath string) string { + log, err := os.ReadFile(logPath) + if err != nil { + return "" + } + + var diagnostics []string + for _, line := range strings.Split(string(log), "\n") { + message := buildOutput(line) + if isCompilerDiagnostic(message) { + diagnostics = append(diagnostics, " "+message) + } + } + return strings.Join(diagnostics, "\n") +} + +func buildOutput(logLine string) string { + message := logLine + if _, after, found := strings.Cut(logLine, logLineSeparator); found { + message = after + } + return strings.TrimRight(message, "\r") +} + +func isCompilerDiagnostic(message string) bool { + // Match javac and kotlinc error formats. + return strings.Contains(message, ": error: ") || + strings.HasPrefix(message, "e: ") || + compilerErrorCount.MatchString(message) +} diff --git a/cli/internal/autobuilder/buildlog_test.go b/cli/internal/autobuilder/buildlog_test.go new file mode 100644 index 0000000000..dfe4b15f8a --- /dev/null +++ b/cli/internal/autobuilder/buildlog_test.go @@ -0,0 +1,81 @@ +package autobuilder + +import ( + "os" + "path/filepath" + "testing" +) + +func writeLog(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "autobuild.log") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestCompilerDiagnostics(t *testing.T) { + tests := []struct { + name string + log string + want string + }{ + { + name: "javac error among the build trace", + log: `20:22:13.627 |I| ProjectResolver - Gradle build start for: /tmp/batch +20:22:13.628 |D| ProjectResolver - /tmp/batch/src/main/java/Model.java:4: error: package org.nosuchlib does not exist +20:22:13.629 |D| ProjectResolver - import org.nosuchlib.Missing; +20:22:13.629 |D| ProjectResolver - 1 error +20:22:13.630 |D| ProjectResolver - FAILURE: Build failed with an exception. +`, + want: " /tmp/batch/src/main/java/Model.java:4: error: package org.nosuchlib does not exist\n 1 error", + }, + { + name: "kotlinc diagnostics", + log: "12:00:00.000 |D| ProjectResolver - e: file:///tmp/batch/Model.kt:7:1 unresolved reference\n", + want: " e: file:///tmp/batch/Model.kt:7:1 unresolved reference", + }, + { + name: "plural error count", + log: "12:00:00.000 |D| ProjectResolver - 3 errors\n", + want: " 3 errors", + }, + { + name: "raw build output with no log prefix", + log: "Model.java:8: error: cannot find symbol\n", + want: " Model.java:8: error: cannot find symbol", + }, + { + name: "successful build has no diagnostics", + log: `20:22:13.627 |I| ProjectResolver - Gradle build start for: /tmp/batch +20:22:13.630 |I| ProjectResolver - BUILD SUCCESSFUL in 4s +`, + want: "", + }, + { + name: "an error mentioned in prose is not a diagnostic", + log: "12:00:00.000 |I| ProjectResolver - Gradle build failed for: /tmp/batch\n", + want: "", + }, + { + name: "windows line endings are trimmed", + log: "12:00:00.000 |D| ProjectResolver - Model.java:8: error: cannot find symbol\r\n", + want: " Model.java:8: error: cannot find symbol", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := CompilerDiagnostics(writeLog(t, test.log)); got != test.want { + t.Errorf("CompilerDiagnostics() =\n%q\nwant\n%q", got, test.want) + } + }) + } +} + +func TestCompilerDiagnosticsMissingLog(t *testing.T) { + if got := CompilerDiagnostics(filepath.Join(t.TempDir(), "absent.log")); got != "" { + t.Errorf("CompilerDiagnostics() = %q, want \"\" for an unreadable log", got) + } +} diff --git a/cli/internal/testproject/testproject.go b/cli/internal/testproject/testproject.go index 5e7d7e70d9..5dfcb53046 100644 --- a/cli/internal/testproject/testproject.go +++ b/cli/internal/testproject/testproject.go @@ -9,13 +9,18 @@ import ( ) func Bootstrap(outputDir, projectName string, dependencies []string) error { + return BootstrapWithLocalJars(outputDir, projectName, dependencies, nil) +} + +// BootstrapWithLocalJars adds project-relative jars to the compile-only classpath. +func BootstrapWithLocalJars(outputDir, projectName string, dependencies, localJars []string) error { return utils.WriteFiles(map[string][]byte{ - filepath.Join(outputDir, "build.gradle.kts"): buildGradle(dependencies), + filepath.Join(outputDir, "build.gradle.kts"): buildGradle(dependencies, localJars), filepath.Join(outputDir, "settings.gradle.kts"): settingsGradle(projectName), }) } -func buildGradle(dependencies []string) []byte { +func buildGradle(dependencies, localJars []string) []byte { var sb strings.Builder sb.WriteString(`plugins { java @@ -32,6 +37,9 @@ repositories { dependencies { `) + for _, jar := range localJars { + fmt.Fprintf(&sb, " compileOnly(files(%q))\n", filepath.ToSlash(jar)) + } for _, dep := range dependencies { fmt.Fprintf(&sb, " compileOnly(%q)\n", dep) } diff --git a/cli/internal/utils/extract.go b/cli/internal/utils/extract.go index 90421152f6..c2262a1333 100644 --- a/cli/internal/utils/extract.go +++ b/cli/internal/utils/extract.go @@ -56,58 +56,6 @@ func ExtractTar(tr *tar.Reader, basePath, destPath string, isSourceDir bool) err return nil } -// ExtractZipPrefix extracts entries whose names begin with prefix from the zip -// at src into destDir, stripping prefix from each entry's relative path. -// Directory entries are skipped — parent directories are created on demand. -// Returns an error if the zip cannot be opened or any entry fails to write. -func ExtractZipPrefix(src, prefix, destDir string) error { - r, err := zip.OpenReader(src) - if err != nil { - return fmt.Errorf("failed to open zip: %w", err) - } - defer func() { _ = r.Close() }() - - for _, f := range r.File { - if !strings.HasPrefix(f.Name, prefix) || f.FileInfo().IsDir() { - continue - } - relPath := strings.TrimPrefix(f.Name, prefix) - if relPath == "" { - continue - } - if err := copyZipEntry(f, filepath.Join(destDir, relPath)); err != nil { - return err - } - } - return nil -} - -func copyZipEntry(f *zip.File, destPath string) (err error) { - if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { - return err - } - src, err := f.Open() - if err != nil { - return err - } - defer func() { - if cerr := src.Close(); cerr != nil && err == nil { - err = cerr - } - }() - dst, err := os.Create(destPath) - if err != nil { - return err - } - defer func() { - if cerr := dst.Close(); cerr != nil && err == nil { - err = cerr - } - }() - _, err = io.Copy(dst, src) - return err -} - // ExtractZip extracts the contents of a ZIP file to the specified destination directory. func ExtractZip(src, dest string) error { zr, err := zip.OpenReader(src) diff --git a/cli/internal/utils/java/detection.go b/cli/internal/utils/java/detection.go index 736bfb1cac..b2cde7b871 100644 --- a/cli/internal/utils/java/detection.go +++ b/cli/internal/utils/java/detection.go @@ -3,9 +3,7 @@ package java import ( "os" "os/exec" - "path/filepath" "regexp" - "runtime" "strconv" "strings" @@ -132,17 +130,6 @@ func validateJavaInstallation(javaPath string) *JavaInstallation { return installation } -// DeriveJavacPath returns the javac binary path that sits next to the given -// java binary. On Windows the ".exe" suffix is preserved. -func DeriveJavacPath(javaPath string) string { - dir := filepath.Dir(javaPath) - name := "javac" - if runtime.GOOS == "windows" { - name = "javac.exe" - } - return filepath.Join(dir, name) -} - func extractVendor(versionOutput string) string { output := strings.ToLower(versionOutput) diff --git a/docs/usage.md b/docs/usage.md index 84f9359ed2..e3c94c06ce 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -85,6 +85,7 @@ Use [CodeChecker](https://github.com/Ericsson/codechecker) for advanced result m | `opentaint health` | Show resolved paths for the analyzer, autobuilder, rules, and Java runtime | | `opentaint test rule` | Create, run, and debug detection-rule tests | | `opentaint test approximation` | Create and run dataflow-approximation tests | +| `opentaint approximation` | Create dataflow approximation projects | | `opentaint pull` | Download analyzer dependencies | | `opentaint update` | Update to latest version | | `opentaint prune` | Remove stale downloaded artifacts and cached models | @@ -115,7 +116,7 @@ These flags are to work with custom approximations: |------|-------------| | `--track-external-methods` | Write external-method coverage files next to the SARIF report | | `--passthrough-approximations` | Apply pass-through approximation YAML files or directories (repeatable) | -| `--dataflow-approximations` | Apply dataflow approximation classes or Java source directories (repeatable) | +| `--dataflow-approximations` | Apply dataflow approximation projects, build outputs, or class directories (repeatable) | Use external-method tracking when a scan may miss flows through library methods. The dropped-methods file shows where taint was killed because no model was available; the approximated-methods file shows methods already covered by built-in or custom models. @@ -160,7 +161,8 @@ opentaint test rule reachability java/security/my-rule.yaml:my-rule --project-mo #### Approximation tests ```bash -opentaint test approximation init .opentaint/test-projects/my-approximation +opentaint test approximation init .opentaint/test-projects/my-approximation \ + --dependency "io.projectreactor:reactor-core:3.8.5" opentaint compile .opentaint/test-projects/my-approximation -o .opentaint/test-compiled/my-approximation opentaint test approximation run .opentaint/test-compiled/my-approximation \ --dataflow-approximations .opentaint/dataflow/my-approximation @@ -188,6 +190,35 @@ opentaint scan --project-model ./my-project-model | `--dry-run` | Validate inputs and show what would run without compiling | | `--log-file` | Path to the log file (default: `/logs/.log`) | +### opentaint approximation + +A dataflow approximation models how taint moves through a method the analyzer cannot see into. Because a model references the library type it models, it needs that library to compile — so the models live in their own project, which pins those versions itself: + +```bash +opentaint approximation init .opentaint/dataflow/my-batch \ + --dependency "io.projectreactor:reactor-core:3.8.5" + +# write the @Approximate classes under src/main/java/, then apply them +opentaint scan --project-model ./my-project-model \ + --dataflow-approximations .opentaint/dataflow/my-batch +``` + +Pin the versions the target application uses. Those pins are the models' compile environment: they, and nothing about the project under analysis, decide what the models compile against, so a model compiles identically wherever it is applied. + +`--dataflow-approximations` accepts an approximation project, a build output, or a directory of compiled classes — and a directory holding any of those, so a tree with one project per batch can be passed as a single flag. Projects are built on demand and rebuilt only when their sources or dependency pins change. + +| Command | Description | +|---------|-------------| +| `opentaint approximation init ` | Create a dataflow approximation project | +| `opentaint compile approximations ` | Compile a dataflow approximation project | + +| Flag | Description | +|------|-------------| +| `--dependency` | Compile-only Maven dependency coordinates the models are written against (repeatable, `init` only) | +| `--output`, `-o` | Path to the compiled models (default: `/.opentaint/build`, `compile approximations` only) | + +Compiling ahead of time is optional — scanning and testing build the project themselves. Run it to see compilation errors on their own. + ### opentaint summary View findings from a SARIF report. By default it prints the Scan Summary; add diff --git a/skills-templates/create-dataflow-approximation/references/java.md.j2 b/skills-templates/create-dataflow-approximation/references/java.md.j2 index 523eb6e5b7..7bb792615b 100644 --- a/skills-templates/create-dataflow-approximation/references/java.md.j2 +++ b/skills-templates/create-dataflow-approximation/references/java.md.j2 @@ -10,7 +10,16 @@ An app-internal method sits in the project's own sources under ``. ### 2. Write the approximation source -Create Java files under `.opentaint/dataflow/` — one `@Approximate` class per target class. `@Approximate(TargetClass.class)` binds a model to exactly that class, so target the EXACT class the analyzer dropped — the dropped FQN reflects how the call resolved: an interface-typed receiver (`Map m = ...; m.computeIfAbsent(...)`) drops `java.util.Map#computeIfAbsent`, a concrete one (`new HashMap<>()`) drops `java.util.HashMap#computeIfAbsent`. An interface is a valid target — write `@Approximate` on it exactly as on a concrete class, modelling whichever FQN the analyzer dropped. Reach the real object with `(TargetClass) (Object) this`, put functional-interface parameters behind `@ArgumentTypeContext`, and branch with `OpentaintNdUtil.nextBool()` so the analyzer walks both paths. Never leave a body empty. +The models live in the batch's own approximation project. Scaffold it once, passing each library the batch's models will reference at the exact version the batch file's `dependencies` records — the same pins the test project took. Those pins are the models' compile environment: nothing about the project under analysis affects it, so the models compile the same way at test time and at scan time. + +```bash +opentaint approximation init .opentaint/dataflow/ \ + --dependency "io.projectreactor:reactor-core:3.8.5" +``` + +Re-invoked for a batch whose project already exists, leave the build file alone unless a model needs a library that isn't pinned yet — then add it with a second `init` carrying the full dependency set, or edit `build.gradle.kts` directly. + +Create Java files under `.opentaint/dataflow//src/main/java` — one `@Approximate` class per target class. `@Approximate(TargetClass.class)` binds a model to exactly that class, so target the EXACT class the analyzer dropped — the dropped FQN reflects how the call resolved: an interface-typed receiver (`Map m = ...; m.computeIfAbsent(...)`) drops `java.util.Map#computeIfAbsent`, a concrete one (`new HashMap<>()`) drops `java.util.HashMap#computeIfAbsent`. An interface is a valid target — write `@Approximate` on it exactly as on a concrete class, modelling whichever FQN the analyzer dropped. Reach the real object with `(TargetClass) (Object) this`, put functional-interface parameters behind `@ArgumentTypeContext`, and branch with `OpentaintNdUtil.nextBool()` so the analyzer walks both paths. Never leave a body empty. ```java package com.example.approximations.batchpkg; // per-batch package (e.g. ...approximations.cn_hutool_001) — see the globally-unique rule below @@ -68,7 +77,13 @@ opentaint test approximation run .opentaint/test-compiled/ \ --dataflow-approximations .opentaint/dataflow/ ``` -`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI auto-compiles the `.java` sources against the analyzer JAR (for `@Approximate`, `OpentaintNdUtil`, `ArgumentTypeContext`) and the project's dependencies; if compilation fails it reports the errors and aborts before the tests. A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: +`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI builds the approximation project against the dependencies it pins (the approximation API — `@Approximate`, `OpentaintNdUtil`, `ArgumentTypeContext` — comes from the project's own `libs/`); if compilation fails it reports the errors and aborts before the tests. A rebuild happens only when the sources or the pins changed, so re-running an unedited batch skips straight to the tests. To see compilation errors on their own, compile the project alone: + +```bash +opentaint compile approximations .opentaint/dataflow/ +``` + +A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: ```bash uv run /scripts/check-test-result.py @@ -91,6 +106,7 @@ Fix by the verdict it reports: ## Constraints - Java 8 source compatibility +- A model may only reference libraries its own project pins. A model that needs a type from an unpinned library is a missing `--dependency`, not a reason to weaken the model to `Object` - Put the `@Approximate` classes in a neutral package (e.g. `com.example.approximations`) — never the target library's own package. Inside the library's package every bare FQN resolves to your approximation's non-generic class instead of the real type, breaking compilation wholesale - Namespace the package PER BATCH (`com.example.approximations.`, e.g. `...approximations.cn_hutool_001`). Every batch is loaded together into one shared index, and approximation class FQCNs must be GLOBALLY unique across all batches — two batches that both model a `…$Nested` (or any recurring simple name) in the bare `com.example.approximations` package collide on `com.example.approximations.Nested` and crash the whole scan at config-load. A per-batch package guarantees uniqueness. - One approximation class per target class — a strict global bijection enforced at load: each approximation FQCN maps to exactly one target and vice-versa; duplicates or a reused FQCN across batches throw `IllegalArgumentException` diff --git a/skills-templates/create-dataflow-approximation/sections/output.md b/skills-templates/create-dataflow-approximation/sections/output.md index 773a3a1e8d..76c5e21f02 100644 --- a/skills-templates/create-dataflow-approximation/sections/output.md +++ b/skills-templates/create-dataflow-approximation/sections/output.md @@ -1,6 +1,6 @@ ### Artifacts -- `.opentaint/dataflow/` — the language-specific code approximation artifacts that the scan consumes; report the path and the exact test command used +- `.opentaint/dataflow/` — the batch's approximation project: its pinned dependencies plus the language-specific code approximation artifacts that the scan consumes, following the language reference's artifact layout; report the path and the exact test command used - the passing methods present in the batch file's `build.done` (new methods appended; repaired methods already recorded, per Tracking) ### Summary diff --git a/skills-templates/create-dataflow-approximation/sections/workflow.md b/skills-templates/create-dataflow-approximation/sections/workflow.md index 5b93dd2d26..58c6429cdd 100644 --- a/skills-templates/create-dataflow-approximation/sections/workflow.md +++ b/skills-templates/create-dataflow-approximation/sections/workflow.md @@ -4,7 +4,7 @@ Find and read each dataflow method's real source: take methods not yet in `build ### 2. Write the approximation -Reproduce that propagation in the language-specific code form under `.opentaint/dataflow/`, following the language reference's artifact layout. Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. +Reproduce that propagation in the language-specific code form in the batch's approximation project at `.opentaint/dataflow/`, following the language reference's artifact layout. The project pins the libraries the models are written against at the same versions the test project takes, because a model references the library type it models. Scaffold it if it doesn't exist yet (the language reference has the command). Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. ### 3. Test against the test project diff --git a/skills-templates/create-test-project/references/java-approximation.md.j2 b/skills-templates/create-test-project/references/java-approximation.md.j2 index e5152e14d2..8cee6ff26b 100644 --- a/skills-templates/create-test-project/references/java-approximation.md.j2 +++ b/skills-templates/create-test-project/references/java-approximation.md.j2 @@ -6,7 +6,7 @@ Language reference for a `dataflow` type, keyed to the body's steps. ### 1. Scaffold the project -`opentaint test approximation init ` scaffolds the Gradle build plus `Taint.java` and the fixed `approximation-rule.yaml` the harness applies. Pass each lib as a `--dependency` at its exact version from the app's dependency management (`.opentaint/project/sources/**/pom.xml`) verbatim — including timestamped snapshots — not a guess: this Gradle build is the approximation's own compile environment, so it must recompile from these pins even after the main project drops that dependency. +`opentaint test approximation init ` scaffolds the Gradle build plus `Taint.java` and the fixed `approximation-rule.yaml` the harness applies. Pass each lib as a `--dependency` at its exact version from the app's dependency management (`.opentaint/project/sources/**/pom.xml`) verbatim — including timestamped snapshots — not a guess. The approximation project scaffolded later takes the same pins, so the samples and the models they exercise compile against one version. ```bash opentaint test approximation init .opentaint/test-projects/ \ diff --git a/skills-templates/run-scan/sections/workflow.md b/skills-templates/run-scan/sections/workflow.md index 4ea2d1f03b..2e938782a6 100644 --- a/skills-templates/run-scan/sections/workflow.md +++ b/skills-templates/run-scan/sections/workflow.md @@ -11,7 +11,7 @@ opentaint scan --project-model .opentaint/project \ - `--rule-id ` — restrict to specific executable rules (repeatable, one per input rule ID); referenced library rules remain available without listing their IDs separately. Omit to run all loaded rules - `--passthrough-approximations .opentaint/pass-through` — add when that directory exists: passThrough configs override built-ins at the rule level, a provided rule overriding a built-in only when it matches one -- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations (sources auto-compiled; pre-compiled `.class` dirs passed through as-is) +- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations, one project per batch (each built on demand against the dependencies it pins, and rebuilt only when its sources change) Both approximation-dir flags walk their trees recursively; pass each parent directory once, not every package or batch separately. diff --git a/skills/create-dataflow-approximation/SKILL.md b/skills/create-dataflow-approximation/SKILL.md index 27b05ac3dc..ab3e1c2868 100644 --- a/skills/create-dataflow-approximation/SKILL.md +++ b/skills/create-dataflow-approximation/SKILL.md @@ -28,7 +28,7 @@ Find and read each dataflow method's real source: take methods not yet in `build ### 2. Write the approximation -Reproduce that propagation in the language-specific code form under `.opentaint/dataflow/`, following the language reference's artifact layout. Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. +Reproduce that propagation in the language-specific code form in the batch's approximation project at `.opentaint/dataflow/`, following the language reference's artifact layout. The project pins the libraries the models are written against at the same versions the test project takes, because a model references the library type it models. Scaffold it if it doesn't exist yet (the language reference has the command). Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. ### 3. Test against the test project @@ -42,7 +42,7 @@ When the sample won't converge after ~3 fixes — whether the trace shows a fait ### Artifacts -- `.opentaint/dataflow/` — the language-specific code approximation artifacts that the scan consumes; report the path and the exact test command used +- `.opentaint/dataflow/` — the batch's approximation project: its pinned dependencies plus the language-specific code approximation artifacts that the scan consumes, following the language reference's artifact layout; report the path and the exact test command used - the passing methods present in the batch file's `build.done` (new methods appended; repaired methods already recorded, per Tracking) ### Summary diff --git a/skills/create-dataflow-approximation/references/java.md b/skills/create-dataflow-approximation/references/java.md index 523eb6e5b7..7bb792615b 100644 --- a/skills/create-dataflow-approximation/references/java.md +++ b/skills/create-dataflow-approximation/references/java.md @@ -10,7 +10,16 @@ An app-internal method sits in the project's own sources under ``. ### 2. Write the approximation source -Create Java files under `.opentaint/dataflow/` — one `@Approximate` class per target class. `@Approximate(TargetClass.class)` binds a model to exactly that class, so target the EXACT class the analyzer dropped — the dropped FQN reflects how the call resolved: an interface-typed receiver (`Map m = ...; m.computeIfAbsent(...)`) drops `java.util.Map#computeIfAbsent`, a concrete one (`new HashMap<>()`) drops `java.util.HashMap#computeIfAbsent`. An interface is a valid target — write `@Approximate` on it exactly as on a concrete class, modelling whichever FQN the analyzer dropped. Reach the real object with `(TargetClass) (Object) this`, put functional-interface parameters behind `@ArgumentTypeContext`, and branch with `OpentaintNdUtil.nextBool()` so the analyzer walks both paths. Never leave a body empty. +The models live in the batch's own approximation project. Scaffold it once, passing each library the batch's models will reference at the exact version the batch file's `dependencies` records — the same pins the test project took. Those pins are the models' compile environment: nothing about the project under analysis affects it, so the models compile the same way at test time and at scan time. + +```bash +opentaint approximation init .opentaint/dataflow/ \ + --dependency "io.projectreactor:reactor-core:3.8.5" +``` + +Re-invoked for a batch whose project already exists, leave the build file alone unless a model needs a library that isn't pinned yet — then add it with a second `init` carrying the full dependency set, or edit `build.gradle.kts` directly. + +Create Java files under `.opentaint/dataflow//src/main/java` — one `@Approximate` class per target class. `@Approximate(TargetClass.class)` binds a model to exactly that class, so target the EXACT class the analyzer dropped — the dropped FQN reflects how the call resolved: an interface-typed receiver (`Map m = ...; m.computeIfAbsent(...)`) drops `java.util.Map#computeIfAbsent`, a concrete one (`new HashMap<>()`) drops `java.util.HashMap#computeIfAbsent`. An interface is a valid target — write `@Approximate` on it exactly as on a concrete class, modelling whichever FQN the analyzer dropped. Reach the real object with `(TargetClass) (Object) this`, put functional-interface parameters behind `@ArgumentTypeContext`, and branch with `OpentaintNdUtil.nextBool()` so the analyzer walks both paths. Never leave a body empty. ```java package com.example.approximations.batchpkg; // per-batch package (e.g. ...approximations.cn_hutool_001) — see the globally-unique rule below @@ -68,7 +77,13 @@ opentaint test approximation run .opentaint/test-compiled/ \ --dataflow-approximations .opentaint/dataflow/ ``` -`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI auto-compiles the `.java` sources against the analyzer JAR (for `@Approximate`, `OpentaintNdUtil`, `ArgumentTypeContext`) and the project's dependencies; if compilation fails it reports the errors and aborts before the tests. A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: +`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI builds the approximation project against the dependencies it pins (the approximation API — `@Approximate`, `OpentaintNdUtil`, `ArgumentTypeContext` — comes from the project's own `libs/`); if compilation fails it reports the errors and aborts before the tests. A rebuild happens only when the sources or the pins changed, so re-running an unedited batch skips straight to the tests. To see compilation errors on their own, compile the project alone: + +```bash +opentaint compile approximations .opentaint/dataflow/ +``` + +A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: ```bash uv run /scripts/check-test-result.py @@ -91,6 +106,7 @@ Fix by the verdict it reports: ## Constraints - Java 8 source compatibility +- A model may only reference libraries its own project pins. A model that needs a type from an unpinned library is a missing `--dependency`, not a reason to weaken the model to `Object` - Put the `@Approximate` classes in a neutral package (e.g. `com.example.approximations`) — never the target library's own package. Inside the library's package every bare FQN resolves to your approximation's non-generic class instead of the real type, breaking compilation wholesale - Namespace the package PER BATCH (`com.example.approximations.`, e.g. `...approximations.cn_hutool_001`). Every batch is loaded together into one shared index, and approximation class FQCNs must be GLOBALLY unique across all batches — two batches that both model a `…$Nested` (or any recurring simple name) in the bare `com.example.approximations` package collide on `com.example.approximations.Nested` and crash the whole scan at config-load. A per-batch package guarantees uniqueness. - One approximation class per target class — a strict global bijection enforced at load: each approximation FQCN maps to exactly one target and vice-versa; duplicates or a reused FQCN across batches throw `IllegalArgumentException` diff --git a/skills/create-test-project/references/java-approximation.md b/skills/create-test-project/references/java-approximation.md index e5152e14d2..8cee6ff26b 100644 --- a/skills/create-test-project/references/java-approximation.md +++ b/skills/create-test-project/references/java-approximation.md @@ -6,7 +6,7 @@ Language reference for a `dataflow` type, keyed to the body's steps. ### 1. Scaffold the project -`opentaint test approximation init ` scaffolds the Gradle build plus `Taint.java` and the fixed `approximation-rule.yaml` the harness applies. Pass each lib as a `--dependency` at its exact version from the app's dependency management (`.opentaint/project/sources/**/pom.xml`) verbatim — including timestamped snapshots — not a guess: this Gradle build is the approximation's own compile environment, so it must recompile from these pins even after the main project drops that dependency. +`opentaint test approximation init ` scaffolds the Gradle build plus `Taint.java` and the fixed `approximation-rule.yaml` the harness applies. Pass each lib as a `--dependency` at its exact version from the app's dependency management (`.opentaint/project/sources/**/pom.xml`) verbatim — including timestamped snapshots — not a guess. The approximation project scaffolded later takes the same pins, so the samples and the models they exercise compile against one version. ```bash opentaint test approximation init .opentaint/test-projects/ \ diff --git a/skills/run-scan/SKILL.md b/skills/run-scan/SKILL.md index 5425cb4cbf..65cc90f70d 100644 --- a/skills/run-scan/SKILL.md +++ b/skills/run-scan/SKILL.md @@ -34,7 +34,7 @@ opentaint scan --project-model .opentaint/project \ - `--rule-id ` — restrict to specific executable rules (repeatable, one per input rule ID); referenced library rules remain available without listing their IDs separately. Omit to run all loaded rules - `--passthrough-approximations .opentaint/pass-through` — add when that directory exists: passThrough configs override built-ins at the rule level, a provided rule overriding a built-in only when it matches one -- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations (sources auto-compiled; pre-compiled `.class` dirs passed through as-is) +- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations, one project per batch (each built on demand against the dependencies it pins, and rebuilt only when its sources change) Both approximation-dir flags walk their trees recursively; pass each parent directory once, not every package or batch separately. From 398738ef314f132c6adb1958ba12db34d4fa24e8 Mon Sep 17 00:00:00 2001 From: Aleksandr Misonizhnik Date: Fri, 4 Sep 2026 02:25:50 +0200 Subject: [PATCH 2/4] fix(cli): close three silent-loss paths in approximation resolution Three defects found by the model in formal/Opentaint/Cli/Internal/Approximation, each with the test the proof predicts. A directory counted as compiled classes as soon as a .class file existed anywhere below it. That is a test of the subtree, not a classification of the directory: a tree holding both a model project and compiled classes collapsed to one classpath root, the project was never compiled, and the scan ran without its models and reported nothing about it. Classify only when nothing below still needs building, and compute both facts in one walk rather than two (Resolve.fixed_loses_nothing, Resolve.shipped_loses_a_project, Resolve.fused_resolves_identically). A directory holding compiled classes of its own next to a project has no valid classification at all, so report it by name instead of dropping either side (Resolve.ambiguous_tree_drops_a_class). Builds staged in a directory named after the output alone, so two builders of one project shared it: one could delete the other's classes, contribute to them, and publish the mixture under a stamp recording it as up to date. Stage under a path carrying the builder's identity, and rename the previous output aside rather than deleting it, so a reader sees the old output, the new one, or nothing (Build.raceA_output_is_not_sound, Build.publishedOutputIsSound). The rebuild stamp digested the model project but not the compiler that reads it, so an upgraded CLI kept serving models compiled by the previous one. Carry the autobuilder's identity in the stamp (Stamp.shipped_stamp_serves_a_stale_output, Stamp.full_stamp_covers). --- cli/cmd/compile_approximations.go | 19 +++ .../approximation/approximation_test.go | 156 +++++++++++++++++- cli/internal/approximation/build.go | 75 +++++++-- cli/internal/approximation/project.go | 57 ++++++- cli/internal/approximation/resolve.go | 24 ++- cli/internal/approximation/stamp.go | 33 ++-- docs/usage.md | 2 +- 7 files changed, 331 insertions(+), 35 deletions(-) diff --git a/cli/cmd/compile_approximations.go b/cli/cmd/compile_approximations.go index b91f058688..52b08ba73f 100644 --- a/cli/cmd/compile_approximations.go +++ b/cli/cmd/compile_approximations.go @@ -7,6 +7,7 @@ import ( "github.com/seqra/opentaint/internal/approximation" "github.com/seqra/opentaint/internal/autobuilder" + "github.com/seqra/opentaint/internal/globals" "github.com/seqra/opentaint/internal/output" ) @@ -21,6 +22,24 @@ func newApproximationBuilder(analyzerJarPath string) approximation.Builder { return &approximationBuilder{analyzerJarPath: analyzerJarPath} } +// Fingerprint identifies the compiler a build was produced with. +// The source stamp covers the model project but not the tool that reads it, so without +// this an upgraded CLI keeps serving models compiled by the previous one. +func (b *approximationBuilder) Fingerprint() (string, error) { + autobuilder := globals.ArtifactByKind("autobuilder") + if autobuilder.Override == "" { + return "autobuilder " + autobuilder.Version, nil + } + + // A local autobuilder has no version to compare, so compare the file itself. + info, err := os.Stat(autobuilder.Override) + if err != nil { + return "", fmt.Errorf("failed to read the autobuilder at %s: %w", autobuilder.Override, err) + } + return fmt.Sprintf("autobuilder %s %d %d", + autobuilder.Override, info.Size(), info.ModTime().UnixNano()), nil +} + // Prepare updates the approximation API jar in the model project. // The builder reads the jar from the analyzer only once. func (b *approximationBuilder) Prepare(projectDir string) error { diff --git a/cli/internal/approximation/approximation_test.go b/cli/internal/approximation/approximation_test.go index 86ae019cb2..ade1a07a88 100644 --- a/cli/internal/approximation/approximation_test.go +++ b/cli/internal/approximation/approximation_test.go @@ -3,6 +3,7 @@ package approximation import ( "os" "path/filepath" + "reflect" "sort" "strings" "testing" @@ -12,12 +13,17 @@ import ( // fakeBuilder records builds and writes a test project model. type fakeBuilder struct { - t *testing.T - compiled []string + t *testing.T + compiled []string + fingerprint string } func newFakeBuilder(t *testing.T) *fakeBuilder { - return &fakeBuilder{t: t} + return &fakeBuilder{t: t, fingerprint: "autobuilder 1.0.0"} +} + +func (b *fakeBuilder) Fingerprint() (string, error) { + return b.fingerprint, nil } // Prepare writes the same API content for each call. @@ -116,6 +122,31 @@ func TestResolveReusesBuildUntilSourcesChange(t *testing.T) { } } +// The stamp covers the project, and the project alone does not determine the compiled +// classes: the compiler does too. See formal/Opentaint/Approximation/Cache.lean, theorem +// shipped_stamp_serves_a_stale_output. +func TestResolveRebuildsWhenTheCompilerChanges(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := newFakeBuilder(t) + + for range 2 { + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("Resolve: %v", err) + } + } + if len(builder.compiled) != 1 { + t.Fatalf("an unchanged project was rebuilt: %v", builder.compiled) + } + + builder.fingerprint = "autobuilder 2.0.0" + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("Resolve after the upgrade: %v", err) + } + if len(builder.compiled) != 2 { + t.Errorf("an upgraded compiler did not rebuild the models: %v", builder.compiled) + } +} + func TestResolveRebuildsWhenDependencyPinChanges(t *testing.T) { project := newProject(t, t.TempDir()) builder := newFakeBuilder(t) @@ -193,6 +224,84 @@ func TestResolveTreeOfProjects(t *testing.T) { } } +// A tree may mix model projects with directories that are already compiled. Resolving +// such a tree must build the project and keep the compiled classes, not collapse the +// whole tree into one classpath root: see formal/Opentaint/Approximation/Shipped.lean, +// theorem shipped_loses_a_project. +func TestResolveTreeMixingProjectAndClasses(t *testing.T) { + root := t.TempDir() + project := newProject(t, filepath.Join(root, "models")) + precompiled := filepath.Join(root, "precompiled") + writeFile(t, filepath.Join(precompiled, "com", "example", "Other.class"), "bytecode") + + builder := newFakeBuilder(t) + classDirs, err := Resolve(root, builder) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + want := []string{ClassesDir(BuildDir(project)), precompiled} + if !reflect.DeepEqual(classDirs, want) { + t.Fatalf("Resolve() = %v, want %v", classDirs, want) + } + if !reflect.DeepEqual(builder.compiled, []string{project}) { + t.Fatalf("expected the project to be compiled, got %v", builder.compiled) + } +} + +// The same tree with a previous build output in place of the loose classes. +func TestResolveTreeMixingProjectAndBuildOutput(t *testing.T) { + root := t.TempDir() + project := newProject(t, filepath.Join(root, "models")) + prebuilt := filepath.Join(root, "prebuilt") + writeFile(t, filepath.Join(prebuilt, descriptorName), "sourceProject: /elsewhere\n") + writeFile(t, filepath.Join(prebuilt, classesDirName, "com", "example", "Other.class"), "bytecode") + + builder := newFakeBuilder(t) + classDirs, err := Resolve(root, builder) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + want := []string{ClassesDir(BuildDir(project)), ClassesDir(prebuilt)} + if !reflect.DeepEqual(classDirs, want) { + t.Fatalf("Resolve() = %v, want %v", classDirs, want) + } + if !reflect.DeepEqual(builder.compiled, []string{project}) { + t.Fatalf("expected the project to be compiled, got %v", builder.compiled) + } +} + +// A directory that holds classes of its own and a project below it belongs to neither +// shape: putting it on the classpath hides the project, recursing past it drops the +// classes. See formal/Opentaint/Approximation/Classes.lean, ambiguous_tree_drops_a_class. +func TestResolveRejectsAmbiguousDirectory(t *testing.T) { + root := t.TempDir() + newProject(t, filepath.Join(root, "models")) + writeFile(t, filepath.Join(root, "Stray.class"), "bytecode") + + _, err := Resolve(root, newFakeBuilder(t)) + if err == nil { + t.Fatal("expected a directory holding both classes and a project to be rejected") + } + if !strings.Contains(err.Error(), root) { + t.Errorf("error should name the ambiguous directory, got: %v", err) + } +} + +// A plain class directory resolves to its root, not to the package directory that +// happens to hold the class files. +func TestResolvePrecompiledClassDirectoryKeepsItsRoot(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "com", "example", "Model.class"), "bytecode") + + classDirs, err := Resolve(root, newFakeBuilder(t)) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if !reflect.DeepEqual(classDirs, []string{root}) { + t.Fatalf("Resolve() = %v, want the classpath root %s", classDirs, root) + } +} + func TestResolveLooseSourcesReportsMigration(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, "com", "example", "Model.java"), "package com.example;\n") @@ -245,6 +354,47 @@ func TestFailedRebuildKeepsThePreviousBuild(t *testing.T) { } } +// A build assembles under a path of its own. A second builder of the same output must +// not be able to delete or contribute to it: see formal/Opentaint/Approximation/ +// Publish.lean, theorem raceA_output_is_not_sound. +func TestBuildDoesNotTouchAnotherBuildersStaging(t *testing.T) { + project := newProject(t, t.TempDir()) + output := BuildDir(project) + + // What another builder of the same output would be assembling. + foreign := output + ".incomplete" + writeFile(t, filepath.Join(foreign, classesDirName, "com", "example", "Other.class"), "bytecode") + + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("Build: %v", err) + } + if _, err := os.Stat(filepath.Join(foreign, classesDirName, "com", "example", "Other.class")); err != nil { + t.Errorf("the build deleted another builder's staging directory: %v", err) + } +} + +// A completed build leaves no staging or superseded directory behind. +func TestBuildCleansUpAfterItself(t *testing.T) { + project := newProject(t, t.TempDir()) + output := BuildDir(project) + + for range 2 { + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("Build: %v", err) + } + } + + entries, err := os.ReadDir(filepath.Dir(output)) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.Name() != filepath.Base(output) { + t.Errorf("build left %s behind next to the output", entry.Name()) + } + } +} + func TestBuildRefusesToOverwriteUnrelatedDirectory(t *testing.T) { project := newProject(t, t.TempDir()) output := t.TempDir() diff --git a/cli/internal/approximation/build.go b/cli/internal/approximation/build.go index e0dbd29218..382ebfe5af 100644 --- a/cli/internal/approximation/build.go +++ b/cli/internal/approximation/build.go @@ -1,5 +1,7 @@ package approximation +// Golden model: formal/Opentaint/Cli/Internal/Approximation/Build.lean + import ( "fmt" "os" @@ -11,6 +13,9 @@ import ( // Builder compiles a model project. type Builder interface { + // Fingerprint identifies the compiler. A build is reused only for the compiler + // that produced it. + Fingerprint() (string, error) // Prepare updates local build inputs before the source stamp is calculated. Prepare(projectDir string) error // Compile writes a project model to projectModelDir. @@ -19,16 +24,16 @@ type Builder interface { // ensureBuilt returns current compiled models for a project. func ensureBuilt(projectDir string, builder Builder) (string, error) { - sources, err := prepare(projectDir, builder) + inputs, err := prepare(projectDir, builder) if err != nil { return "", err } outputDir := BuildDir(projectDir) - if upToDate(outputDir, sources) { + if upToDate(outputDir, inputs) { return ClassesDir(outputDir), nil } - if err := build(projectDir, outputDir, sources, builder); err != nil { + if err := build(projectDir, outputDir, inputs, builder); err != nil { return "", err } return ClassesDir(outputDir), nil @@ -36,25 +41,42 @@ func ensureBuilt(projectDir string, builder Builder) (string, error) { // Build compiles a model project into outputDir. func Build(projectDir, outputDir string, builder Builder) error { - sources, err := prepare(projectDir, builder) + inputs, err := prepare(projectDir, builder) if err != nil { return err } - return build(projectDir, outputDir, sources, builder) + return build(projectDir, outputDir, inputs, builder) } -// prepare updates build inputs and returns their source stamp. -func prepare(projectDir string, builder Builder) (string, error) { +// prepare updates build inputs and returns the stamp identifying them. +func prepare(projectDir string, builder Builder) (stamp, error) { + toolchain, err := builder.Fingerprint() + if err != nil { + return stamp{}, err + } if err := builder.Prepare(projectDir); err != nil { - return "", err + return stamp{}, err } - return computeStamp(projectDir) + sources, err := computeSourceDigest(projectDir) + if err != nil { + return stamp{}, err + } + return stamp{Sources: sources, Toolchain: toolchain}, nil +} + +// stagingDir names the directory a build assembles in. The process id keeps concurrent +// builders of one output out of each other's staging directory: a shared one lets a build +// publish a mixture of two compiles, and record it as up to date. +func stagingDir(outputDir string) string { + return fmt.Sprintf("%s.incomplete-%d", outputDir, os.Getpid()) } -// stagingSuffix identifies a build that is not complete. -const stagingSuffix = ".incomplete" +// asideDir names the superseded output while the new one is renamed into place. +func asideDir(outputDir string) string { + return fmt.Sprintf("%s.superseded-%d", outputDir, os.Getpid()) +} -func build(projectDir, outputDir, sources string, builder Builder) error { +func build(projectDir, outputDir string, inputs stamp, builder Builder) error { if err := checkOutputDir(outputDir); err != nil { return err } @@ -73,7 +95,7 @@ func build(projectDir, outputDir, sources string, builder Builder) error { } // Assemble the new output before it replaces the current output. - staging := outputDir + stagingSuffix + staging := stagingDir(outputDir) defer func() { _ = os.RemoveAll(staging) }() if err := os.RemoveAll(staging); err != nil { return fmt.Errorf("failed to clear %s: %w", staging, err) @@ -93,7 +115,7 @@ func build(projectDir, outputDir, sources string, builder Builder) error { if err := writeDescriptor(staging, descriptor{SourceProject: projectDir, Dependencies: dependencies}); err != nil { return err } - if err := writeStamp(staging, sources); err != nil { + if err := writeStamp(staging, inputs); err != nil { return err } @@ -129,16 +151,37 @@ func checkOutputDir(outputDir string) error { } // replaceDir replaces the current output with a complete build. +// Both moves are single renames, so anyone reading the output directory sees the previous +// build, the new build, or nothing at all, and never a directory that is half of each. func replaceDir(staging, outputDir string) error { if err := os.MkdirAll(filepath.Dir(outputDir), 0o755); err != nil { return fmt.Errorf("failed to create %s: %w", filepath.Dir(outputDir), err) } - if err := os.RemoveAll(outputDir); err != nil { - return fmt.Errorf("failed to clear approximation build output %s: %w", outputDir, err) + + aside := asideDir(outputDir) + if err := os.RemoveAll(aside); err != nil { + return fmt.Errorf("failed to clear %s: %w", aside, err) + } + superseded := isDir(outputDir) + if superseded { + if err := os.Rename(outputDir, aside); err != nil { + return fmt.Errorf("failed to move the previous build out of %s: %w", outputDir, err) + } } + if err := os.Rename(staging, outputDir); err != nil { + // Put the previous build back rather than leave the output missing. + if superseded { + _ = os.Rename(aside, outputDir) + } return fmt.Errorf("failed to move the compiled models into %s: %w", outputDir, err) } + + if superseded { + if err := os.RemoveAll(aside); err != nil { + return fmt.Errorf("failed to remove the previous build %s: %w", aside, err) + } + } return nil } diff --git a/cli/internal/approximation/project.go b/cli/internal/approximation/project.go index 4e1fe57fdc..3f26fee206 100644 --- a/cli/internal/approximation/project.go +++ b/cli/internal/approximation/project.go @@ -4,6 +4,8 @@ // The scanned project does not supply these compile dependencies. package approximation +// Golden model: formal/Opentaint/Cli/Internal/Approximation/Project.lean + import ( "os" "path/filepath" @@ -74,7 +76,7 @@ func containsClassFiles(dir string) bool { } return nil } - if strings.EqualFold(filepath.Ext(entry.Name()), ".class") { + if isClassFileName(entry.Name()) { found = true return filepath.SkipAll } @@ -83,6 +85,59 @@ func containsClassFiles(dir string) bool { return found } +// subtreeFacts records what a directory tree holds. +// Classifying a directory needs both facts, and one walk answers both. +type subtreeFacts struct { + // classFiles reports a compiled class anywhere in the tree. + classFiles bool + // unitSource reports a model project or a build output anywhere in the tree, + // that is, models this directory does not already hold compiled. + unitSource bool +} + +// scanSubtree walks dir once. It does not search generated directories. +func scanSubtree(dir string) subtreeFacts { + var facts subtreeFacts + _ = filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if path != dir && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + switch { + case isClassFileName(entry.Name()): + facts.classFiles = true + case entry.Name() == gradleBuildFile: + facts.unitSource = true + case entry.Name() == descriptorName && isDir(filepath.Join(filepath.Dir(path), classesDirName)): + facts.unitSource = true + } + if facts.classFiles && facts.unitSource { + return filepath.SkipAll + } + return nil + }) + return facts +} + +// directClassFile returns one compiled class held by the directory itself. +func directClassFile(entries []os.DirEntry) (string, bool) { + for _, entry := range entries { + if !entry.IsDir() && isClassFileName(entry.Name()) { + return entry.Name(), true + } + } + return "", false +} + +func isClassFileName(name string) bool { + return strings.EqualFold(filepath.Ext(name), ".class") +} + // firstJavaSource returns one Java source file under dir. func firstJavaSource(dir string) (string, bool) { var found string diff --git a/cli/internal/approximation/resolve.go b/cli/internal/approximation/resolve.go index e3be9197ce..a7e3ae9697 100644 --- a/cli/internal/approximation/resolve.go +++ b/cli/internal/approximation/resolve.go @@ -1,5 +1,7 @@ package approximation +// Golden model: formal/Opentaint/Cli/Internal/Approximation/Resolve.lean + import ( "fmt" "os" @@ -46,7 +48,11 @@ func resolveDir(dir string, builder Builder) ([]string, error) { return []string{ClassesDir(dir)}, nil } - if containsClassFiles(dir) { + // A directory is a class directory only when nothing below it still needs building. + // Without that condition a tree holding both a project and compiled classes reads as + // one class directory, and the project's models never reach the analysis. + facts := scanSubtree(dir) + if facts.classFiles && !facts.unitSource { return []string{dir}, nil } @@ -56,6 +62,11 @@ func resolveDir(dir string, builder Builder) ([]string, error) { } sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + // Descending past classes this directory holds itself would leave them unused. + if classFile, found := directClassFile(entries); found { + return nil, ambiguousDirError(dir, classFile) + } + var classDirs []string for _, entry := range entries { if !entry.IsDir() || skippedDirs[entry.Name()] { @@ -70,6 +81,17 @@ func resolveDir(dir string, builder Builder) ([]string, error) { return classDirs, nil } +// ambiguousDirError reports a directory that is neither a class directory nor a +// directory of approximation paths. +func ambiguousDirError(dir, classFile string) error { + return fmt.Errorf( + "%s holds compiled classes of its own (e.g. %s) and an approximation project or build "+ + "output below it, so it is neither a class directory nor a directory of them.\n"+ + "Move the classes into a directory of their own, or pass each approximation path to "+ + "--dataflow-approximations separately", + dir, classFile) +} + // emptyPathError reports why a path does not contain usable models. func emptyPathError(path string) error { if source, found := firstJavaSource(path); found { diff --git a/cli/internal/approximation/stamp.go b/cli/internal/approximation/stamp.go index 99b25a9126..587a349113 100644 --- a/cli/internal/approximation/stamp.go +++ b/cli/internal/approximation/stamp.go @@ -1,5 +1,7 @@ package approximation +// Golden model: formal/Opentaint/Cli/Internal/Approximation/Stamp.lean + import ( "crypto/sha256" "encoding/hex" @@ -11,14 +13,19 @@ import ( "sort" ) -// stamp identifies the inputs of a model build. +// stamp identifies the inputs of a model build: the project, and the compiler that read +// it. The project alone does not determine the compiled classes, so a build carrying only +// a source digest would be reused across a compiler upgrade. type stamp struct { - Sources string `json:"sources"` + Sources string `json:"sources"` + Toolchain string `json:"toolchain"` } -// computeStamp hashes all source files and build inputs. +func (s stamp) complete() bool { return s.Sources != "" && s.Toolchain != "" } + +// computeSourceDigest hashes all source files and build inputs. // It does not hash generated files. -func computeStamp(projectDir string) (string, error) { +func computeSourceDigest(projectDir string) (string, error) { var paths []string err := filepath.WalkDir(projectDir, func(path string, entry os.DirEntry, err error) error { if err != nil { @@ -60,30 +67,30 @@ func computeStamp(projectDir string) (string, error) { return hex.EncodeToString(digest.Sum(nil)), nil } -func readStamp(outputDir string) (string, bool) { +func readStamp(outputDir string) (stamp, bool) { data, err := os.ReadFile(filepath.Join(outputDir, stampFileName)) if err != nil { - return "", false + return stamp{}, false } var s stamp if err := json.Unmarshal(data, &s); err != nil { - return "", false + return stamp{}, false } - return s.Sources, s.Sources != "" + return s, s.complete() } -func writeStamp(outputDir, sources string) error { - data, err := json.MarshalIndent(stamp{Sources: sources}, "", " ") +func writeStamp(outputDir string, s stamp) error { + data, err := json.MarshalIndent(s, "", " ") if err != nil { return err } return os.WriteFile(filepath.Join(outputDir, stampFileName), append(data, '\n'), 0o644) } -// upToDate reports whether outputDir matches the source stamp. -func upToDate(outputDir, sources string) bool { +// upToDate reports whether outputDir was built from these inputs. +func upToDate(outputDir string, current stamp) bool { previous, ok := readStamp(outputDir) - if !ok || previous != sources { + if !ok || previous != current { return false } return containsClassFiles(ClassesDir(outputDir)) diff --git a/docs/usage.md b/docs/usage.md index e3c94c06ce..b47036342a 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -205,7 +205,7 @@ opentaint scan --project-model ./my-project-model \ Pin the versions the target application uses. Those pins are the models' compile environment: they, and nothing about the project under analysis, decide what the models compile against, so a model compiles identically wherever it is applied. -`--dataflow-approximations` accepts an approximation project, a build output, or a directory of compiled classes — and a directory holding any of those, so a tree with one project per batch can be passed as a single flag. Projects are built on demand and rebuilt only when their sources or dependency pins change. +`--dataflow-approximations` accepts an approximation project, a build output, or a directory of compiled classes — and a directory holding any of those, so a tree with one project per batch can be passed as a single flag. A directory counts as compiled classes only when nothing below it still needs building; one that holds compiled classes of its own alongside a project is reported rather than guessed at. Projects are built on demand and rebuilt when their sources, their dependency pins, or the compiler change. | Command | Description | |---------|-------------| From 8517ab99e0d8f0cbd1c4cb68980bae50c7efef0a Mon Sep 17 00:00:00 2001 From: Aleksandr Misonizhnik Date: Mon, 7 Sep 2026 21:47:35 +0200 Subject: [PATCH 3/4] fix(cli): pin the approximation classpath and reclaim abandoned builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remaining findings from the stack review. Two resolved class directories could offer the same class with different content, and nothing says which one an analysis reads, so the finding set depended on a choice the CLI never made. ResolveAll resolves every --dataflow-approximations path together, passes each directory once, and rejects a classpath whose directories disagree about a class they share. Identical copies are the ordinary result of overlapping paths and stay free (Resolve.agreement_pins_the_choice, Resolve.identical_roots_agree, Resolve.duplicate_root_is_harmless). Staging directories now live in a .opentaint-staging directory beside the output, on the skip list, so that what a killed build leaves behind is never resolved as compiled models — half-built classes applied to a scan is the same silent-loss shape as the rest of this review. The next build reclaims them, but only once the owning process is gone, so a build in progress is never disturbed; sweeping cannot reach the published output at all (Build.sweep_leaves_the_output). A build output recorded the absolute path of the project it came from, so it meant nothing on another machine. Record it relative to the output. A javac error count is a trailer for the diagnostics above it, not a diagnostic: a line that merely reads like one no longer stands in for the compiler output. Also drop a redundant sort of os.ReadDir output and a deferred close inside a loop. --- cli/cmd/analyzer_inputs.go | 25 ++-- .../approximation/approximation_test.go | 113 +++++++++++++++--- cli/internal/approximation/build.go | 90 ++++++++++++-- cli/internal/approximation/descriptor.go | 12 +- cli/internal/approximation/project.go | 48 +++++++- cli/internal/approximation/resolve.go | 67 ++++++++++- cli/internal/approximation/scaffold.go | 19 +-- cli/internal/approximation/stamp.go | 2 - cli/internal/autobuilder/buildlog.go | 12 +- cli/internal/autobuilder/buildlog_test.go | 12 +- docs/usage.md | 2 +- 11 files changed, 345 insertions(+), 57 deletions(-) diff --git a/cli/cmd/analyzer_inputs.go b/cli/cmd/analyzer_inputs.go index e13a7d5680..d9fabfe8b4 100644 --- a/cli/cmd/analyzer_inputs.go +++ b/cli/cmd/analyzer_inputs.go @@ -5,19 +5,24 @@ import ( "github.com/seqra/opentaint/internal/utils/log" ) -// addDataflowApproximations resolves each path to compiled model directories. +// addDataflowApproximations resolves the paths to compiled model directories. // It rebuilds a model project when its inputs change. func addDataflowApproximations(b *AnalyzerBuilder, paths []string, analyzerJarPath string) { - builder := newApproximationBuilder(analyzerJarPath) + if len(paths) == 0 { + return + } + + absPaths := make([]string, 0, len(paths)) for _, approxPath := range paths { - absApproxPath := log.AbsPathOrExit(approxPath, "dataflow-approximations") - classDirs, err := approximation.Resolve(absApproxPath, builder) - if err != nil { - out.Fatalf("Failed to apply dataflow approximations: %s", err) - } - for _, classDir := range classDirs { - b.AddDataflowApproximations(classDir) - } + absPaths = append(absPaths, log.AbsPathOrExit(approxPath, "dataflow-approximations")) + } + + classDirs, err := approximation.ResolveAll(absPaths, newApproximationBuilder(analyzerJarPath)) + if err != nil { + out.Fatalf("Failed to apply dataflow approximations: %s", err) + } + for _, classDir := range classDirs { + b.AddDataflowApproximations(classDir) } } diff --git a/cli/internal/approximation/approximation_test.go b/cli/internal/approximation/approximation_test.go index ade1a07a88..619f83333d 100644 --- a/cli/internal/approximation/approximation_test.go +++ b/cli/internal/approximation/approximation_test.go @@ -123,8 +123,7 @@ func TestResolveReusesBuildUntilSourcesChange(t *testing.T) { } // The stamp covers the project, and the project alone does not determine the compiled -// classes: the compiler does too. See formal/Opentaint/Approximation/Cache.lean, theorem -// shipped_stamp_serves_a_stale_output. +// classes: the compiler does too. func TestResolveRebuildsWhenTheCompilerChanges(t *testing.T) { project := newProject(t, t.TempDir()) builder := newFakeBuilder(t) @@ -226,8 +225,7 @@ func TestResolveTreeOfProjects(t *testing.T) { // A tree may mix model projects with directories that are already compiled. Resolving // such a tree must build the project and keep the compiled classes, not collapse the -// whole tree into one classpath root: see formal/Opentaint/Approximation/Shipped.lean, -// theorem shipped_loses_a_project. +// whole tree into one classpath root. func TestResolveTreeMixingProjectAndClasses(t *testing.T) { root := t.TempDir() project := newProject(t, filepath.Join(root, "models")) @@ -272,7 +270,7 @@ func TestResolveTreeMixingProjectAndBuildOutput(t *testing.T) { // A directory that holds classes of its own and a project below it belongs to neither // shape: putting it on the classpath hides the project, recursing past it drops the -// classes. See formal/Opentaint/Approximation/Classes.lean, ambiguous_tree_drops_a_class. +// classes. func TestResolveRejectsAmbiguousDirectory(t *testing.T) { root := t.TempDir() newProject(t, filepath.Join(root, "models")) @@ -302,6 +300,59 @@ func TestResolvePrecompiledClassDirectoryKeepsItsRoot(t *testing.T) { } } +// The analyzer is handed every path's answer at once, so a directory reached twice must +// contribute once. +func TestResolveAllPassesEachDirectoryOnce(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := newFakeBuilder(t) + + classDirs, err := ResolveAll([]string{project, project}, builder) + if err != nil { + t.Fatalf("ResolveAll: %v", err) + } + if !reflect.DeepEqual(classDirs, []string{ClassesDir(BuildDir(project))}) { + t.Fatalf("ResolveAll() = %v, want the class directory once", classDirs) + } +} + +// Copies of one class are the ordinary result of passing overlapping paths, and the +// analyzer reads the same class whichever copy it picks. +func TestResolveAllAcceptsIdenticalCopiesOfAClass(t *testing.T) { + root := t.TempDir() + first := filepath.Join(root, "first") + second := filepath.Join(root, "second") + writeFile(t, filepath.Join(first, "com", "example", "Model.class"), "bytecode") + writeFile(t, filepath.Join(second, "com", "example", "Model.class"), "bytecode") + + classDirs, err := ResolveAll([]string{first, second}, newFakeBuilder(t)) + if err != nil { + t.Fatalf("ResolveAll: %v", err) + } + if !reflect.DeepEqual(classDirs, []string{first, second}) { + t.Fatalf("ResolveAll() = %v, want both directories", classDirs) + } +} + +// Two directories offering one class with different content leave the analyzer to pick, +// and nothing says which it picks: the directories must agree on the class content. +func TestResolveAllRejectsClashingClasses(t *testing.T) { + root := t.TempDir() + first := filepath.Join(root, "first") + second := filepath.Join(root, "second") + writeFile(t, filepath.Join(first, "com", "example", "Model.class"), "one bytecode") + writeFile(t, filepath.Join(second, "com", "example", "Model.class"), "another bytecode") + + _, err := ResolveAll([]string{first, second}, newFakeBuilder(t)) + if err == nil { + t.Fatal("expected two different copies of one class to be rejected") + } + for _, want := range []string{"com/example/Model.class", first, second} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error should mention %s, got: %v", want, err) + } + } +} + func TestResolveLooseSourcesReportsMigration(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, "com", "example", "Model.java"), "package com.example;\n") @@ -354,22 +405,51 @@ func TestFailedRebuildKeepsThePreviousBuild(t *testing.T) { } } -// A build assembles under a path of its own. A second builder of the same output must -// not be able to delete or contribute to it: see formal/Opentaint/Approximation/ -// Publish.lean, theorem raceA_output_is_not_sound. -func TestBuildDoesNotTouchAnotherBuildersStaging(t *testing.T) { +// A build assembles under a path of its own. A second builder of the same output must not +// be able to delete or contribute to it. +func TestBuildKeepsStagingOfARunningBuild(t *testing.T) { project := newProject(t, t.TempDir()) output := BuildDir(project) - // What another builder of the same output would be assembling. - foreign := output + ".incomplete" + // What another builder of the same output would be assembling. Process 1 always runs. + foreign := filepath.Join(stagingRoot(output), filepath.Base(output)+"-1") writeFile(t, filepath.Join(foreign, classesDirName, "com", "example", "Other.class"), "bytecode") if err := Build(project, output, newFakeBuilder(t)); err != nil { t.Fatalf("Build: %v", err) } if _, err := os.Stat(filepath.Join(foreign, classesDirName, "com", "example", "Other.class")); err != nil { - t.Errorf("the build deleted another builder's staging directory: %v", err) + t.Errorf("the build deleted a running builder's staging directory: %v", err) + } +} + +// A build killed outright leaves its staging directory behind. The next build reclaims it, +// but only once its owner is gone. +func TestBuildSweepsStagingOfAKilledBuild(t *testing.T) { + project := newProject(t, t.TempDir()) + output := BuildDir(project) + + abandoned := filepath.Join(stagingRoot(output), filepath.Base(output)+"-2147483646") + writeFile(t, filepath.Join(abandoned, classesDirName, "com", "example", "Other.class"), "bytecode") + + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("Build: %v", err) + } + if _, err := os.Stat(abandoned); !os.IsNotExist(err) { + t.Errorf("a killed build's staging directory was not reclaimed: %v", err) + } +} + +// Whatever is left behind must be invisible to the resolver: staged classes are not +// compiled models, and resolving them would apply a half-built batch to a scan. +func TestStagingIsInvisibleToTheResolver(t *testing.T) { + root := t.TempDir() + output := filepath.Join(root, "out") + abandoned := filepath.Join(stagingRoot(output), filepath.Base(output)+"-1") + writeFile(t, filepath.Join(abandoned, classesDirName, "com", "example", "Other.class"), "bytecode") + + if _, err := Resolve(root, newFakeBuilder(t)); err == nil { + t.Error("a leftover staging directory was resolved as compiled models") } } @@ -443,8 +523,13 @@ func TestBuildWritesReadableDescriptor(t *testing.T) { if err := yaml.Unmarshal(raw, &parsed); err != nil { t.Fatalf("descriptor is not valid YAML: %v\n%s", err, raw) } - if parsed.SourceProject != project { - t.Errorf("sourceProject = %q, want %q", parsed.SourceProject, project) + // The descriptor must mean the same thing wherever the output is read, so it may not + // carry a path that only exists on the machine that built it. + if filepath.IsAbs(parsed.SourceProject) { + t.Errorf("sourceProject = %q, want a path relative to the output", parsed.SourceProject) + } + if resolved := filepath.Clean(filepath.Join(output, parsed.SourceProject)); resolved != project { + t.Errorf("sourceProject resolves to %q, want %q", resolved, project) } if parsed.Dependencies == nil { t.Error("dependencies should be an empty list, not null") diff --git a/cli/internal/approximation/build.go b/cli/internal/approximation/build.go index 382ebfe5af..20f1a89d97 100644 --- a/cli/internal/approximation/build.go +++ b/cli/internal/approximation/build.go @@ -1,11 +1,14 @@ package approximation -// Golden model: formal/Opentaint/Cli/Internal/Approximation/Build.lean - import ( + "errors" "fmt" "os" "path/filepath" + "runtime" + "strconv" + "strings" + "syscall" "github.com/seqra/opentaint/internal/utils" "github.com/seqra/opentaint/internal/utils/project" @@ -64,22 +67,82 @@ func prepare(projectDir string, builder Builder) (stamp, error) { return stamp{Sources: sources, Toolchain: toolchain}, nil } -// stagingDir names the directory a build assembles in. The process id keeps concurrent -// builders of one output out of each other's staging directory: a shared one lets a build -// publish a mixture of two compiles, and record it as up to date. +// stagingDirName holds the directories builds assemble in. It sits beside the output so +// that publishing is a single rename, and it is on the skip list so that a directory a +// killed build left behind is never mistaken for compiled models. +const stagingDirName = ".opentaint-staging" + +func stagingRoot(outputDir string) string { + return filepath.Join(filepath.Dir(outputDir), stagingDirName) +} + +// stagingDir names this process's staging directory for one output. The process id keeps +// concurrent builders of one output out of each other's staging directory: a shared one +// lets a build publish a mixture of two compiles, and record it as up to date. func stagingDir(outputDir string) string { - return fmt.Sprintf("%s.incomplete-%d", outputDir, os.Getpid()) + return filepath.Join(stagingRoot(outputDir), + fmt.Sprintf("%s-%d", filepath.Base(outputDir), os.Getpid())) } // asideDir names the superseded output while the new one is renamed into place. func asideDir(outputDir string) string { - return fmt.Sprintf("%s.superseded-%d", outputDir, os.Getpid()) + return filepath.Join(stagingRoot(outputDir), + fmt.Sprintf("%s-superseded-%d", filepath.Base(outputDir), os.Getpid())) +} + +// sweepStaging reclaims staging directories left behind by builds that are no longer +// running. A directory is removed only when its owner is known to be gone, so a build in +// progress is never disturbed; being wrong the other way costs a directory that stays +// until its process id comes round again. +func sweepStaging(outputDir string) { + root := stagingRoot(outputDir) + entries, err := os.ReadDir(root) + if err != nil { + return + } + prefix := filepath.Base(outputDir) + "-" + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), prefix) { + continue + } + owner, named := stagingOwner(entry.Name()) + if !named || owner == os.Getpid() || processRunning(owner) { + continue + } + _ = os.RemoveAll(filepath.Join(root, entry.Name())) + } +} + +// stagingOwner reads the process id a staging directory name ends with. +func stagingOwner(name string) (int, bool) { + separator := strings.LastIndex(name, "-") + if separator < 0 { + return 0, false + } + owner, err := strconv.Atoi(name[separator+1:]) + return owner, err == nil +} + +// processRunning reports whether a process is still running. It errs towards "running": +// a staging directory is reclaimed only when its owner is known to be gone. +func processRunning(pid int) bool { + if runtime.GOOS == "windows" { + return true + } + process, err := os.FindProcess(pid) + if err != nil { + return true + } + // A process that exists but belongs to someone else answers with EPERM. + err = process.Signal(syscall.Signal(0)) + return err == nil || errors.Is(err, syscall.EPERM) } func build(projectDir, outputDir string, inputs stamp, builder Builder) error { if err := checkOutputDir(outputDir); err != nil { return err } + sweepStaging(outputDir) buildTempDir, err := os.MkdirTemp("", "opentaint-approximation-build-*") if err != nil { @@ -112,14 +175,23 @@ func build(projectDir, outputDir string, inputs stamp, builder Builder) error { return fmt.Errorf("approximation project %s compiled to no classes; it has no models under src/main/java", projectDir) } - if err := writeDescriptor(staging, descriptor{SourceProject: projectDir, Dependencies: dependencies}); err != nil { + if err := writeDescriptor(staging, descriptor{ + SourceProject: relativeSourceProject(outputDir, projectDir), + Dependencies: dependencies, + }); err != nil { return err } if err := writeStamp(staging, inputs); err != nil { return err } - return replaceDir(staging, outputDir) + if err := replaceDir(staging, outputDir); err != nil { + return err + } + + // Leave nothing beside the output once no build is using it. + _ = os.Remove(stagingRoot(outputDir)) + return nil } // checkOutputDir prevents replacement of an unrelated directory. diff --git a/cli/internal/approximation/descriptor.go b/cli/internal/approximation/descriptor.go index 1a088fada4..cb118783ce 100644 --- a/cli/internal/approximation/descriptor.go +++ b/cli/internal/approximation/descriptor.go @@ -12,12 +12,22 @@ const descriptorHeader = "# Generated by 'opentaint compile approximations'. Do // descriptor identifies the source and dependencies of a model build. type descriptor struct { - // SourceProject is the project that supplied the models. + // SourceProject is the project that supplied the models, relative to this directory. SourceProject string `yaml:"sourceProject"` // Dependencies are the libraries used to compile the models. Dependencies []string `yaml:"dependencies"` } +// relativeSourceProject expresses the model project relative to the build output, so that +// a build output says the same thing on every machine that reads it. +func relativeSourceProject(outputDir, projectDir string) string { + relative, err := filepath.Rel(outputDir, projectDir) + if err != nil { + return filepath.Base(projectDir) + } + return filepath.ToSlash(relative) +} + func writeDescriptor(outputDir string, d descriptor) error { if d.Dependencies == nil { d.Dependencies = []string{} diff --git a/cli/internal/approximation/project.go b/cli/internal/approximation/project.go index 3f26fee206..eb8a721d94 100644 --- a/cli/internal/approximation/project.go +++ b/cli/internal/approximation/project.go @@ -4,9 +4,9 @@ // The scanned project does not supply these compile dependencies. package approximation -// Golden model: formal/Opentaint/Cli/Internal/Approximation/Project.lean - import ( + "bytes" + "fmt" "os" "path/filepath" "strings" @@ -32,6 +32,7 @@ const opentaintDirName = ".opentaint" // skippedDirs contains generated and version-control directories. var skippedDirs = map[string]bool{ opentaintDirName: true, + stagingDirName: true, "build": true, ".gradle": true, ".git": true, @@ -138,6 +139,49 @@ func isClassFileName(name string) bool { return strings.EqualFold(filepath.Ext(name), ".class") } +// classFilesIn returns the compiled classes a class directory offers, as slash-separated +// paths relative to that directory. +func classFilesIn(dir string) ([]string, error) { + var classes []string + err := filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if path != dir && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + if !isClassFileName(entry.Name()) { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + classes = append(classes, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to read the compiled models in %s: %w", dir, err) + } + return classes, nil +} + +// sameContent reports whether two files hold the same bytes. +func sameContent(first, second string) (bool, error) { + firstBytes, err := os.ReadFile(first) + if err != nil { + return false, fmt.Errorf("failed to read %s: %w", first, err) + } + secondBytes, err := os.ReadFile(second) + if err != nil { + return false, fmt.Errorf("failed to read %s: %w", second, err) + } + return bytes.Equal(firstBytes, secondBytes), nil +} + // firstJavaSource returns one Java source file under dir. func firstJavaSource(dir string) (string, bool) { var found string diff --git a/cli/internal/approximation/resolve.go b/cli/internal/approximation/resolve.go index a7e3ae9697..c0505e3474 100644 --- a/cli/internal/approximation/resolve.go +++ b/cli/internal/approximation/resolve.go @@ -1,14 +1,72 @@ package approximation -// Golden model: formal/Opentaint/Cli/Internal/Approximation/Resolve.lean - import ( "fmt" "os" "path/filepath" - "sort" ) +// ResolveAll returns the class directories for every model path, in the order the paths +// were given. A directory reached more than once contributes once, and the directories +// must agree about every class they share: which copy an analysis reads is not defined, so +// two different copies of one class would leave the result of the scan unpinned. +func ResolveAll(paths []string, builder Builder) ([]string, error) { + var classDirs []string + resolved := make(map[string]bool) + for _, path := range paths { + dirs, err := Resolve(path, builder) + if err != nil { + return nil, err + } + for _, dir := range dirs { + dir = filepath.Clean(dir) + if resolved[dir] { + continue + } + resolved[dir] = true + classDirs = append(classDirs, dir) + } + } + if err := checkClasspathAgrees(classDirs); err != nil { + return nil, err + } + return classDirs, nil +} + +// checkClasspathAgrees rejects class directories that offer one class with different +// content. Identical copies are the ordinary result of overlapping paths and are left +// alone: what has to hold is that the same class is read whichever copy is picked. +func checkClasspathAgrees(classDirs []string) error { + offeredBy := make(map[string]string) + for _, dir := range classDirs { + classes, err := classFilesIn(dir) + if err != nil { + return err + } + for _, class := range classes { + previous, offered := offeredBy[class] + if !offered { + offeredBy[class] = dir + continue + } + same, err := sameContent( + filepath.Join(previous, filepath.FromSlash(class)), + filepath.Join(dir, filepath.FromSlash(class))) + if err != nil { + return err + } + if !same { + return fmt.Errorf( + "%s is offered with different content by %s and by %s; which one an analysis "+ + "reads is not defined.\nDrop one of the copies, or pass only one of these "+ + "approximation paths", + class, previous, dir) + } + } + } + return nil +} + // Resolve returns the class directories for one model path. // The path can have one of these forms: // @@ -56,11 +114,12 @@ func resolveDir(dir string, builder Builder) ([]string, error) { return []string{dir}, nil } + // os.ReadDir returns entries sorted by name, so the answer does not depend on the + // order the filesystem happens to hand them back. entries, err := os.ReadDir(dir) if err != nil { return nil, fmt.Errorf("failed to read approximation directory %s: %w", dir, err) } - sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) // Descending past classes this directory holds itself would leave them unused. if classFile, found := directClassFile(entries); found { diff --git a/cli/internal/approximation/scaffold.go b/cli/internal/approximation/scaffold.go index 357b7f5ed8..d3d9b1e09d 100644 --- a/cli/internal/approximation/scaffold.go +++ b/cli/internal/approximation/scaffold.go @@ -40,19 +40,22 @@ func ExtractApiJar(analyzerJarPath string) ([]byte, error) { entryName := apiJarResourcePrefix + apiJarName for _, file := range reader.File { - if file.Name != entryName { - continue + if file.Name == entryName { + return readZipEntry(file) } - content, err := file.Open() - if err != nil { - return nil, fmt.Errorf("failed to read %s from the analyzer jar: %w", apiJarName, err) - } - defer func() { _ = content.Close() }() - return io.ReadAll(content) } return nil, fmt.Errorf("analyzer jar %s bundles no %s", analyzerJarPath, apiJarName) } +func readZipEntry(file *zip.File) ([]byte, error) { + content, err := file.Open() + if err != nil { + return nil, fmt.Errorf("failed to read %s from the analyzer jar: %w", file.Name, err) + } + defer func() { _ = content.Close() }() + return io.ReadAll(content) +} + // WriteApiJar writes the approximation API jar to a model project. // It does not write the file when the content is current. func WriteApiJar(projectDir string, apiJar []byte) error { diff --git a/cli/internal/approximation/stamp.go b/cli/internal/approximation/stamp.go index 587a349113..98cd6323b2 100644 --- a/cli/internal/approximation/stamp.go +++ b/cli/internal/approximation/stamp.go @@ -1,7 +1,5 @@ package approximation -// Golden model: formal/Opentaint/Cli/Internal/Approximation/Stamp.lean - import ( "crypto/sha256" "encoding/hex" diff --git a/cli/internal/autobuilder/buildlog.go b/cli/internal/autobuilder/buildlog.go index 848e69498a..60140243bb 100644 --- a/cli/internal/autobuilder/buildlog.go +++ b/cli/internal/autobuilder/buildlog.go @@ -24,7 +24,13 @@ func CompilerDiagnostics(logPath string) string { var diagnostics []string for _, line := range strings.Split(string(log), "\n") { message := buildOutput(line) - if isCompilerDiagnostic(message) { + switch { + case isCompilerDiagnostic(message): + diagnostics = append(diagnostics, " "+message) + case len(diagnostics) > 0 && compilerErrorCount.MatchString(message): + // javac closes with a count of what it just reported. On its own the line is + // only something that reads like one, so it is kept as a trailer and never as + // the diagnostic. diagnostics = append(diagnostics, " "+message) } } @@ -41,7 +47,5 @@ func buildOutput(logLine string) string { func isCompilerDiagnostic(message string) bool { // Match javac and kotlinc error formats. - return strings.Contains(message, ": error: ") || - strings.HasPrefix(message, "e: ") || - compilerErrorCount.MatchString(message) + return strings.Contains(message, ": error: ") || strings.HasPrefix(message, "e: ") } diff --git a/cli/internal/autobuilder/buildlog_test.go b/cli/internal/autobuilder/buildlog_test.go index dfe4b15f8a..066e20af39 100644 --- a/cli/internal/autobuilder/buildlog_test.go +++ b/cli/internal/autobuilder/buildlog_test.go @@ -37,9 +37,17 @@ func TestCompilerDiagnostics(t *testing.T) { want: " e: file:///tmp/batch/Model.kt:7:1 unresolved reference", }, { - name: "plural error count", + name: "a count of errors follows the errors it counts", + log: `12:00:00.000 |D| ProjectResolver - Model.java:4: error: cannot find symbol +12:00:00.000 |D| ProjectResolver - Model.java:9: error: cannot find symbol +12:00:00.000 |D| ProjectResolver - 2 errors +`, + want: " Model.java:4: error: cannot find symbol\n Model.java:9: error: cannot find symbol\n 2 errors", + }, + { + name: "a line that only reads like a count is not a diagnostic", log: "12:00:00.000 |D| ProjectResolver - 3 errors\n", - want: " 3 errors", + want: "", }, { name: "raw build output with no log prefix", diff --git a/docs/usage.md b/docs/usage.md index b47036342a..eb2d776d2b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -205,7 +205,7 @@ opentaint scan --project-model ./my-project-model \ Pin the versions the target application uses. Those pins are the models' compile environment: they, and nothing about the project under analysis, decide what the models compile against, so a model compiles identically wherever it is applied. -`--dataflow-approximations` accepts an approximation project, a build output, or a directory of compiled classes — and a directory holding any of those, so a tree with one project per batch can be passed as a single flag. A directory counts as compiled classes only when nothing below it still needs building; one that holds compiled classes of its own alongside a project is reported rather than guessed at. Projects are built on demand and rebuilt when their sources, their dependency pins, or the compiler change. +`--dataflow-approximations` accepts an approximation project, a build output, or a directory of compiled classes — and a directory holding any of those, so a tree with one project per batch can be passed as a single flag. A directory counts as compiled classes only when nothing below it still needs building; one that holds compiled classes of its own alongside a project is reported rather than guessed at. Paths may overlap, and the same models may be reached twice, but two of them may not offer the same class with different content — which copy an analysis would read is not defined. Projects are built on demand and rebuilt when their sources, their dependency pins, or the compiler change. | Command | Description | |---------|-------------| From 8d546273f98f27079e5d04faaf23e0bb3e0890eb Mon Sep 17 00:00:00 2001 From: Gr-i-niy Date: Mon, 12 Oct 2026 14:16:06 +0000 Subject: [PATCH 4/4] fix(skills): trim approximation-project instructions to the essentials Take the approximation project's pins from the batch's test project, drop rationale and CLI internals that duplicate existing instructions, and keep compilation feedback in the approximation test run. --- .../references/java.md.j2 | 15 ++++----------- .../sections/output.md | 2 +- .../sections/workflow.md | 2 +- skills-templates/run-scan/sections/workflow.md | 2 +- skills/create-dataflow-approximation/SKILL.md | 4 ++-- .../references/java.md | 15 ++++----------- skills/run-scan/SKILL.md | 2 +- 7 files changed, 14 insertions(+), 28 deletions(-) diff --git a/skills-templates/create-dataflow-approximation/references/java.md.j2 b/skills-templates/create-dataflow-approximation/references/java.md.j2 index 7bb792615b..125fabdef3 100644 --- a/skills-templates/create-dataflow-approximation/references/java.md.j2 +++ b/skills-templates/create-dataflow-approximation/references/java.md.j2 @@ -10,14 +10,14 @@ An app-internal method sits in the project's own sources under ``. ### 2. Write the approximation source -The models live in the batch's own approximation project. Scaffold it once, passing each library the batch's models will reference at the exact version the batch file's `dependencies` records — the same pins the test project took. Those pins are the models' compile environment: nothing about the project under analysis affects it, so the models compile the same way at test time and at scan time. +Scaffold the approximation project with the same `--dependency` pins as the batch's test project (`.opentaint/test-projects//build.gradle.kts`): ```bash opentaint approximation init .opentaint/dataflow/ \ --dependency "io.projectreactor:reactor-core:3.8.5" ``` -Re-invoked for a batch whose project already exists, leave the build file alone unless a model needs a library that isn't pinned yet — then add it with a second `init` carrying the full dependency set, or edit `build.gradle.kts` directly. +When a model needs a library the project doesn't pin yet, add it to the project's `build.gradle.kts`. Create Java files under `.opentaint/dataflow//src/main/java` — one `@Approximate` class per target class. `@Approximate(TargetClass.class)` binds a model to exactly that class, so target the EXACT class the analyzer dropped — the dropped FQN reflects how the call resolved: an interface-typed receiver (`Map m = ...; m.computeIfAbsent(...)`) drops `java.util.Map#computeIfAbsent`, a concrete one (`new HashMap<>()`) drops `java.util.HashMap#computeIfAbsent`. An interface is a valid target — write `@Approximate` on it exactly as on a concrete class, modelling whichever FQN the analyzer dropped. Reach the real object with `(TargetClass) (Object) this`, put functional-interface parameters behind `@ArgumentTypeContext`, and branch with `OpentaintNdUtil.nextBool()` so the analyzer walks both paths. Never leave a body empty. @@ -77,13 +77,7 @@ opentaint test approximation run .opentaint/test-compiled/ \ --dataflow-approximations .opentaint/dataflow/ ``` -`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI builds the approximation project against the dependencies it pins (the approximation API — `@Approximate`, `OpentaintNdUtil`, `ArgumentTypeContext` — comes from the project's own `libs/`); if compilation fails it reports the errors and aborts before the tests. A rebuild happens only when the sources or the pins changed, so re-running an unedited batch skips straight to the tests. To see compilation errors on their own, compile the project alone: - -```bash -opentaint compile approximations .opentaint/dataflow/ -``` - -A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: +`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI builds the approximation project first. if compilation fails it reports the errors and aborts before the tests. A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: ```bash uv run /scripts/check-test-result.py @@ -98,7 +92,7 @@ Fix by the verdict it reports: | Pattern | Usage | |---|---| -| `@Approximate(TargetClass.class)` | Link the approximation to its target class. Must be on the compile classpath (a project dependency or a JDK type) | +| `@Approximate(TargetClass.class)` | Link the approximation to its target class. Must be on the compile classpath (a pinned dependency or a JDK type) | | `(TargetClass) (Object) this` | Cast to reach the real object's methods | | `@ArgumentTypeContext` | On lambda / functional-interface parameters | | `OpentaintNdUtil.nextBool()` | Non-deterministic branch — the analyzer considers both paths | @@ -106,7 +100,6 @@ Fix by the verdict it reports: ## Constraints - Java 8 source compatibility -- A model may only reference libraries its own project pins. A model that needs a type from an unpinned library is a missing `--dependency`, not a reason to weaken the model to `Object` - Put the `@Approximate` classes in a neutral package (e.g. `com.example.approximations`) — never the target library's own package. Inside the library's package every bare FQN resolves to your approximation's non-generic class instead of the real type, breaking compilation wholesale - Namespace the package PER BATCH (`com.example.approximations.`, e.g. `...approximations.cn_hutool_001`). Every batch is loaded together into one shared index, and approximation class FQCNs must be GLOBALLY unique across all batches — two batches that both model a `…$Nested` (or any recurring simple name) in the bare `com.example.approximations` package collide on `com.example.approximations.Nested` and crash the whole scan at config-load. A per-batch package guarantees uniqueness. - One approximation class per target class — a strict global bijection enforced at load: each approximation FQCN maps to exactly one target and vice-versa; duplicates or a reused FQCN across batches throw `IllegalArgumentException` diff --git a/skills-templates/create-dataflow-approximation/sections/output.md b/skills-templates/create-dataflow-approximation/sections/output.md index 76c5e21f02..eec392b57c 100644 --- a/skills-templates/create-dataflow-approximation/sections/output.md +++ b/skills-templates/create-dataflow-approximation/sections/output.md @@ -1,6 +1,6 @@ ### Artifacts -- `.opentaint/dataflow/` — the batch's approximation project: its pinned dependencies plus the language-specific code approximation artifacts that the scan consumes, following the language reference's artifact layout; report the path and the exact test command used +- `.opentaint/dataflow/` — the batch's approximation project with the language-specific code approximation artifacts that the scan consumes; report the path and the exact test command used - the passing methods present in the batch file's `build.done` (new methods appended; repaired methods already recorded, per Tracking) ### Summary diff --git a/skills-templates/create-dataflow-approximation/sections/workflow.md b/skills-templates/create-dataflow-approximation/sections/workflow.md index 58c6429cdd..1b642a03e3 100644 --- a/skills-templates/create-dataflow-approximation/sections/workflow.md +++ b/skills-templates/create-dataflow-approximation/sections/workflow.md @@ -4,7 +4,7 @@ Find and read each dataflow method's real source: take methods not yet in `build ### 2. Write the approximation -Reproduce that propagation in the language-specific code form in the batch's approximation project at `.opentaint/dataflow/`, following the language reference's artifact layout. The project pins the libraries the models are written against at the same versions the test project takes, because a model references the library type it models. Scaffold it if it doesn't exist yet (the language reference has the command). Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. +Reproduce that propagation in the language-specific code form in the batch's approximation project at `.opentaint/dataflow/` — scaffold it if it doesn't exist yet — following the language reference's artifact layout. Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. ### 3. Test against the test project diff --git a/skills-templates/run-scan/sections/workflow.md b/skills-templates/run-scan/sections/workflow.md index 2e938782a6..d351480681 100644 --- a/skills-templates/run-scan/sections/workflow.md +++ b/skills-templates/run-scan/sections/workflow.md @@ -11,7 +11,7 @@ opentaint scan --project-model .opentaint/project \ - `--rule-id ` — restrict to specific executable rules (repeatable, one per input rule ID); referenced library rules remain available without listing their IDs separately. Omit to run all loaded rules - `--passthrough-approximations .opentaint/pass-through` — add when that directory exists: passThrough configs override built-ins at the rule level, a provided rule overriding a built-in only when it matches one -- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations, one project per batch (each built on demand against the dependencies it pins, and rebuilt only when its sources change) +- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations, one project per batch Both approximation-dir flags walk their trees recursively; pass each parent directory once, not every package or batch separately. diff --git a/skills/create-dataflow-approximation/SKILL.md b/skills/create-dataflow-approximation/SKILL.md index ab3e1c2868..c09015791d 100644 --- a/skills/create-dataflow-approximation/SKILL.md +++ b/skills/create-dataflow-approximation/SKILL.md @@ -28,7 +28,7 @@ Find and read each dataflow method's real source: take methods not yet in `build ### 2. Write the approximation -Reproduce that propagation in the language-specific code form in the batch's approximation project at `.opentaint/dataflow/`, following the language reference's artifact layout. The project pins the libraries the models are written against at the same versions the test project takes, because a model references the library type it models. Scaffold it if it doesn't exist yet (the language reference has the command). Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. +Reproduce that propagation in the language-specific code form in the batch's approximation project at `.opentaint/dataflow/` — scaffold it if it doesn't exist yet — following the language reference's artifact layout. Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. ### 3. Test against the test project @@ -42,7 +42,7 @@ When the sample won't converge after ~3 fixes — whether the trace shows a fait ### Artifacts -- `.opentaint/dataflow/` — the batch's approximation project: its pinned dependencies plus the language-specific code approximation artifacts that the scan consumes, following the language reference's artifact layout; report the path and the exact test command used +- `.opentaint/dataflow/` — the batch's approximation project with the language-specific code approximation artifacts that the scan consumes; report the path and the exact test command used - the passing methods present in the batch file's `build.done` (new methods appended; repaired methods already recorded, per Tracking) ### Summary diff --git a/skills/create-dataflow-approximation/references/java.md b/skills/create-dataflow-approximation/references/java.md index 7bb792615b..125fabdef3 100644 --- a/skills/create-dataflow-approximation/references/java.md +++ b/skills/create-dataflow-approximation/references/java.md @@ -10,14 +10,14 @@ An app-internal method sits in the project's own sources under ``. ### 2. Write the approximation source -The models live in the batch's own approximation project. Scaffold it once, passing each library the batch's models will reference at the exact version the batch file's `dependencies` records — the same pins the test project took. Those pins are the models' compile environment: nothing about the project under analysis affects it, so the models compile the same way at test time and at scan time. +Scaffold the approximation project with the same `--dependency` pins as the batch's test project (`.opentaint/test-projects//build.gradle.kts`): ```bash opentaint approximation init .opentaint/dataflow/ \ --dependency "io.projectreactor:reactor-core:3.8.5" ``` -Re-invoked for a batch whose project already exists, leave the build file alone unless a model needs a library that isn't pinned yet — then add it with a second `init` carrying the full dependency set, or edit `build.gradle.kts` directly. +When a model needs a library the project doesn't pin yet, add it to the project's `build.gradle.kts`. Create Java files under `.opentaint/dataflow//src/main/java` — one `@Approximate` class per target class. `@Approximate(TargetClass.class)` binds a model to exactly that class, so target the EXACT class the analyzer dropped — the dropped FQN reflects how the call resolved: an interface-typed receiver (`Map m = ...; m.computeIfAbsent(...)`) drops `java.util.Map#computeIfAbsent`, a concrete one (`new HashMap<>()`) drops `java.util.HashMap#computeIfAbsent`. An interface is a valid target — write `@Approximate` on it exactly as on a concrete class, modelling whichever FQN the analyzer dropped. Reach the real object with `(TargetClass) (Object) this`, put functional-interface parameters behind `@ArgumentTypeContext`, and branch with `OpentaintNdUtil.nextBool()` so the analyzer walks both paths. Never leave a body empty. @@ -77,13 +77,7 @@ opentaint test approximation run .opentaint/test-compiled/ \ --dataflow-approximations .opentaint/dataflow/ ``` -`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI builds the approximation project against the dependencies it pins (the approximation API — `@Approximate`, `OpentaintNdUtil`, `ArgumentTypeContext` — comes from the project's own `libs/`); if compilation fails it reports the errors and aborts before the tests. A rebuild happens only when the sources or the pins changed, so re-running an unedited batch skips straight to the tests. To see compilation errors on their own, compile the project alone: - -```bash -opentaint compile approximations .opentaint/dataflow/ -``` - -A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: +`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI builds the approximation project first. if compilation fails it reports the errors and aborts before the tests. A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: ```bash uv run /scripts/check-test-result.py @@ -98,7 +92,7 @@ Fix by the verdict it reports: | Pattern | Usage | |---|---| -| `@Approximate(TargetClass.class)` | Link the approximation to its target class. Must be on the compile classpath (a project dependency or a JDK type) | +| `@Approximate(TargetClass.class)` | Link the approximation to its target class. Must be on the compile classpath (a pinned dependency or a JDK type) | | `(TargetClass) (Object) this` | Cast to reach the real object's methods | | `@ArgumentTypeContext` | On lambda / functional-interface parameters | | `OpentaintNdUtil.nextBool()` | Non-deterministic branch — the analyzer considers both paths | @@ -106,7 +100,6 @@ Fix by the verdict it reports: ## Constraints - Java 8 source compatibility -- A model may only reference libraries its own project pins. A model that needs a type from an unpinned library is a missing `--dependency`, not a reason to weaken the model to `Object` - Put the `@Approximate` classes in a neutral package (e.g. `com.example.approximations`) — never the target library's own package. Inside the library's package every bare FQN resolves to your approximation's non-generic class instead of the real type, breaking compilation wholesale - Namespace the package PER BATCH (`com.example.approximations.`, e.g. `...approximations.cn_hutool_001`). Every batch is loaded together into one shared index, and approximation class FQCNs must be GLOBALLY unique across all batches — two batches that both model a `…$Nested` (or any recurring simple name) in the bare `com.example.approximations` package collide on `com.example.approximations.Nested` and crash the whole scan at config-load. A per-batch package guarantees uniqueness. - One approximation class per target class — a strict global bijection enforced at load: each approximation FQCN maps to exactly one target and vice-versa; duplicates or a reused FQCN across batches throw `IllegalArgumentException` diff --git a/skills/run-scan/SKILL.md b/skills/run-scan/SKILL.md index 65cc90f70d..221f4a67df 100644 --- a/skills/run-scan/SKILL.md +++ b/skills/run-scan/SKILL.md @@ -34,7 +34,7 @@ opentaint scan --project-model .opentaint/project \ - `--rule-id ` — restrict to specific executable rules (repeatable, one per input rule ID); referenced library rules remain available without listing their IDs separately. Omit to run all loaded rules - `--passthrough-approximations .opentaint/pass-through` — add when that directory exists: passThrough configs override built-ins at the rule level, a provided rule overriding a built-in only when it matches one -- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations, one project per batch (each built on demand against the dependencies it pins, and rebuilt only when its sources change) +- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations, one project per batch Both approximation-dir flags walk their trees recursively; pass each parent directory once, not every package or batch separately.