diff --git a/cli/cmd/analyzer_inputs.go b/cli/cmd/analyzer_inputs.go index 3290a995fa..d9fabfe8b4 100644 --- a/cli/cmd/analyzer_inputs.go +++ b/cli/cmd/analyzer_inputs.go @@ -1,17 +1,28 @@ package cmd import ( + "github.com/seqra/opentaint/internal/approximation" "github.com/seqra/opentaint/internal/utils/log" ) -func addDataflowApproximations(b *AnalyzerBuilder, paths []string, analyzerJarPath, projectModelDir string) { +// addDataflowApproximations resolves the paths to compiled model directories. +// It rebuilds a model project when its inputs change. +func addDataflowApproximations(b *AnalyzerBuilder, paths []string, analyzerJarPath string) { + if len(paths) == 0 { + return + } + + absPaths := make([]string, 0, len(paths)) for _, approxPath := range paths { - absApproxPath := log.AbsPathOrExit(approxPath, "dataflow-approximations") - compiledPath, err := compileApproximationsIfNeeded(absApproxPath, analyzerJarPath, projectModelDir) - if err != nil { - out.Fatalf("Approximation compilation failed: %s", err) - } - b.AddDataflowApproximations(compiledPath) + absPaths = append(absPaths, log.AbsPathOrExit(approxPath, "dataflow-approximations")) + } + + classDirs, err := approximation.ResolveAll(absPaths, newApproximationBuilder(analyzerJarPath)) + if err != nil { + out.Fatalf("Failed to apply dataflow approximations: %s", err) + } + for _, classDir := range classDirs { + b.AddDataflowApproximations(classDir) } } diff --git a/cli/cmd/approximation.go b/cli/cmd/approximation.go new file mode 100644 index 0000000000..61d5050c88 --- /dev/null +++ b/cli/cmd/approximation.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "fmt" + + "github.com/seqra/opentaint/internal/approximation" + "github.com/seqra/opentaint/internal/utils/log" + "github.com/spf13/cobra" +) + +var initApproximationDeps []string + +var approximationCmd = &cobra.Command{ + Use: "approximation", + Short: "Create dataflow approximation projects", + Long: `A dataflow approximation project holds code-based models of how taint moves through +methods the analyzer cannot see into, together with the exact dependency versions those +models are written against. + +Compile one with 'opentaint compile approximations', and apply it with the +--dataflow-approximations flag of 'opentaint scan' or 'opentaint test approximation run'.`, +} + +var approximationInitCmd = &cobra.Command{ + Use: "init ", + Short: "Create a dataflow approximation project", + Long: `Create a Gradle project for OpenTaint dataflow approximation models. + +The project includes: + - build.gradle.kts with compile-only dependencies, settings.gradle.kts + - libs/opentaint-approximations-api.jar, the @Approximate annotations and the + OpentaintNdUtil / ArgumentTypeContext support types + - src/main/java/ for the model sources, one @Approximate class per target class + +Use --dependency to pin the libraries the models are written against, at the exact versions +the target application uses. Those pins are the models' compile environment: they, and +nothing about the project under analysis, decide what the models compile against.`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + projectPath := log.AbsPathOrExit(args[0], "output-dir") + + analyzerJarPath, err := ensureAnalyzerAvailable() + if err != nil { + out.Fatalf("Failed to resolve analyzer: %s", err) + } + + // Init replaces the build file but keeps the model sources. + // Warn the user because this removes dependency pins that they added manually. + existing := approximation.IsProject(projectPath) + if existing { + out.Warnf("Replacing the build file of the existing approximation project at %s; "+ + "pass every dependency the models need, not only the new ones", projectPath) + } + + if err := approximation.Scaffold(projectPath, initApproximationDeps, analyzerJarPath); err != nil { + out.Fatalf("Failed to create approximation project: %s", err) + } + + if existing { + fmt.Printf("Approximation project updated at %s\n", projectPath) + } else { + fmt.Printf("Approximation project initialized at %s\n", projectPath) + } + suggest("To compile the models run", fmt.Sprintf("opentaint compile approximations %s", args[0])) + }, +} + +func init() { + rootCmd.AddCommand(approximationCmd) + approximationCmd.AddCommand(approximationInitCmd) + + approximationInitCmd.Flags().StringArrayVar(&initApproximationDeps, "dependency", nil, + "Compile-only Maven dependency coordinates the models are written against (repeatable)") +} diff --git a/cli/cmd/command_builder.go b/cli/cmd/command_builder.go index 1f38f2ca69..263c45e58c 100644 --- a/cli/cmd/command_builder.go +++ b/cli/cmd/command_builder.go @@ -12,9 +12,13 @@ type BaseCommandBuilder struct { debug bool } -// appendVerbosityFlag passes the JAR's own --verbosity flag based on opentaint's -// debug bool. The Java tool's CLI surface is independent of opentaint's; we just -// translate. info ↔ false, debug ↔ true. +// ForceDebugVerbosity sets the tool log level to debug. +// The CLI can then read detailed output from the log file. +func (b *BaseCommandBuilder) ForceDebugVerbosity() { + b.debug = true +} + +// appendVerbosityFlag maps the CLI debug setting to the tool log level. func (b *BaseCommandBuilder) appendVerbosityFlag(flags []string) []string { if b.debug { return append(flags, "--verbosity=debug") diff --git a/cli/cmd/compile.go b/cli/cmd/compile.go index 3d33c8d337..3d04172990 100644 --- a/cli/cmd/compile.go +++ b/cli/cmd/compile.go @@ -143,12 +143,18 @@ func printCompileSummary(absOutputProjectModelPath string) { } func compileProject(absOutputProjectModelPath, absProjectRoot, autobuilderJarPath string, javaRunner java.JavaRunner) error { - var err error tempLogsDir, err := os.MkdirTemp("", "opentaint-*") if err != nil { return fmt.Errorf("failed to create temporary directory: %w", err) } - tempLogsFile := filepath.Join(tempLogsDir, "autobuild.log") + return compileProjectWithLogs(absOutputProjectModelPath, absProjectRoot, autobuilderJarPath, + filepath.Join(tempLogsDir, "autobuild.log"), false, javaRunner) +} + +// compileProjectWithLogs writes the autobuilder log to logsFile. +// captureBuildOutput adds the output from the build tool to this log. +func compileProjectWithLogs(absOutputProjectModelPath, absProjectRoot, autobuilderJarPath, tempLogsFile string, captureBuildOutput bool, javaRunner java.JavaRunner) error { + var err error builder := NewAutobuilderBuilder(). SetProjectRootDir(absProjectRoot). @@ -157,6 +163,10 @@ func compileProject(absOutputProjectModelPath, absProjectRoot, autobuilderJarPat SetLogsFile(tempLogsFile). SetJarPath(autobuilderJarPath) + if captureBuildOutput { + builder.ForceDebugVerbosity() + } + autobuilderCommand := builder.BuildNativeCommand() commandSucceeded := func(_ error) bool { diff --git a/cli/cmd/compile_approximations.go b/cli/cmd/compile_approximations.go index ed61db7b1f..52b08ba73f 100644 --- a/cli/cmd/compile_approximations.go +++ b/cli/cmd/compile_approximations.go @@ -3,168 +3,87 @@ package cmd import ( "fmt" "os" - "os/exec" "path/filepath" - "strings" + "github.com/seqra/opentaint/internal/approximation" + "github.com/seqra/opentaint/internal/autobuilder" "github.com/seqra/opentaint/internal/globals" "github.com/seqra/opentaint/internal/output" - "github.com/seqra/opentaint/internal/utils" - "github.com/seqra/opentaint/internal/utils/java" - "github.com/seqra/opentaint/internal/utils/project" ) -// approxClassesJarPrefix is the path prefix under which the analyzer fat JAR -// bundles approximation support sources (OpentaintNdUtil, ArgumentTypeContext). -const approxClassesJarPrefix = "opentaint-dataflow-approximations/" - -// compileApproximationsIfNeeded checks whether a --dataflow-approximations directory -// contains .java source files. If so, it compiles them using javac (with the -// analyzer JAR + project dependencies on the classpath) and returns the path to -// the compiled .class output directory. If the directory already contains only -// .class files (or no .java files at all), it is returned as-is. -// -// projectModelDir is the directory containing project.yaml — used to resolve -// project dependencies for the javac classpath (approximation code may reference -// library types like org.apache.pdfbox.pdmodel.PDDocument). -func compileApproximationsIfNeeded(approxPath string, analyzerJarPath string, projectModelDir string) (string, error) { - info, err := os.Stat(approxPath) - if err != nil { - return "", fmt.Errorf("approximation path does not exist: %w", err) - } - if !info.IsDir() { - return approxPath, nil - } - - javaFiles, err := collectJavaSources(approxPath) - if err != nil { - return "", err - } - if len(javaFiles) == 0 { - return approxPath, nil - } - - output.LogInfof("Found %d .java file(s) in approximations directory, compiling...", len(javaFiles)) - - javacPath, err := resolveJavacPath() - if err != nil { - return "", err - } +// approximationBuilder compiles a model project with the autobuilder. +// The model project supplies the compile dependencies. +type approximationBuilder struct { + analyzerJarPath string + apiJar []byte +} - extractedDir, err := os.MkdirTemp("", "opentaint-approx-deps-*") - if err != nil { - return "", fmt.Errorf("failed to create temp directory for approximation deps: %w", err) - } - defer func() { _ = os.RemoveAll(extractedDir) }() +func newApproximationBuilder(analyzerJarPath string) approximation.Builder { + return &approximationBuilder{analyzerJarPath: analyzerJarPath} +} - if err := utils.ExtractZipPrefix(analyzerJarPath, approxClassesJarPrefix, extractedDir); err != nil { - return "", fmt.Errorf("failed to extract approximation classes from analyzer JAR: %w", err) +// Fingerprint identifies the compiler a build was produced with. +// The source stamp covers the model project but not the tool that reads it, so without +// this an upgraded CLI keeps serving models compiled by the previous one. +func (b *approximationBuilder) Fingerprint() (string, error) { + autobuilder := globals.ArtifactByKind("autobuilder") + if autobuilder.Override == "" { + return "autobuilder " + autobuilder.Version, nil } - outputDir, err := os.MkdirTemp("", "opentaint-approx-compiled-*") + // A local autobuilder has no version to compare, so compare the file itself. + info, err := os.Stat(autobuilder.Override) if err != nil { - return "", fmt.Errorf("failed to create temp directory for compiled approximations: %w", err) - } - - classpath := buildApproxClasspath(analyzerJarPath, extractedDir, projectModelDir) - if err := runJavac(javacPath, classpath, outputDir, javaFiles); err != nil { - _ = os.RemoveAll(outputDir) - return "", err + return "", fmt.Errorf("failed to read the autobuilder at %s: %w", autobuilder.Override, err) } - - output.LogInfof("Approximation compilation succeeded, output: %s", outputDir) - return outputDir, nil + return fmt.Sprintf("autobuilder %s %d %d", + autobuilder.Override, info.Size(), info.ModTime().UnixNano()), nil } -func collectJavaSources(root string) ([]string, error) { - var javaFiles []string - err := filepath.Walk(root, func(path string, fi os.FileInfo, walkErr error) error { - if walkErr != nil { - return walkErr - } - if !fi.IsDir() && strings.HasSuffix(fi.Name(), ".java") { - javaFiles = append(javaFiles, path) +// Prepare updates the approximation API jar in the model project. +// The builder reads the jar from the analyzer only once. +func (b *approximationBuilder) Prepare(projectDir string) error { + if b.apiJar == nil { + apiJar, err := approximation.ExtractApiJar(b.analyzerJarPath) + if err != nil { + return err } - return nil - }) - if err != nil { - return nil, fmt.Errorf("failed to walk approximations directory: %w", err) + b.apiJar = apiJar } - return javaFiles, nil + return approximation.WriteApiJar(projectDir, b.apiJar) } -func resolveJavacPath() (string, error) { - javacRunner := java.NewJavaRunner(). - WithSkipVerify(globals.Config.SkipVerify). - WithImageType(java.AdoptiumImageJDK). - TrySystem(). - TrySpecificVersion(globals.DefaultJavaVersion) - - javaPath, err := javacRunner.EnsureJava() +func (b *approximationBuilder) Compile(projectDir, projectModelDir string) error { + autobuilderJarPath, err := ensureAutobuilderAvailable() if err != nil { - return "", fmt.Errorf("failed to resolve Java for approximation compilation: %w", err) + return fmt.Errorf("failed to resolve autobuilder: %w", err) } - javacPath := java.DeriveJavacPath(javaPath) - if _, err := os.Stat(javacPath); err != nil { - return "", fmt.Errorf("javac not found at %s (resolved from java at %s). A JDK (not JRE) is required to compile approximation sources", javacPath, javaPath) - } - return javacPath, nil -} - -// buildApproxClasspath assembles the javac classpath for approximation compilation: -// 1. Analyzer JAR — contains @Approximate, @ApproximateByName annotations -// 2. Extracted approximation utilities — OpentaintNdUtil, ArgumentTypeContext -// 3. Project dependencies — library JARs that approximation code may reference -func buildApproxClasspath(analyzerJarPath, extractedDir, projectModelDir string) string { - parts := []string{analyzerJarPath, extractedDir} - parts = append(parts, resolveProjectDependencies(projectModelDir)...) - return strings.Join(parts, string(os.PathListSeparator)) -} - -func runJavac(javacPath, classpath, outputDir string, javaFiles []string) error { - args := []string{ - "-source", "8", - "-target", "8", - "-cp", classpath, - "-d", outputDir, + javaRunner := newAutobuilderJavaRunner() + if _, err := javaRunner.EnsureJava(); err != nil { + return fmt.Errorf("failed to resolve Java for approximation compilation: %w", err) } - args = append(args, javaFiles...) - output.LogDebugf("Running javac: %s %s", javacPath, strings.Join(args, " ")) - - cmd := exec.Command(javacPath, args...) - cmdOutput, cmdErr := cmd.CombinedOutput() - if cmdErr != nil { - return fmt.Errorf( - "approximation compilation failed:\n%s\njavac exited with: %w", - string(cmdOutput), cmdErr, - ) + logsDir, err := os.MkdirTemp("", "opentaint-approximation-logs-*") + if err != nil { + return fmt.Errorf("failed to create temp directory for approximation build logs: %w", err) } - return nil -} + logsFile := filepath.Join(logsDir, "approximation-build.log") -// resolveProjectDependencies reads project.yaml from the project model directory -// and returns absolute paths to the dependency JARs listed there. -func resolveProjectDependencies(projectModelDir string) []string { - if projectModelDir == "" { - return nil + // Show the spinner for direct and automatic builds. + output.LogInfof("Compiling approximation project %s", projectDir) + compile := func() error { + return compileProjectWithLogs(projectModelDir, projectDir, autobuilderJarPath, logsFile, true, javaRunner) } - config, err := project.LoadConfig(projectModelDir) - if err != nil { - output.LogDebugf("Could not read project config for approximation compilation: %v", err) - return nil - } - var absDeps []string - for _, dep := range config.AllDependencies() { - absPath := dep - if !filepath.IsAbs(dep) { - absPath = filepath.Join(projectModelDir, dep) - } - if _, err := os.Stat(absPath); err == nil { - absDeps = append(absDeps, absPath) + if err := out.RunWithSpinner("Compiling approximation project "+filepath.Base(projectDir), compile); err != nil { + // Show compiler errors when they are available. + // Keep the full log if no compiler error is available. + if diagnostics := autobuilder.CompilerDiagnostics(logsFile); diagnostics != "" { + return fmt.Errorf("approximation project %s failed to compile:\n%s", projectDir, diagnostics) } + return fmt.Errorf("failed to compile approximation project %s (build log: %s): %w", projectDir, logsFile, err) } - output.LogDebugf("Resolved %d project dependencies for approximation classpath", len(absDeps)) - return absDeps + + _ = os.RemoveAll(logsDir) + return nil } diff --git a/cli/cmd/compile_approximations_cmd.go b/cli/cmd/compile_approximations_cmd.go new file mode 100644 index 0000000000..d9ab54bcaf --- /dev/null +++ b/cli/cmd/compile_approximations_cmd.go @@ -0,0 +1,65 @@ +package cmd + +import ( + "fmt" + + "github.com/seqra/opentaint/internal/approximation" + "github.com/seqra/opentaint/internal/utils/log" + "github.com/spf13/cobra" +) + +var compileApproximationsOutput string + +var compileApproximationsCmd = &cobra.Command{ + Use: "approximations ", + Short: "Compile a dataflow approximation project", + Long: `Compile the dataflow approximation models in against the +dependencies that project pins in its own build file. + +This is the model project, not the project to scan: its build.gradle.kts lists the exact +library versions the models are written against, so the same models compile identically +wherever they are applied. + +Scanning and testing build an approximation project on demand, so running this is only +needed to compile ahead of time or to see compilation errors on their own.`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + projectPath := log.AbsPathOrExit(args[0], "approximation-project") + + if !approximation.IsProject(projectPath) { + out.Fatalf("%s is not an approximation project: it has no build.gradle.kts declaring the "+ + "dependencies its models are written against.\nCreate one with 'opentaint approximation init %s "+ + "--dependency '", projectPath, args[0]) + } + + outputPath := compileApproximationsOutput + if outputPath == "" { + outputPath = approximation.BuildDir(projectPath) + } else { + outputPath = log.AbsPathOrExit(outputPath, "output") + } + + analyzerJarPath, err := ensureAnalyzerAvailable() + if err != nil { + out.Fatalf("Failed to resolve analyzer: %s", err) + } + + if err := approximation.Build(projectPath, outputPath, newApproximationBuilder(analyzerJarPath)); err != nil { + out.Fatalf("Approximation compilation failed: %s", err) + } + + out.Section("Approximation Compile Summary"). + Field("Approximation project", projectPath). + Field("Compiled models written to", approximation.ClassesDir(outputPath)). + Render() + suggest("To apply these models in a scan run", + fmt.Sprintf("opentaint scan --dataflow-approximations %s", args[0])) + }, +} + +func init() { + compileCmd.AddCommand(compileApproximationsCmd) + + compileApproximationsCmd.Flags().StringVarP(&compileApproximationsOutput, "output", "o", "", + "Path to the compiled models (default: /.opentaint/build)") +} diff --git a/cli/cmd/scan.go b/cli/cmd/scan.go index 8ef75c3739..30f51ed2fa 100644 --- a/cli/cmd/scan.go +++ b/cli/cmd/scan.go @@ -144,7 +144,7 @@ func addScanFlags(cmd *cobra.Command) { cmd.Flags().StringArrayVar(&scanFlags.PassthroughApproximations, "passthrough-approximations", nil, "Pass-through approximation YAML file or directory (repeatable)") - cmd.Flags().StringArrayVar(&scanFlags.DataflowApproximations, "dataflow-approximations", nil, "Dataflow approximation class directory or Java source directory (repeatable)") + cmd.Flags().StringArrayVar(&scanFlags.DataflowApproximations, "dataflow-approximations", nil, "Dataflow approximation project, build output, or class directory (repeatable)") cmd.Flags().BoolVar(&scanFlags.TrackExternalMethods, "track-external-methods", false, "Write external-method coverage files next to the SARIF report") @@ -408,8 +408,7 @@ func runScan(cmd *cobra.Command, cfg ScanConfig) { } nativeBuilder.SetJarPath(analyzerJarPath) - // Process --dataflow-approximations: auto-compile .java sources if needed - addDataflowApproximations(nativeBuilder, cfg.DataflowApproximations, analyzerJarPath, absProjectModelPath) + addDataflowApproximations(nativeBuilder, cfg.DataflowApproximations, analyzerJarPath) analyzerJavaRunner := newAnalyzerJavaRunner() if _, err := analyzerJavaRunner.EnsureJava(); err != nil { diff --git a/cli/cmd/test.go b/cli/cmd/test.go index 409240606c..a31a3cc73b 100644 --- a/cli/cmd/test.go +++ b/cli/cmd/test.go @@ -43,5 +43,5 @@ func addTestRunFlags(cmd *cobra.Command, outputDir *string, timeout *time.Durati cmd.Flags().StringVarP(outputDir, "output", "o", "", "Directory for test-result.json and test-results.sarif") cmd.Flags().DurationVar(timeout, "timeout", 600*time.Second, "Analysis timeout") cmd.Flags().StringVar(maxMemory, "max-memory", "8G", "Maximum analyzer heap size (e.g., 8G)") - cmd.Flags().StringArrayVar(dataflow, "dataflow-approximations", nil, "Dataflow approximation class directory or Java source directory (repeatable)") + cmd.Flags().StringArrayVar(dataflow, "dataflow-approximations", nil, "Dataflow approximation project, build output, or class directory (repeatable)") } diff --git a/cli/cmd/test_rule_run.go b/cli/cmd/test_rule_run.go index 4b380c617b..6c7261e0fa 100644 --- a/cli/cmd/test_rule_run.go +++ b/cli/cmd/test_rule_run.go @@ -128,7 +128,7 @@ func runTestProject(projectModelArg string, opts testProjectOptions) { } builder.SetJarPath(analyzerJarPath) - addDataflowApproximations(builder, opts.dataflowApprox, analyzerJarPath, projectPath) + addDataflowApproximations(builder, opts.dataflowApprox, analyzerJarPath) addPassthroughApproximations(builder, opts.passthroughApprox) javaRunner := newAnalyzerJavaRunner() diff --git a/cli/internal/approximation/approximation_test.go b/cli/internal/approximation/approximation_test.go new file mode 100644 index 0000000000..619f83333d --- /dev/null +++ b/cli/internal/approximation/approximation_test.go @@ -0,0 +1,558 @@ +package approximation + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v2" +) + +// fakeBuilder records builds and writes a test project model. +type fakeBuilder struct { + t *testing.T + compiled []string + fingerprint string +} + +func newFakeBuilder(t *testing.T) *fakeBuilder { + return &fakeBuilder{t: t, fingerprint: "autobuilder 1.0.0"} +} + +func (b *fakeBuilder) Fingerprint() (string, error) { + return b.fingerprint, nil +} + +// Prepare writes the same API content for each call. +func (b *fakeBuilder) Prepare(projectDir string) error { + writeFile(b.t, apiJarPath(projectDir), "api-jar-bytes") + return nil +} + +func (b *fakeBuilder) Compile(projectDir, projectModelDir string) error { + b.compiled = append(b.compiled, projectDir) + writeModuleClasses(b.t, projectModelDir, map[string][]string{"c0_main": {"Model"}}) + return nil +} + +// writeModuleClasses writes classes and a project descriptor for tests. +func writeModuleClasses(t *testing.T, projectModelDir string, modules map[string][]string) { + t.Helper() + moduleDirs := make([]string, 0, len(modules)) + for module, classes := range modules { + for _, class := range classes { + writeFile(t, filepath.Join(projectModelDir, "classes", module, "com", "example", class+".class"), "bytecode") + } + moduleDirs = append(moduleDirs, module) + } + sort.Strings(moduleDirs) + + config := "javaProjects:\n - sourceRoot: /project\n modules:\n - moduleSourceRoot: /project\n moduleClasses:\n" + for _, module := range moduleDirs { + config += " - classes/" + module + "\n" + } + writeFile(t, filepath.Join(projectModelDir, "project.yaml"), config) +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// newProject writes a model project for tests. +func newProject(t *testing.T, dir string) string { + t.Helper() + writeFile(t, filepath.Join(dir, "build.gradle.kts"), "dependencies {\n compileOnly(\"com.foo:bar:1.0\")\n}\n") + writeFile(t, filepath.Join(dir, "src", "main", "java", "com", "example", "Model.java"), "package com.example;\n") + return dir +} + +func TestResolveBuildsProject(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := newFakeBuilder(t) + + classDirs, err := Resolve(project, builder) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + + want := ClassesDir(BuildDir(project)) + if len(classDirs) != 1 || classDirs[0] != want { + t.Fatalf("Resolve() = %v, want [%s]", classDirs, want) + } + if len(builder.compiled) != 1 { + t.Fatalf("expected one build, got %v", builder.compiled) + } + if _, err := os.Stat(filepath.Join(want, "com", "example", "Model.class")); err != nil { + t.Errorf("compiled model not collected: %v", err) + } + if _, err := os.Stat(filepath.Join(BuildDir(project), descriptorName)); err != nil { + t.Errorf("descriptor not written: %v", err) + } +} + +func TestResolveReusesBuildUntilSourcesChange(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := newFakeBuilder(t) + + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("first Resolve: %v", err) + } + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("second Resolve: %v", err) + } + if len(builder.compiled) != 1 { + t.Fatalf("unchanged sources rebuilt: %v", builder.compiled) + } + + writeFile(t, filepath.Join(project, "src", "main", "java", "com", "example", "Model.java"), "package com.example;\n// edited\n") + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("third Resolve: %v", err) + } + if len(builder.compiled) != 2 { + t.Fatalf("edited model not rebuilt: %v", builder.compiled) + } +} + +// The stamp covers the project, and the project alone does not determine the compiled +// classes: the compiler does too. +func TestResolveRebuildsWhenTheCompilerChanges(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := newFakeBuilder(t) + + for range 2 { + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("Resolve: %v", err) + } + } + if len(builder.compiled) != 1 { + t.Fatalf("an unchanged project was rebuilt: %v", builder.compiled) + } + + builder.fingerprint = "autobuilder 2.0.0" + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("Resolve after the upgrade: %v", err) + } + if len(builder.compiled) != 2 { + t.Errorf("an upgraded compiler did not rebuild the models: %v", builder.compiled) + } +} + +func TestResolveRebuildsWhenDependencyPinChanges(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := newFakeBuilder(t) + + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("first Resolve: %v", err) + } + writeFile(t, filepath.Join(project, "build.gradle.kts"), "dependencies {\n compileOnly(\"com.foo:bar:2.0\")\n}\n") + if _, err := Resolve(project, builder); err != nil { + t.Fatalf("second Resolve: %v", err) + } + if len(builder.compiled) != 2 { + t.Fatalf("changed dependency pin not rebuilt: %v", builder.compiled) + } +} + +func TestResolveBuiltOutput(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, descriptorName), "sourceProject: \"/somewhere\"\n") + writeFile(t, filepath.Join(dir, classesDirName, "com", "example", "Model.class"), "bytecode") + + builder := newFakeBuilder(t) + classDirs, err := Resolve(dir, builder) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if len(classDirs) != 1 || classDirs[0] != ClassesDir(dir) { + t.Fatalf("Resolve() = %v, want [%s]", classDirs, ClassesDir(dir)) + } + if len(builder.compiled) != 0 { + t.Errorf("built output should not be rebuilt, got %v", builder.compiled) + } +} + +func TestResolvePrecompiledClassDirectory(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "com", "example", "Model.class"), "bytecode") + + classDirs, err := Resolve(dir, newFakeBuilder(t)) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if len(classDirs) != 1 || classDirs[0] != dir { + t.Fatalf("Resolve() = %v, want [%s]", classDirs, dir) + } +} + +func TestResolveTreeOfProjects(t *testing.T) { + root := t.TempDir() + newProject(t, filepath.Join(root, "batch-a")) + newProject(t, filepath.Join(root, "batch-b")) + + builder := newFakeBuilder(t) + classDirs, err := Resolve(root, builder) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if len(classDirs) != 2 { + t.Fatalf("Resolve() = %v, want both batches", classDirs) + } + if len(builder.compiled) != 2 { + t.Fatalf("expected both projects built, got %v", builder.compiled) + } + + // The second call must ignore the nested build outputs. + again, err := Resolve(root, builder) + if err != nil { + t.Fatalf("second Resolve: %v", err) + } + if len(again) != 2 { + t.Fatalf("second Resolve() = %v, want both batches", again) + } + if len(builder.compiled) != 2 { + t.Fatalf("unchanged projects rebuilt: %v", builder.compiled) + } +} + +// A tree may mix model projects with directories that are already compiled. Resolving +// such a tree must build the project and keep the compiled classes, not collapse the +// whole tree into one classpath root. +func TestResolveTreeMixingProjectAndClasses(t *testing.T) { + root := t.TempDir() + project := newProject(t, filepath.Join(root, "models")) + precompiled := filepath.Join(root, "precompiled") + writeFile(t, filepath.Join(precompiled, "com", "example", "Other.class"), "bytecode") + + builder := newFakeBuilder(t) + classDirs, err := Resolve(root, builder) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + want := []string{ClassesDir(BuildDir(project)), precompiled} + if !reflect.DeepEqual(classDirs, want) { + t.Fatalf("Resolve() = %v, want %v", classDirs, want) + } + if !reflect.DeepEqual(builder.compiled, []string{project}) { + t.Fatalf("expected the project to be compiled, got %v", builder.compiled) + } +} + +// The same tree with a previous build output in place of the loose classes. +func TestResolveTreeMixingProjectAndBuildOutput(t *testing.T) { + root := t.TempDir() + project := newProject(t, filepath.Join(root, "models")) + prebuilt := filepath.Join(root, "prebuilt") + writeFile(t, filepath.Join(prebuilt, descriptorName), "sourceProject: /elsewhere\n") + writeFile(t, filepath.Join(prebuilt, classesDirName, "com", "example", "Other.class"), "bytecode") + + builder := newFakeBuilder(t) + classDirs, err := Resolve(root, builder) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + want := []string{ClassesDir(BuildDir(project)), ClassesDir(prebuilt)} + if !reflect.DeepEqual(classDirs, want) { + t.Fatalf("Resolve() = %v, want %v", classDirs, want) + } + if !reflect.DeepEqual(builder.compiled, []string{project}) { + t.Fatalf("expected the project to be compiled, got %v", builder.compiled) + } +} + +// A directory that holds classes of its own and a project below it belongs to neither +// shape: putting it on the classpath hides the project, recursing past it drops the +// classes. +func TestResolveRejectsAmbiguousDirectory(t *testing.T) { + root := t.TempDir() + newProject(t, filepath.Join(root, "models")) + writeFile(t, filepath.Join(root, "Stray.class"), "bytecode") + + _, err := Resolve(root, newFakeBuilder(t)) + if err == nil { + t.Fatal("expected a directory holding both classes and a project to be rejected") + } + if !strings.Contains(err.Error(), root) { + t.Errorf("error should name the ambiguous directory, got: %v", err) + } +} + +// A plain class directory resolves to its root, not to the package directory that +// happens to hold the class files. +func TestResolvePrecompiledClassDirectoryKeepsItsRoot(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "com", "example", "Model.class"), "bytecode") + + classDirs, err := Resolve(root, newFakeBuilder(t)) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if !reflect.DeepEqual(classDirs, []string{root}) { + t.Fatalf("Resolve() = %v, want the classpath root %s", classDirs, root) + } +} + +// The analyzer is handed every path's answer at once, so a directory reached twice must +// contribute once. +func TestResolveAllPassesEachDirectoryOnce(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := newFakeBuilder(t) + + classDirs, err := ResolveAll([]string{project, project}, builder) + if err != nil { + t.Fatalf("ResolveAll: %v", err) + } + if !reflect.DeepEqual(classDirs, []string{ClassesDir(BuildDir(project))}) { + t.Fatalf("ResolveAll() = %v, want the class directory once", classDirs) + } +} + +// Copies of one class are the ordinary result of passing overlapping paths, and the +// analyzer reads the same class whichever copy it picks. +func TestResolveAllAcceptsIdenticalCopiesOfAClass(t *testing.T) { + root := t.TempDir() + first := filepath.Join(root, "first") + second := filepath.Join(root, "second") + writeFile(t, filepath.Join(first, "com", "example", "Model.class"), "bytecode") + writeFile(t, filepath.Join(second, "com", "example", "Model.class"), "bytecode") + + classDirs, err := ResolveAll([]string{first, second}, newFakeBuilder(t)) + if err != nil { + t.Fatalf("ResolveAll: %v", err) + } + if !reflect.DeepEqual(classDirs, []string{first, second}) { + t.Fatalf("ResolveAll() = %v, want both directories", classDirs) + } +} + +// Two directories offering one class with different content leave the analyzer to pick, +// and nothing says which it picks: the directories must agree on the class content. +func TestResolveAllRejectsClashingClasses(t *testing.T) { + root := t.TempDir() + first := filepath.Join(root, "first") + second := filepath.Join(root, "second") + writeFile(t, filepath.Join(first, "com", "example", "Model.class"), "one bytecode") + writeFile(t, filepath.Join(second, "com", "example", "Model.class"), "another bytecode") + + _, err := ResolveAll([]string{first, second}, newFakeBuilder(t)) + if err == nil { + t.Fatal("expected two different copies of one class to be rejected") + } + for _, want := range []string{"com/example/Model.class", first, second} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error should mention %s, got: %v", want, err) + } + } +} + +func TestResolveLooseSourcesReportsMigration(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "com", "example", "Model.java"), "package com.example;\n") + + _, err := Resolve(dir, newFakeBuilder(t)) + if err == nil { + t.Fatal("expected loose .java sources to be rejected") + } + if !strings.Contains(err.Error(), "approximation init") { + t.Errorf("error should point at the migration command, got: %v", err) + } +} + +func TestResolveEmptyDirectory(t *testing.T) { + _, err := Resolve(t.TempDir(), newFakeBuilder(t)) + if err == nil { + t.Fatal("expected an empty directory to be rejected") + } +} + +func TestFailedBuildLeavesTheProjectBuildable(t *testing.T) { + project := newProject(t, t.TempDir()) + + // A failed build must not block the next build. + if err := Build(project, BuildDir(project), &collidingBuilder{fakeBuilder{t: t}}); err == nil { + t.Fatal("expected the colliding build to fail") + } + if err := Build(project, BuildDir(project), newFakeBuilder(t)); err != nil { + t.Fatalf("rebuild after a failed build: %v", err) + } +} + +func TestFailedRebuildKeepsThePreviousBuild(t *testing.T) { + project := newProject(t, t.TempDir()) + output := BuildDir(project) + + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("first build: %v", err) + } + if err := Build(project, output, &collidingBuilder{fakeBuilder{t: t}}); err == nil { + t.Fatal("expected the colliding build to fail") + } + + // The previous models must remain available. + if _, err := os.Stat(filepath.Join(ClassesDir(output), "com", "example", "Model.class")); err != nil { + t.Errorf("a failed rebuild destroyed the previous build: %v", err) + } + if !isBuiltOutput(output) { + t.Error("a failed rebuild left the output unusable") + } +} + +// A build assembles under a path of its own. A second builder of the same output must not +// be able to delete or contribute to it. +func TestBuildKeepsStagingOfARunningBuild(t *testing.T) { + project := newProject(t, t.TempDir()) + output := BuildDir(project) + + // What another builder of the same output would be assembling. Process 1 always runs. + foreign := filepath.Join(stagingRoot(output), filepath.Base(output)+"-1") + writeFile(t, filepath.Join(foreign, classesDirName, "com", "example", "Other.class"), "bytecode") + + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("Build: %v", err) + } + if _, err := os.Stat(filepath.Join(foreign, classesDirName, "com", "example", "Other.class")); err != nil { + t.Errorf("the build deleted a running builder's staging directory: %v", err) + } +} + +// A build killed outright leaves its staging directory behind. The next build reclaims it, +// but only once its owner is gone. +func TestBuildSweepsStagingOfAKilledBuild(t *testing.T) { + project := newProject(t, t.TempDir()) + output := BuildDir(project) + + abandoned := filepath.Join(stagingRoot(output), filepath.Base(output)+"-2147483646") + writeFile(t, filepath.Join(abandoned, classesDirName, "com", "example", "Other.class"), "bytecode") + + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("Build: %v", err) + } + if _, err := os.Stat(abandoned); !os.IsNotExist(err) { + t.Errorf("a killed build's staging directory was not reclaimed: %v", err) + } +} + +// Whatever is left behind must be invisible to the resolver: staged classes are not +// compiled models, and resolving them would apply a half-built batch to a scan. +func TestStagingIsInvisibleToTheResolver(t *testing.T) { + root := t.TempDir() + output := filepath.Join(root, "out") + abandoned := filepath.Join(stagingRoot(output), filepath.Base(output)+"-1") + writeFile(t, filepath.Join(abandoned, classesDirName, "com", "example", "Other.class"), "bytecode") + + if _, err := Resolve(root, newFakeBuilder(t)); err == nil { + t.Error("a leftover staging directory was resolved as compiled models") + } +} + +// A completed build leaves no staging or superseded directory behind. +func TestBuildCleansUpAfterItself(t *testing.T) { + project := newProject(t, t.TempDir()) + output := BuildDir(project) + + for range 2 { + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("Build: %v", err) + } + } + + entries, err := os.ReadDir(filepath.Dir(output)) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.Name() != filepath.Base(output) { + t.Errorf("build left %s behind next to the output", entry.Name()) + } + } +} + +func TestBuildRefusesToOverwriteUnrelatedDirectory(t *testing.T) { + project := newProject(t, t.TempDir()) + output := t.TempDir() + precious := filepath.Join(output, "notes.txt") + writeFile(t, precious, "important user data") + + // The build must not replace an unrelated directory. + err := Build(project, output, newFakeBuilder(t)) + if err == nil { + t.Fatal("expected a build into an unrelated directory to be refused") + } + if !strings.Contains(err.Error(), "not an approximation build output") { + t.Errorf("unexpected error: %v", err) + } + if content, readErr := os.ReadFile(precious); readErr != nil || string(content) != "important user data" { + t.Errorf("existing content was destroyed: content=%q err=%v", content, readErr) + } +} + +func TestBuildAcceptsEmptyAndPreviousOutputDirectories(t *testing.T) { + project := newProject(t, t.TempDir()) + output := t.TempDir() // exists and is empty + + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("build into an empty directory: %v", err) + } + // A new build can replace the first build. + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("rebuild over a previous build output: %v", err) + } +} + +func TestBuildWritesReadableDescriptor(t *testing.T) { + project := newProject(t, t.TempDir()) + output := filepath.Join(t.TempDir(), "out") + + if err := Build(project, output, newFakeBuilder(t)); err != nil { + t.Fatalf("Build: %v", err) + } + + raw, err := os.ReadFile(filepath.Join(output, descriptorName)) + if err != nil { + t.Fatal(err) + } + var parsed descriptor + if err := yaml.Unmarshal(raw, &parsed); err != nil { + t.Fatalf("descriptor is not valid YAML: %v\n%s", err, raw) + } + // The descriptor must mean the same thing wherever the output is read, so it may not + // carry a path that only exists on the machine that built it. + if filepath.IsAbs(parsed.SourceProject) { + t.Errorf("sourceProject = %q, want a path relative to the output", parsed.SourceProject) + } + if resolved := filepath.Clean(filepath.Join(output, parsed.SourceProject)); resolved != project { + t.Errorf("sourceProject resolves to %q, want %q", resolved, project) + } + if parsed.Dependencies == nil { + t.Error("dependencies should be an empty list, not null") + } +} + +func TestBuildRejectsCollidingModelClasses(t *testing.T) { + project := newProject(t, t.TempDir()) + builder := &collidingBuilder{fakeBuilder{t: t}} + + err := Build(project, BuildDir(project), builder) + if err == nil { + t.Fatal("expected colliding approximation classes to fail the build") + } + if !strings.Contains(err.Error(), "exactly one target class") { + t.Errorf("unexpected error: %v", err) + } +} + +// collidingBuilder writes the same class from two modules. +type collidingBuilder struct{ fakeBuilder } + +func (b *collidingBuilder) Compile(projectDir, projectModelDir string) error { + writeModuleClasses(b.t, projectModelDir, map[string][]string{"c0_main": {"Model"}, "c1_main": {"Model"}}) + return nil +} diff --git a/cli/internal/approximation/build.go b/cli/internal/approximation/build.go new file mode 100644 index 0000000000..20f1a89d97 --- /dev/null +++ b/cli/internal/approximation/build.go @@ -0,0 +1,319 @@ +package approximation + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "syscall" + + "github.com/seqra/opentaint/internal/utils" + "github.com/seqra/opentaint/internal/utils/project" +) + +// Builder compiles a model project. +type Builder interface { + // Fingerprint identifies the compiler. A build is reused only for the compiler + // that produced it. + Fingerprint() (string, error) + // Prepare updates local build inputs before the source stamp is calculated. + Prepare(projectDir string) error + // Compile writes a project model to projectModelDir. + Compile(projectDir, projectModelDir string) error +} + +// ensureBuilt returns current compiled models for a project. +func ensureBuilt(projectDir string, builder Builder) (string, error) { + inputs, err := prepare(projectDir, builder) + if err != nil { + return "", err + } + + outputDir := BuildDir(projectDir) + if upToDate(outputDir, inputs) { + return ClassesDir(outputDir), nil + } + if err := build(projectDir, outputDir, inputs, builder); err != nil { + return "", err + } + return ClassesDir(outputDir), nil +} + +// Build compiles a model project into outputDir. +func Build(projectDir, outputDir string, builder Builder) error { + inputs, err := prepare(projectDir, builder) + if err != nil { + return err + } + return build(projectDir, outputDir, inputs, builder) +} + +// prepare updates build inputs and returns the stamp identifying them. +func prepare(projectDir string, builder Builder) (stamp, error) { + toolchain, err := builder.Fingerprint() + if err != nil { + return stamp{}, err + } + if err := builder.Prepare(projectDir); err != nil { + return stamp{}, err + } + sources, err := computeSourceDigest(projectDir) + if err != nil { + return stamp{}, err + } + return stamp{Sources: sources, Toolchain: toolchain}, nil +} + +// stagingDirName holds the directories builds assemble in. It sits beside the output so +// that publishing is a single rename, and it is on the skip list so that a directory a +// killed build left behind is never mistaken for compiled models. +const stagingDirName = ".opentaint-staging" + +func stagingRoot(outputDir string) string { + return filepath.Join(filepath.Dir(outputDir), stagingDirName) +} + +// stagingDir names this process's staging directory for one output. The process id keeps +// concurrent builders of one output out of each other's staging directory: a shared one +// lets a build publish a mixture of two compiles, and record it as up to date. +func stagingDir(outputDir string) string { + return filepath.Join(stagingRoot(outputDir), + fmt.Sprintf("%s-%d", filepath.Base(outputDir), os.Getpid())) +} + +// asideDir names the superseded output while the new one is renamed into place. +func asideDir(outputDir string) string { + return filepath.Join(stagingRoot(outputDir), + fmt.Sprintf("%s-superseded-%d", filepath.Base(outputDir), os.Getpid())) +} + +// sweepStaging reclaims staging directories left behind by builds that are no longer +// running. A directory is removed only when its owner is known to be gone, so a build in +// progress is never disturbed; being wrong the other way costs a directory that stays +// until its process id comes round again. +func sweepStaging(outputDir string) { + root := stagingRoot(outputDir) + entries, err := os.ReadDir(root) + if err != nil { + return + } + prefix := filepath.Base(outputDir) + "-" + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), prefix) { + continue + } + owner, named := stagingOwner(entry.Name()) + if !named || owner == os.Getpid() || processRunning(owner) { + continue + } + _ = os.RemoveAll(filepath.Join(root, entry.Name())) + } +} + +// stagingOwner reads the process id a staging directory name ends with. +func stagingOwner(name string) (int, bool) { + separator := strings.LastIndex(name, "-") + if separator < 0 { + return 0, false + } + owner, err := strconv.Atoi(name[separator+1:]) + return owner, err == nil +} + +// processRunning reports whether a process is still running. It errs towards "running": +// a staging directory is reclaimed only when its owner is known to be gone. +func processRunning(pid int) bool { + if runtime.GOOS == "windows" { + return true + } + process, err := os.FindProcess(pid) + if err != nil { + return true + } + // A process that exists but belongs to someone else answers with EPERM. + err = process.Signal(syscall.Signal(0)) + return err == nil || errors.Is(err, syscall.EPERM) +} + +func build(projectDir, outputDir string, inputs stamp, builder Builder) error { + if err := checkOutputDir(outputDir); err != nil { + return err + } + sweepStaging(outputDir) + + buildTempDir, err := os.MkdirTemp("", "opentaint-approximation-build-*") + if err != nil { + return fmt.Errorf("failed to create temp directory for approximation build: %w", err) + } + defer func() { _ = os.RemoveAll(buildTempDir) }() + + // A missing output shows that the compiler did not produce a model. + projectModelDir := filepath.Join(buildTempDir, "model") + + if err := builder.Compile(projectDir, projectModelDir); err != nil { + return err + } + + // Assemble the new output before it replaces the current output. + staging := stagingDir(outputDir) + defer func() { _ = os.RemoveAll(staging) }() + if err := os.RemoveAll(staging); err != nil { + return fmt.Errorf("failed to clear %s: %w", staging, err) + } + if err := os.MkdirAll(ClassesDir(staging), 0o755); err != nil { + return fmt.Errorf("failed to create approximation build output %s: %w", staging, err) + } + + dependencies, err := collectClasses(projectModelDir, ClassesDir(staging)) + if err != nil { + return err + } + if !containsClassFiles(ClassesDir(staging)) { + return fmt.Errorf("approximation project %s compiled to no classes; it has no models under src/main/java", projectDir) + } + + if err := writeDescriptor(staging, descriptor{ + SourceProject: relativeSourceProject(outputDir, projectDir), + Dependencies: dependencies, + }); err != nil { + return err + } + if err := writeStamp(staging, inputs); err != nil { + return err + } + + if err := replaceDir(staging, outputDir); err != nil { + return err + } + + // Leave nothing beside the output once no build is using it. + _ = os.Remove(stagingRoot(outputDir)) + return nil +} + +// checkOutputDir prevents replacement of an unrelated directory. +func checkOutputDir(outputDir string) error { + info, err := os.Stat(outputDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("failed to access output directory %s: %w", outputDir, err) + } + if !info.IsDir() { + return fmt.Errorf("output path %s is not a directory", outputDir) + } + if isBuiltOutput(outputDir) { + return nil + } + + empty, err := isEmptyDir(outputDir) + if err != nil { + return err + } + if !empty { + return fmt.Errorf( + "output directory %s already exists and is not an approximation build output; "+ + "remove it or choose another --output path", outputDir) + } + return nil +} + +// replaceDir replaces the current output with a complete build. +// Both moves are single renames, so anyone reading the output directory sees the previous +// build, the new build, or nothing at all, and never a directory that is half of each. +func replaceDir(staging, outputDir string) error { + if err := os.MkdirAll(filepath.Dir(outputDir), 0o755); err != nil { + return fmt.Errorf("failed to create %s: %w", filepath.Dir(outputDir), err) + } + + aside := asideDir(outputDir) + if err := os.RemoveAll(aside); err != nil { + return fmt.Errorf("failed to clear %s: %w", aside, err) + } + superseded := isDir(outputDir) + if superseded { + if err := os.Rename(outputDir, aside); err != nil { + return fmt.Errorf("failed to move the previous build out of %s: %w", outputDir, err) + } + } + + if err := os.Rename(staging, outputDir); err != nil { + // Put the previous build back rather than leave the output missing. + if superseded { + _ = os.Rename(aside, outputDir) + } + return fmt.Errorf("failed to move the compiled models into %s: %w", outputDir, err) + } + + if superseded { + if err := os.RemoveAll(aside); err != nil { + return fmt.Errorf("failed to remove the previous build %s: %w", aside, err) + } + } + return nil +} + +func isEmptyDir(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return false, fmt.Errorf("failed to read output directory %s: %w", dir, err) + } + return len(entries) == 0, nil +} + +// collectClasses merges the compiled classes from all modules. +// It returns an error if two modules contain the same class. +func collectClasses(projectModelDir, destDir string) ([]string, error) { + config, err := project.LoadConfig(projectModelDir) + if err != nil { + return nil, fmt.Errorf("approximation build produced no readable project model: %w", err) + } + + for _, module := range config.AllModules() { + for _, classes := range module.ModuleClasses { + if err := copyTree(absoluteUnder(projectModelDir, classes), destDir); err != nil { + return nil, err + } + } + } + + dependencies := make([]string, 0, len(config.AllDependencies())) + for _, dep := range config.AllDependencies() { + dependencies = append(dependencies, filepath.Base(dep)) + } + return dependencies, nil +} + +func absoluteUnder(baseDir, path string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(baseDir, path) +} + +func copyTree(srcDir, destDir string) error { + return filepath.WalkDir(srcDir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + target := filepath.Join(destDir, rel) + if entry.IsDir() { + return os.MkdirAll(target, 0o755) + } + if _, err := os.Stat(target); err == nil { + return fmt.Errorf( + "two approximation modules produce %s; an approximation class maps to exactly one target class", + filepath.ToSlash(rel), + ) + } + return utils.CopyFile(path, target) + }) +} diff --git a/cli/internal/approximation/descriptor.go b/cli/internal/approximation/descriptor.go new file mode 100644 index 0000000000..cb118783ce --- /dev/null +++ b/cli/internal/approximation/descriptor.go @@ -0,0 +1,44 @@ +package approximation + +import ( + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v2" +) + +const descriptorHeader = "# Generated by 'opentaint compile approximations'. Do not edit.\n" + +// descriptor identifies the source and dependencies of a model build. +type descriptor struct { + // SourceProject is the project that supplied the models, relative to this directory. + SourceProject string `yaml:"sourceProject"` + // Dependencies are the libraries used to compile the models. + Dependencies []string `yaml:"dependencies"` +} + +// relativeSourceProject expresses the model project relative to the build output, so that +// a build output says the same thing on every machine that reads it. +func relativeSourceProject(outputDir, projectDir string) string { + relative, err := filepath.Rel(outputDir, projectDir) + if err != nil { + return filepath.Base(projectDir) + } + return filepath.ToSlash(relative) +} + +func writeDescriptor(outputDir string, d descriptor) error { + if d.Dependencies == nil { + d.Dependencies = []string{} + } + body, err := yaml.Marshal(d) + if err != nil { + return fmt.Errorf("failed to render the approximation descriptor: %w", err) + } + path := filepath.Join(outputDir, descriptorName) + if err := os.WriteFile(path, append([]byte(descriptorHeader), body...), 0o644); err != nil { + return fmt.Errorf("failed to write %s: %w", path, err) + } + return nil +} diff --git a/cli/internal/approximation/project.go b/cli/internal/approximation/project.go new file mode 100644 index 0000000000..eb8a721d94 --- /dev/null +++ b/cli/internal/approximation/project.go @@ -0,0 +1,215 @@ +// Package approximation manages Gradle projects for dataflow models. +// +// Each project specifies the library versions that its models use. +// The scanned project does not supply these compile dependencies. +package approximation + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" +) + +const ( + // apiJarName contains the annotations and support types for model projects. + apiJarName = "opentaint-approximations-api.jar" + // apiJarResourcePrefix must match API_JAR_RESOURCE_DIR in the core build. + apiJarResourcePrefix = "opentaint-approximations-api/" + + libsDirName = "libs" + classesDirName = "classes" + descriptorName = "approximation.yaml" + + stampFileName = "stamp.json" + gradleBuildFile = "build.gradle.kts" +) + +// opentaintDirName contains the local build output. +const opentaintDirName = ".opentaint" + +// skippedDirs contains generated and version-control directories. +var skippedDirs = map[string]bool{ + opentaintDirName: true, + stagingDirName: true, + "build": true, + ".gradle": true, + ".git": true, +} + +// IsProject reports whether dir is a model project root. +func IsProject(dir string) bool { + return isFile(filepath.Join(dir, gradleBuildFile)) +} + +// isBuiltOutput reports whether dir contains a model build. +func isBuiltOutput(dir string) bool { + return isFile(filepath.Join(dir, descriptorName)) && isDir(filepath.Join(dir, classesDirName)) +} + +// BuildDir returns the path of the local build output. +func BuildDir(projectDir string) string { + return filepath.Join(projectDir, opentaintDirName, "build") +} + +// ClassesDir returns the class directory in a build output. +func ClassesDir(outputDir string) string { + return filepath.Join(outputDir, classesDirName) +} + +// apiJarPath returns the path of the project API jar. +func apiJarPath(projectDir string) string { + return filepath.Join(projectDir, libsDirName, apiJarName) +} + +// containsClassFiles reports whether dir contains compiled classes. +// It does not search generated directories. +func containsClassFiles(dir string) bool { + found := false + _ = filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if path != dir && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + if isClassFileName(entry.Name()) { + found = true + return filepath.SkipAll + } + return nil + }) + return found +} + +// subtreeFacts records what a directory tree holds. +// Classifying a directory needs both facts, and one walk answers both. +type subtreeFacts struct { + // classFiles reports a compiled class anywhere in the tree. + classFiles bool + // unitSource reports a model project or a build output anywhere in the tree, + // that is, models this directory does not already hold compiled. + unitSource bool +} + +// scanSubtree walks dir once. It does not search generated directories. +func scanSubtree(dir string) subtreeFacts { + var facts subtreeFacts + _ = filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if path != dir && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + switch { + case isClassFileName(entry.Name()): + facts.classFiles = true + case entry.Name() == gradleBuildFile: + facts.unitSource = true + case entry.Name() == descriptorName && isDir(filepath.Join(filepath.Dir(path), classesDirName)): + facts.unitSource = true + } + if facts.classFiles && facts.unitSource { + return filepath.SkipAll + } + return nil + }) + return facts +} + +// directClassFile returns one compiled class held by the directory itself. +func directClassFile(entries []os.DirEntry) (string, bool) { + for _, entry := range entries { + if !entry.IsDir() && isClassFileName(entry.Name()) { + return entry.Name(), true + } + } + return "", false +} + +func isClassFileName(name string) bool { + return strings.EqualFold(filepath.Ext(name), ".class") +} + +// classFilesIn returns the compiled classes a class directory offers, as slash-separated +// paths relative to that directory. +func classFilesIn(dir string) ([]string, error) { + var classes []string + err := filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if path != dir && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + if !isClassFileName(entry.Name()) { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + classes = append(classes, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to read the compiled models in %s: %w", dir, err) + } + return classes, nil +} + +// sameContent reports whether two files hold the same bytes. +func sameContent(first, second string) (bool, error) { + firstBytes, err := os.ReadFile(first) + if err != nil { + return false, fmt.Errorf("failed to read %s: %w", first, err) + } + secondBytes, err := os.ReadFile(second) + if err != nil { + return false, fmt.Errorf("failed to read %s: %w", second, err) + } + return bytes.Equal(firstBytes, secondBytes), nil +} + +// firstJavaSource returns one Java source file under dir. +func firstJavaSource(dir string) (string, bool) { + var found string + _ = filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if path != dir && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + if strings.EqualFold(filepath.Ext(entry.Name()), ".java") { + found = path + return filepath.SkipAll + } + return nil + }) + return found, found != "" +} + +func isFile(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} + +func isDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} diff --git a/cli/internal/approximation/resolve.go b/cli/internal/approximation/resolve.go new file mode 100644 index 0000000000..c0505e3474 --- /dev/null +++ b/cli/internal/approximation/resolve.go @@ -0,0 +1,166 @@ +package approximation + +import ( + "fmt" + "os" + "path/filepath" +) + +// ResolveAll returns the class directories for every model path, in the order the paths +// were given. A directory reached more than once contributes once, and the directories +// must agree about every class they share: which copy an analysis reads is not defined, so +// two different copies of one class would leave the result of the scan unpinned. +func ResolveAll(paths []string, builder Builder) ([]string, error) { + var classDirs []string + resolved := make(map[string]bool) + for _, path := range paths { + dirs, err := Resolve(path, builder) + if err != nil { + return nil, err + } + for _, dir := range dirs { + dir = filepath.Clean(dir) + if resolved[dir] { + continue + } + resolved[dir] = true + classDirs = append(classDirs, dir) + } + } + if err := checkClasspathAgrees(classDirs); err != nil { + return nil, err + } + return classDirs, nil +} + +// checkClasspathAgrees rejects class directories that offer one class with different +// content. Identical copies are the ordinary result of overlapping paths and are left +// alone: what has to hold is that the same class is read whichever copy is picked. +func checkClasspathAgrees(classDirs []string) error { + offeredBy := make(map[string]string) + for _, dir := range classDirs { + classes, err := classFilesIn(dir) + if err != nil { + return err + } + for _, class := range classes { + previous, offered := offeredBy[class] + if !offered { + offeredBy[class] = dir + continue + } + same, err := sameContent( + filepath.Join(previous, filepath.FromSlash(class)), + filepath.Join(dir, filepath.FromSlash(class))) + if err != nil { + return err + } + if !same { + return fmt.Errorf( + "%s is offered with different content by %s and by %s; which one an analysis "+ + "reads is not defined.\nDrop one of the copies, or pass only one of these "+ + "approximation paths", + class, previous, dir) + } + } + } + return nil +} + +// Resolve returns the class directories for one model path. +// The path can have one of these forms: +// +// 1. A model project. +// 2. A model build output. +// 3. A compiled class directory. +// 4. A directory tree that contains these items. +func Resolve(path string, builder Builder) ([]string, error) { + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("approximation path does not exist: %w", err) + } + if !info.IsDir() { + return nil, fmt.Errorf("approximation path %s is not a directory", path) + } + + classDirs, err := resolveDir(path, builder) + if err != nil { + return nil, err + } + if len(classDirs) == 0 { + return nil, emptyPathError(path) + } + return classDirs, nil +} + +func resolveDir(dir string, builder Builder) ([]string, error) { + if IsProject(dir) { + classes, err := ensureBuilt(dir, builder) + if err != nil { + return nil, err + } + return []string{classes}, nil + } + + if isBuiltOutput(dir) { + return []string{ClassesDir(dir)}, nil + } + + // A directory is a class directory only when nothing below it still needs building. + // Without that condition a tree holding both a project and compiled classes reads as + // one class directory, and the project's models never reach the analysis. + facts := scanSubtree(dir) + if facts.classFiles && !facts.unitSource { + return []string{dir}, nil + } + + // os.ReadDir returns entries sorted by name, so the answer does not depend on the + // order the filesystem happens to hand them back. + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("failed to read approximation directory %s: %w", dir, err) + } + + // Descending past classes this directory holds itself would leave them unused. + if classFile, found := directClassFile(entries); found { + return nil, ambiguousDirError(dir, classFile) + } + + var classDirs []string + for _, entry := range entries { + if !entry.IsDir() || skippedDirs[entry.Name()] { + continue + } + nested, err := resolveDir(filepath.Join(dir, entry.Name()), builder) + if err != nil { + return nil, err + } + classDirs = append(classDirs, nested...) + } + return classDirs, nil +} + +// ambiguousDirError reports a directory that is neither a class directory nor a +// directory of approximation paths. +func ambiguousDirError(dir, classFile string) error { + return fmt.Errorf( + "%s holds compiled classes of its own (e.g. %s) and an approximation project or build "+ + "output below it, so it is neither a class directory nor a directory of them.\n"+ + "Move the classes into a directory of their own, or pass each approximation path to "+ + "--dataflow-approximations separately", + dir, classFile) +} + +// emptyPathError reports why a path does not contain usable models. +func emptyPathError(path string) error { + if source, found := firstJavaSource(path); found { + return fmt.Errorf( + "%s holds approximation sources (e.g. %s) but no approximation project: nothing declares "+ + "the dependencies to compile the models against.\n"+ + "Create a project per model directory with 'opentaint approximation init "+ + "--dependency ' and move its sources under /src/main/java", + path, source, + ) + } + return fmt.Errorf("%s contains no approximation project, build output or compiled classes", path) +} diff --git a/cli/internal/approximation/scaffold.go b/cli/internal/approximation/scaffold.go new file mode 100644 index 0000000000..d3d9b1e09d --- /dev/null +++ b/cli/internal/approximation/scaffold.go @@ -0,0 +1,70 @@ +package approximation + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/seqra/opentaint/internal/testproject" +) + +const gradleProjectName = "approximation-models" + +// Scaffold creates a Gradle project for dataflow models. +func Scaffold(projectDir string, dependencies []string, analyzerJarPath string) error { + apiJar, err := ExtractApiJar(analyzerJarPath) + if err != nil { + return err + } + + localJar := filepath.ToSlash(filepath.Join(libsDirName, apiJarName)) + if err := testproject.BootstrapWithLocalJars(projectDir, gradleProjectName, dependencies, []string{localJar}); err != nil { + return err + } + if err := WriteApiJar(projectDir, apiJar); err != nil { + return err + } + return os.MkdirAll(filepath.Join(projectDir, "src", "main", "java"), 0o755) +} + +// ExtractApiJar reads the approximation API jar from the analyzer jar. +func ExtractApiJar(analyzerJarPath string) ([]byte, error) { + reader, err := zip.OpenReader(analyzerJarPath) + if err != nil { + return nil, fmt.Errorf("failed to open analyzer jar %s: %w", analyzerJarPath, err) + } + defer func() { _ = reader.Close() }() + + entryName := apiJarResourcePrefix + apiJarName + for _, file := range reader.File { + if file.Name == entryName { + return readZipEntry(file) + } + } + return nil, fmt.Errorf("analyzer jar %s bundles no %s", analyzerJarPath, apiJarName) +} + +func readZipEntry(file *zip.File) ([]byte, error) { + content, err := file.Open() + if err != nil { + return nil, fmt.Errorf("failed to read %s from the analyzer jar: %w", file.Name, err) + } + defer func() { _ = content.Close() }() + return io.ReadAll(content) +} + +// WriteApiJar writes the approximation API jar to a model project. +// It does not write the file when the content is current. +func WriteApiJar(projectDir string, apiJar []byte) error { + path := apiJarPath(projectDir) + if current, err := os.ReadFile(path); err == nil && bytes.Equal(current, apiJar) { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("failed to create %s: %w", filepath.Dir(path), err) + } + return os.WriteFile(path, apiJar, 0o644) +} diff --git a/cli/internal/approximation/stamp.go b/cli/internal/approximation/stamp.go new file mode 100644 index 0000000000..98cd6323b2 --- /dev/null +++ b/cli/internal/approximation/stamp.go @@ -0,0 +1,95 @@ +package approximation + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" +) + +// stamp identifies the inputs of a model build: the project, and the compiler that read +// it. The project alone does not determine the compiled classes, so a build carrying only +// a source digest would be reused across a compiler upgrade. +type stamp struct { + Sources string `json:"sources"` + Toolchain string `json:"toolchain"` +} + +func (s stamp) complete() bool { return s.Sources != "" && s.Toolchain != "" } + +// computeSourceDigest hashes all source files and build inputs. +// It does not hash generated files. +func computeSourceDigest(projectDir string) (string, error) { + var paths []string + err := filepath.WalkDir(projectDir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if path != projectDir && skippedDirs[entry.Name()] { + return filepath.SkipDir + } + return nil + } + rel, err := filepath.Rel(projectDir, path) + if err != nil { + return err + } + paths = append(paths, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return "", fmt.Errorf("failed to read approximation project %s: %w", projectDir, err) + } + sort.Strings(paths) + + digest := sha256.New() + for _, rel := range paths { + if _, err := fmt.Fprintf(digest, "%s\x00", rel); err != nil { + return "", fmt.Errorf("failed to hash path %s: %w", rel, err) + } + file, err := os.Open(filepath.Join(projectDir, filepath.FromSlash(rel))) + if err != nil { + return "", fmt.Errorf("failed to read %s: %w", rel, err) + } + _, copyErr := io.Copy(digest, file) + _ = file.Close() + if copyErr != nil { + return "", fmt.Errorf("failed to read %s: %w", rel, copyErr) + } + } + return hex.EncodeToString(digest.Sum(nil)), nil +} + +func readStamp(outputDir string) (stamp, bool) { + data, err := os.ReadFile(filepath.Join(outputDir, stampFileName)) + if err != nil { + return stamp{}, false + } + var s stamp + if err := json.Unmarshal(data, &s); err != nil { + return stamp{}, false + } + return s, s.complete() +} + +func writeStamp(outputDir string, s stamp) error { + data, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(outputDir, stampFileName), append(data, '\n'), 0o644) +} + +// upToDate reports whether outputDir was built from these inputs. +func upToDate(outputDir string, current stamp) bool { + previous, ok := readStamp(outputDir) + if !ok || previous != current { + return false + } + return containsClassFiles(ClassesDir(outputDir)) +} diff --git a/cli/internal/autobuilder/buildlog.go b/cli/internal/autobuilder/buildlog.go new file mode 100644 index 0000000000..60140243bb --- /dev/null +++ b/cli/internal/autobuilder/buildlog.go @@ -0,0 +1,51 @@ +// Package autobuilder reads autobuilder output. +package autobuilder + +import ( + "os" + "regexp" + "strings" +) + +// logLineSeparator precedes the build-tool output in an autobuilder log line. +const logLineSeparator = " - " + +// compilerErrorCount matches a javac error summary. +var compilerErrorCount = regexp.MustCompile(`^\d+ errors?$`) + +// CompilerDiagnostics returns compiler errors from an autobuilder log. +// It returns an empty string if the log has no compiler error. +func CompilerDiagnostics(logPath string) string { + log, err := os.ReadFile(logPath) + if err != nil { + return "" + } + + var diagnostics []string + for _, line := range strings.Split(string(log), "\n") { + message := buildOutput(line) + switch { + case isCompilerDiagnostic(message): + diagnostics = append(diagnostics, " "+message) + case len(diagnostics) > 0 && compilerErrorCount.MatchString(message): + // javac closes with a count of what it just reported. On its own the line is + // only something that reads like one, so it is kept as a trailer and never as + // the diagnostic. + diagnostics = append(diagnostics, " "+message) + } + } + return strings.Join(diagnostics, "\n") +} + +func buildOutput(logLine string) string { + message := logLine + if _, after, found := strings.Cut(logLine, logLineSeparator); found { + message = after + } + return strings.TrimRight(message, "\r") +} + +func isCompilerDiagnostic(message string) bool { + // Match javac and kotlinc error formats. + return strings.Contains(message, ": error: ") || strings.HasPrefix(message, "e: ") +} diff --git a/cli/internal/autobuilder/buildlog_test.go b/cli/internal/autobuilder/buildlog_test.go new file mode 100644 index 0000000000..066e20af39 --- /dev/null +++ b/cli/internal/autobuilder/buildlog_test.go @@ -0,0 +1,89 @@ +package autobuilder + +import ( + "os" + "path/filepath" + "testing" +) + +func writeLog(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "autobuild.log") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestCompilerDiagnostics(t *testing.T) { + tests := []struct { + name string + log string + want string + }{ + { + name: "javac error among the build trace", + log: `20:22:13.627 |I| ProjectResolver - Gradle build start for: /tmp/batch +20:22:13.628 |D| ProjectResolver - /tmp/batch/src/main/java/Model.java:4: error: package org.nosuchlib does not exist +20:22:13.629 |D| ProjectResolver - import org.nosuchlib.Missing; +20:22:13.629 |D| ProjectResolver - 1 error +20:22:13.630 |D| ProjectResolver - FAILURE: Build failed with an exception. +`, + want: " /tmp/batch/src/main/java/Model.java:4: error: package org.nosuchlib does not exist\n 1 error", + }, + { + name: "kotlinc diagnostics", + log: "12:00:00.000 |D| ProjectResolver - e: file:///tmp/batch/Model.kt:7:1 unresolved reference\n", + want: " e: file:///tmp/batch/Model.kt:7:1 unresolved reference", + }, + { + name: "a count of errors follows the errors it counts", + log: `12:00:00.000 |D| ProjectResolver - Model.java:4: error: cannot find symbol +12:00:00.000 |D| ProjectResolver - Model.java:9: error: cannot find symbol +12:00:00.000 |D| ProjectResolver - 2 errors +`, + want: " Model.java:4: error: cannot find symbol\n Model.java:9: error: cannot find symbol\n 2 errors", + }, + { + name: "a line that only reads like a count is not a diagnostic", + log: "12:00:00.000 |D| ProjectResolver - 3 errors\n", + want: "", + }, + { + name: "raw build output with no log prefix", + log: "Model.java:8: error: cannot find symbol\n", + want: " Model.java:8: error: cannot find symbol", + }, + { + name: "successful build has no diagnostics", + log: `20:22:13.627 |I| ProjectResolver - Gradle build start for: /tmp/batch +20:22:13.630 |I| ProjectResolver - BUILD SUCCESSFUL in 4s +`, + want: "", + }, + { + name: "an error mentioned in prose is not a diagnostic", + log: "12:00:00.000 |I| ProjectResolver - Gradle build failed for: /tmp/batch\n", + want: "", + }, + { + name: "windows line endings are trimmed", + log: "12:00:00.000 |D| ProjectResolver - Model.java:8: error: cannot find symbol\r\n", + want: " Model.java:8: error: cannot find symbol", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := CompilerDiagnostics(writeLog(t, test.log)); got != test.want { + t.Errorf("CompilerDiagnostics() =\n%q\nwant\n%q", got, test.want) + } + }) + } +} + +func TestCompilerDiagnosticsMissingLog(t *testing.T) { + if got := CompilerDiagnostics(filepath.Join(t.TempDir(), "absent.log")); got != "" { + t.Errorf("CompilerDiagnostics() = %q, want \"\" for an unreadable log", got) + } +} diff --git a/cli/internal/testproject/testproject.go b/cli/internal/testproject/testproject.go index 5e7d7e70d9..5dfcb53046 100644 --- a/cli/internal/testproject/testproject.go +++ b/cli/internal/testproject/testproject.go @@ -9,13 +9,18 @@ import ( ) func Bootstrap(outputDir, projectName string, dependencies []string) error { + return BootstrapWithLocalJars(outputDir, projectName, dependencies, nil) +} + +// BootstrapWithLocalJars adds project-relative jars to the compile-only classpath. +func BootstrapWithLocalJars(outputDir, projectName string, dependencies, localJars []string) error { return utils.WriteFiles(map[string][]byte{ - filepath.Join(outputDir, "build.gradle.kts"): buildGradle(dependencies), + filepath.Join(outputDir, "build.gradle.kts"): buildGradle(dependencies, localJars), filepath.Join(outputDir, "settings.gradle.kts"): settingsGradle(projectName), }) } -func buildGradle(dependencies []string) []byte { +func buildGradle(dependencies, localJars []string) []byte { var sb strings.Builder sb.WriteString(`plugins { java @@ -32,6 +37,9 @@ repositories { dependencies { `) + for _, jar := range localJars { + fmt.Fprintf(&sb, " compileOnly(files(%q))\n", filepath.ToSlash(jar)) + } for _, dep := range dependencies { fmt.Fprintf(&sb, " compileOnly(%q)\n", dep) } diff --git a/cli/internal/utils/extract.go b/cli/internal/utils/extract.go index 90421152f6..c2262a1333 100644 --- a/cli/internal/utils/extract.go +++ b/cli/internal/utils/extract.go @@ -56,58 +56,6 @@ func ExtractTar(tr *tar.Reader, basePath, destPath string, isSourceDir bool) err return nil } -// ExtractZipPrefix extracts entries whose names begin with prefix from the zip -// at src into destDir, stripping prefix from each entry's relative path. -// Directory entries are skipped — parent directories are created on demand. -// Returns an error if the zip cannot be opened or any entry fails to write. -func ExtractZipPrefix(src, prefix, destDir string) error { - r, err := zip.OpenReader(src) - if err != nil { - return fmt.Errorf("failed to open zip: %w", err) - } - defer func() { _ = r.Close() }() - - for _, f := range r.File { - if !strings.HasPrefix(f.Name, prefix) || f.FileInfo().IsDir() { - continue - } - relPath := strings.TrimPrefix(f.Name, prefix) - if relPath == "" { - continue - } - if err := copyZipEntry(f, filepath.Join(destDir, relPath)); err != nil { - return err - } - } - return nil -} - -func copyZipEntry(f *zip.File, destPath string) (err error) { - if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { - return err - } - src, err := f.Open() - if err != nil { - return err - } - defer func() { - if cerr := src.Close(); cerr != nil && err == nil { - err = cerr - } - }() - dst, err := os.Create(destPath) - if err != nil { - return err - } - defer func() { - if cerr := dst.Close(); cerr != nil && err == nil { - err = cerr - } - }() - _, err = io.Copy(dst, src) - return err -} - // ExtractZip extracts the contents of a ZIP file to the specified destination directory. func ExtractZip(src, dest string) error { zr, err := zip.OpenReader(src) diff --git a/cli/internal/utils/java/detection.go b/cli/internal/utils/java/detection.go index 736bfb1cac..b2cde7b871 100644 --- a/cli/internal/utils/java/detection.go +++ b/cli/internal/utils/java/detection.go @@ -3,9 +3,7 @@ package java import ( "os" "os/exec" - "path/filepath" "regexp" - "runtime" "strconv" "strings" @@ -132,17 +130,6 @@ func validateJavaInstallation(javaPath string) *JavaInstallation { return installation } -// DeriveJavacPath returns the javac binary path that sits next to the given -// java binary. On Windows the ".exe" suffix is preserved. -func DeriveJavacPath(javaPath string) string { - dir := filepath.Dir(javaPath) - name := "javac" - if runtime.GOOS == "windows" { - name = "javac.exe" - } - return filepath.Join(dir, name) -} - func extractVendor(versionOutput string) string { output := strings.ToLower(versionOutput) diff --git a/docs/usage.md b/docs/usage.md index 84f9359ed2..eb2d776d2b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -85,6 +85,7 @@ Use [CodeChecker](https://github.com/Ericsson/codechecker) for advanced result m | `opentaint health` | Show resolved paths for the analyzer, autobuilder, rules, and Java runtime | | `opentaint test rule` | Create, run, and debug detection-rule tests | | `opentaint test approximation` | Create and run dataflow-approximation tests | +| `opentaint approximation` | Create dataflow approximation projects | | `opentaint pull` | Download analyzer dependencies | | `opentaint update` | Update to latest version | | `opentaint prune` | Remove stale downloaded artifacts and cached models | @@ -115,7 +116,7 @@ These flags are to work with custom approximations: |------|-------------| | `--track-external-methods` | Write external-method coverage files next to the SARIF report | | `--passthrough-approximations` | Apply pass-through approximation YAML files or directories (repeatable) | -| `--dataflow-approximations` | Apply dataflow approximation classes or Java source directories (repeatable) | +| `--dataflow-approximations` | Apply dataflow approximation projects, build outputs, or class directories (repeatable) | Use external-method tracking when a scan may miss flows through library methods. The dropped-methods file shows where taint was killed because no model was available; the approximated-methods file shows methods already covered by built-in or custom models. @@ -160,7 +161,8 @@ opentaint test rule reachability java/security/my-rule.yaml:my-rule --project-mo #### Approximation tests ```bash -opentaint test approximation init .opentaint/test-projects/my-approximation +opentaint test approximation init .opentaint/test-projects/my-approximation \ + --dependency "io.projectreactor:reactor-core:3.8.5" opentaint compile .opentaint/test-projects/my-approximation -o .opentaint/test-compiled/my-approximation opentaint test approximation run .opentaint/test-compiled/my-approximation \ --dataflow-approximations .opentaint/dataflow/my-approximation @@ -188,6 +190,35 @@ opentaint scan --project-model ./my-project-model | `--dry-run` | Validate inputs and show what would run without compiling | | `--log-file` | Path to the log file (default: `/logs/.log`) | +### opentaint approximation + +A dataflow approximation models how taint moves through a method the analyzer cannot see into. Because a model references the library type it models, it needs that library to compile — so the models live in their own project, which pins those versions itself: + +```bash +opentaint approximation init .opentaint/dataflow/my-batch \ + --dependency "io.projectreactor:reactor-core:3.8.5" + +# write the @Approximate classes under src/main/java/, then apply them +opentaint scan --project-model ./my-project-model \ + --dataflow-approximations .opentaint/dataflow/my-batch +``` + +Pin the versions the target application uses. Those pins are the models' compile environment: they, and nothing about the project under analysis, decide what the models compile against, so a model compiles identically wherever it is applied. + +`--dataflow-approximations` accepts an approximation project, a build output, or a directory of compiled classes — and a directory holding any of those, so a tree with one project per batch can be passed as a single flag. A directory counts as compiled classes only when nothing below it still needs building; one that holds compiled classes of its own alongside a project is reported rather than guessed at. Paths may overlap, and the same models may be reached twice, but two of them may not offer the same class with different content — which copy an analysis would read is not defined. Projects are built on demand and rebuilt when their sources, their dependency pins, or the compiler change. + +| Command | Description | +|---------|-------------| +| `opentaint approximation init ` | Create a dataflow approximation project | +| `opentaint compile approximations ` | Compile a dataflow approximation project | + +| Flag | Description | +|------|-------------| +| `--dependency` | Compile-only Maven dependency coordinates the models are written against (repeatable, `init` only) | +| `--output`, `-o` | Path to the compiled models (default: `/.opentaint/build`, `compile approximations` only) | + +Compiling ahead of time is optional — scanning and testing build the project themselves. Run it to see compilation errors on their own. + ### opentaint summary View findings from a SARIF report. By default it prints the Scan Summary; add diff --git a/skills-templates/create-dataflow-approximation/references/java.md.j2 b/skills-templates/create-dataflow-approximation/references/java.md.j2 index 523eb6e5b7..125fabdef3 100644 --- a/skills-templates/create-dataflow-approximation/references/java.md.j2 +++ b/skills-templates/create-dataflow-approximation/references/java.md.j2 @@ -10,7 +10,16 @@ An app-internal method sits in the project's own sources under ``. ### 2. Write the approximation source -Create Java files under `.opentaint/dataflow/` — one `@Approximate` class per target class. `@Approximate(TargetClass.class)` binds a model to exactly that class, so target the EXACT class the analyzer dropped — the dropped FQN reflects how the call resolved: an interface-typed receiver (`Map m = ...; m.computeIfAbsent(...)`) drops `java.util.Map#computeIfAbsent`, a concrete one (`new HashMap<>()`) drops `java.util.HashMap#computeIfAbsent`. An interface is a valid target — write `@Approximate` on it exactly as on a concrete class, modelling whichever FQN the analyzer dropped. Reach the real object with `(TargetClass) (Object) this`, put functional-interface parameters behind `@ArgumentTypeContext`, and branch with `OpentaintNdUtil.nextBool()` so the analyzer walks both paths. Never leave a body empty. +Scaffold the approximation project with the same `--dependency` pins as the batch's test project (`.opentaint/test-projects//build.gradle.kts`): + +```bash +opentaint approximation init .opentaint/dataflow/ \ + --dependency "io.projectreactor:reactor-core:3.8.5" +``` + +When a model needs a library the project doesn't pin yet, add it to the project's `build.gradle.kts`. + +Create Java files under `.opentaint/dataflow//src/main/java` — one `@Approximate` class per target class. `@Approximate(TargetClass.class)` binds a model to exactly that class, so target the EXACT class the analyzer dropped — the dropped FQN reflects how the call resolved: an interface-typed receiver (`Map m = ...; m.computeIfAbsent(...)`) drops `java.util.Map#computeIfAbsent`, a concrete one (`new HashMap<>()`) drops `java.util.HashMap#computeIfAbsent`. An interface is a valid target — write `@Approximate` on it exactly as on a concrete class, modelling whichever FQN the analyzer dropped. Reach the real object with `(TargetClass) (Object) this`, put functional-interface parameters behind `@ArgumentTypeContext`, and branch with `OpentaintNdUtil.nextBool()` so the analyzer walks both paths. Never leave a body empty. ```java package com.example.approximations.batchpkg; // per-batch package (e.g. ...approximations.cn_hutool_001) — see the globally-unique rule below @@ -68,7 +77,7 @@ opentaint test approximation run .opentaint/test-compiled/ \ --dataflow-approximations .opentaint/dataflow/ ``` -`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI auto-compiles the `.java` sources against the analyzer JAR (for `@Approximate`, `OpentaintNdUtil`, `ArgumentTypeContext`) and the project's dependencies; if compilation fails it reports the errors and aborts before the tests. A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: +`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI builds the approximation project first. if compilation fails it reports the errors and aborts before the tests. A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: ```bash uv run /scripts/check-test-result.py @@ -83,7 +92,7 @@ Fix by the verdict it reports: | Pattern | Usage | |---|---| -| `@Approximate(TargetClass.class)` | Link the approximation to its target class. Must be on the compile classpath (a project dependency or a JDK type) | +| `@Approximate(TargetClass.class)` | Link the approximation to its target class. Must be on the compile classpath (a pinned dependency or a JDK type) | | `(TargetClass) (Object) this` | Cast to reach the real object's methods | | `@ArgumentTypeContext` | On lambda / functional-interface parameters | | `OpentaintNdUtil.nextBool()` | Non-deterministic branch — the analyzer considers both paths | diff --git a/skills-templates/create-dataflow-approximation/sections/output.md b/skills-templates/create-dataflow-approximation/sections/output.md index 773a3a1e8d..eec392b57c 100644 --- a/skills-templates/create-dataflow-approximation/sections/output.md +++ b/skills-templates/create-dataflow-approximation/sections/output.md @@ -1,6 +1,6 @@ ### Artifacts -- `.opentaint/dataflow/` — the language-specific code approximation artifacts that the scan consumes; report the path and the exact test command used +- `.opentaint/dataflow/` — the batch's approximation project with the language-specific code approximation artifacts that the scan consumes; report the path and the exact test command used - the passing methods present in the batch file's `build.done` (new methods appended; repaired methods already recorded, per Tracking) ### Summary diff --git a/skills-templates/create-dataflow-approximation/sections/workflow.md b/skills-templates/create-dataflow-approximation/sections/workflow.md index 5b93dd2d26..1b642a03e3 100644 --- a/skills-templates/create-dataflow-approximation/sections/workflow.md +++ b/skills-templates/create-dataflow-approximation/sections/workflow.md @@ -4,7 +4,7 @@ Find and read each dataflow method's real source: take methods not yet in `build ### 2. Write the approximation -Reproduce that propagation in the language-specific code form under `.opentaint/dataflow/`, following the language reference's artifact layout. Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. +Reproduce that propagation in the language-specific code form in the batch's approximation project at `.opentaint/dataflow/` — scaffold it if it doesn't exist yet — following the language reference's artifact layout. Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. ### 3. Test against the test project diff --git a/skills-templates/create-test-project/references/java-approximation.md.j2 b/skills-templates/create-test-project/references/java-approximation.md.j2 index e5152e14d2..8cee6ff26b 100644 --- a/skills-templates/create-test-project/references/java-approximation.md.j2 +++ b/skills-templates/create-test-project/references/java-approximation.md.j2 @@ -6,7 +6,7 @@ Language reference for a `dataflow` type, keyed to the body's steps. ### 1. Scaffold the project -`opentaint test approximation init ` scaffolds the Gradle build plus `Taint.java` and the fixed `approximation-rule.yaml` the harness applies. Pass each lib as a `--dependency` at its exact version from the app's dependency management (`.opentaint/project/sources/**/pom.xml`) verbatim — including timestamped snapshots — not a guess: this Gradle build is the approximation's own compile environment, so it must recompile from these pins even after the main project drops that dependency. +`opentaint test approximation init ` scaffolds the Gradle build plus `Taint.java` and the fixed `approximation-rule.yaml` the harness applies. Pass each lib as a `--dependency` at its exact version from the app's dependency management (`.opentaint/project/sources/**/pom.xml`) verbatim — including timestamped snapshots — not a guess. The approximation project scaffolded later takes the same pins, so the samples and the models they exercise compile against one version. ```bash opentaint test approximation init .opentaint/test-projects/ \ diff --git a/skills-templates/run-scan/sections/workflow.md b/skills-templates/run-scan/sections/workflow.md index 4ea2d1f03b..d351480681 100644 --- a/skills-templates/run-scan/sections/workflow.md +++ b/skills-templates/run-scan/sections/workflow.md @@ -11,7 +11,7 @@ opentaint scan --project-model .opentaint/project \ - `--rule-id ` — restrict to specific executable rules (repeatable, one per input rule ID); referenced library rules remain available without listing their IDs separately. Omit to run all loaded rules - `--passthrough-approximations .opentaint/pass-through` — add when that directory exists: passThrough configs override built-ins at the rule level, a provided rule overriding a built-in only when it matches one -- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations (sources auto-compiled; pre-compiled `.class` dirs passed through as-is) +- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations, one project per batch Both approximation-dir flags walk their trees recursively; pass each parent directory once, not every package or batch separately. diff --git a/skills/create-dataflow-approximation/SKILL.md b/skills/create-dataflow-approximation/SKILL.md index 27b05ac3dc..c09015791d 100644 --- a/skills/create-dataflow-approximation/SKILL.md +++ b/skills/create-dataflow-approximation/SKILL.md @@ -28,7 +28,7 @@ Find and read each dataflow method's real source: take methods not yet in `build ### 2. Write the approximation -Reproduce that propagation in the language-specific code form under `.opentaint/dataflow/`, following the language reference's artifact layout. Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. +Reproduce that propagation in the language-specific code form in the batch's approximation project at `.opentaint/dataflow/` — scaffold it if it doesn't exist yet — following the language reference's artifact layout. Cover every assigned callable variant, repair an explicitly handed callable in the existing source, and add new ones there rather than rewriting the file. The engine is field-sensitive — taint is tracked per field — so route data field-to-field exactly as the source does rather than tainting the whole object. The test project's negative samples (if present) verify this by storing taint in one field and reading another, so an over-broad model makes them fire. The concrete constructs and patterns are in the language reference. ### 3. Test against the test project @@ -42,7 +42,7 @@ When the sample won't converge after ~3 fixes — whether the trace shows a fait ### Artifacts -- `.opentaint/dataflow/` — the language-specific code approximation artifacts that the scan consumes; report the path and the exact test command used +- `.opentaint/dataflow/` — the batch's approximation project with the language-specific code approximation artifacts that the scan consumes; report the path and the exact test command used - the passing methods present in the batch file's `build.done` (new methods appended; repaired methods already recorded, per Tracking) ### Summary diff --git a/skills/create-dataflow-approximation/references/java.md b/skills/create-dataflow-approximation/references/java.md index 523eb6e5b7..125fabdef3 100644 --- a/skills/create-dataflow-approximation/references/java.md +++ b/skills/create-dataflow-approximation/references/java.md @@ -10,7 +10,16 @@ An app-internal method sits in the project's own sources under ``. ### 2. Write the approximation source -Create Java files under `.opentaint/dataflow/` — one `@Approximate` class per target class. `@Approximate(TargetClass.class)` binds a model to exactly that class, so target the EXACT class the analyzer dropped — the dropped FQN reflects how the call resolved: an interface-typed receiver (`Map m = ...; m.computeIfAbsent(...)`) drops `java.util.Map#computeIfAbsent`, a concrete one (`new HashMap<>()`) drops `java.util.HashMap#computeIfAbsent`. An interface is a valid target — write `@Approximate` on it exactly as on a concrete class, modelling whichever FQN the analyzer dropped. Reach the real object with `(TargetClass) (Object) this`, put functional-interface parameters behind `@ArgumentTypeContext`, and branch with `OpentaintNdUtil.nextBool()` so the analyzer walks both paths. Never leave a body empty. +Scaffold the approximation project with the same `--dependency` pins as the batch's test project (`.opentaint/test-projects//build.gradle.kts`): + +```bash +opentaint approximation init .opentaint/dataflow/ \ + --dependency "io.projectreactor:reactor-core:3.8.5" +``` + +When a model needs a library the project doesn't pin yet, add it to the project's `build.gradle.kts`. + +Create Java files under `.opentaint/dataflow//src/main/java` — one `@Approximate` class per target class. `@Approximate(TargetClass.class)` binds a model to exactly that class, so target the EXACT class the analyzer dropped — the dropped FQN reflects how the call resolved: an interface-typed receiver (`Map m = ...; m.computeIfAbsent(...)`) drops `java.util.Map#computeIfAbsent`, a concrete one (`new HashMap<>()`) drops `java.util.HashMap#computeIfAbsent`. An interface is a valid target — write `@Approximate` on it exactly as on a concrete class, modelling whichever FQN the analyzer dropped. Reach the real object with `(TargetClass) (Object) this`, put functional-interface parameters behind `@ArgumentTypeContext`, and branch with `OpentaintNdUtil.nextBool()` so the analyzer walks both paths. Never leave a body empty. ```java package com.example.approximations.batchpkg; // per-batch package (e.g. ...approximations.cn_hutool_001) — see the globally-unique rule below @@ -68,7 +77,7 @@ opentaint test approximation run .opentaint/test-compiled/ \ --dataflow-approximations .opentaint/dataflow/ ``` -`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI auto-compiles the `.java` sources against the analyzer JAR (for `@Approximate`, `OpentaintNdUtil`, `ArgumentTypeContext`) and the project's dependencies; if compilation fails it reports the errors and aborts before the tests. A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: +`test approximation run` applies its own bundled fixed source→sink rule automatically — you don't author or pass one. The CLI builds the approximation project first. if compilation fails it reports the errors and aborts before the tests. A positive sample is a `falseNegative` until the model propagates taint. Read the result with the bundled script — it prints the pass/fail counts and names each failing sample, so you don't parse the JSON by hand: ```bash uv run /scripts/check-test-result.py @@ -83,7 +92,7 @@ Fix by the verdict it reports: | Pattern | Usage | |---|---| -| `@Approximate(TargetClass.class)` | Link the approximation to its target class. Must be on the compile classpath (a project dependency or a JDK type) | +| `@Approximate(TargetClass.class)` | Link the approximation to its target class. Must be on the compile classpath (a pinned dependency or a JDK type) | | `(TargetClass) (Object) this` | Cast to reach the real object's methods | | `@ArgumentTypeContext` | On lambda / functional-interface parameters | | `OpentaintNdUtil.nextBool()` | Non-deterministic branch — the analyzer considers both paths | diff --git a/skills/create-test-project/references/java-approximation.md b/skills/create-test-project/references/java-approximation.md index e5152e14d2..8cee6ff26b 100644 --- a/skills/create-test-project/references/java-approximation.md +++ b/skills/create-test-project/references/java-approximation.md @@ -6,7 +6,7 @@ Language reference for a `dataflow` type, keyed to the body's steps. ### 1. Scaffold the project -`opentaint test approximation init ` scaffolds the Gradle build plus `Taint.java` and the fixed `approximation-rule.yaml` the harness applies. Pass each lib as a `--dependency` at its exact version from the app's dependency management (`.opentaint/project/sources/**/pom.xml`) verbatim — including timestamped snapshots — not a guess: this Gradle build is the approximation's own compile environment, so it must recompile from these pins even after the main project drops that dependency. +`opentaint test approximation init ` scaffolds the Gradle build plus `Taint.java` and the fixed `approximation-rule.yaml` the harness applies. Pass each lib as a `--dependency` at its exact version from the app's dependency management (`.opentaint/project/sources/**/pom.xml`) verbatim — including timestamped snapshots — not a guess. The approximation project scaffolded later takes the same pins, so the samples and the models they exercise compile against one version. ```bash opentaint test approximation init .opentaint/test-projects/ \ diff --git a/skills/run-scan/SKILL.md b/skills/run-scan/SKILL.md index 5425cb4cbf..221f4a67df 100644 --- a/skills/run-scan/SKILL.md +++ b/skills/run-scan/SKILL.md @@ -34,7 +34,7 @@ opentaint scan --project-model .opentaint/project \ - `--rule-id ` — restrict to specific executable rules (repeatable, one per input rule ID); referenced library rules remain available without listing their IDs separately. Omit to run all loaded rules - `--passthrough-approximations .opentaint/pass-through` — add when that directory exists: passThrough configs override built-ins at the rule level, a provided rule overriding a built-in only when it matches one -- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations (sources auto-compiled; pre-compiled `.class` dirs passed through as-is) +- `--dataflow-approximations .opentaint/dataflow` — add when that directory exists: code-based approximations, one project per batch Both approximation-dir flags walk their trees recursively; pass each parent directory once, not every package or batch separately.