diff --git a/core/opentaint-config/go-config/src/main/kotlin/org/opentaint/go/config/GoDefaultModelLoader.kt b/core/opentaint-config/go-config/src/main/kotlin/org/opentaint/go/config/GoDefaultModelLoader.kt new file mode 100644 index 0000000000..c34ca5b980 --- /dev/null +++ b/core/opentaint-config/go-config/src/main/kotlin/org/opentaint/go/config/GoDefaultModelLoader.kt @@ -0,0 +1,49 @@ +package org.opentaint.go.config + +import java.net.URI +import java.nio.file.FileSystems +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.copyToRecursively +import kotlin.io.path.createTempDirectory +import kotlin.streams.asSequence + +object GoDefaultModelLoader { + private const val MODEL_ROOT = "/model/go/dataflow" + + private val unpackedModelPaths: List by lazy { + loadModelPaths() + } + + fun modelPaths(): List = unpackedModelPaths + + private fun loadModelPaths(): List { + val resource = javaClass.getResource(MODEL_ROOT) ?: return emptyList() + return when (resource.protocol) { + "file" -> findModelModules(Path.of(resource.toURI())) + "jar" -> unpackJarModels(resource.toURI()) + else -> error("Unsupported bundled Go model resource protocol: ${resource.protocol}") + } + } + + @OptIn(ExperimentalPathApi::class) + private fun unpackJarModels(resource: URI): List { + val unpacked = createTempDirectory("opentaint-go-models") + FileSystems.newFileSystem(resource, emptyMap()).use { fs -> + fs.getPath(MODEL_ROOT).copyToRecursively( + unpacked, + followLinks = false, + overwrite = false, + ) + } + return findModelModules(unpacked) + } + + private fun findModelModules(root: Path): List = Files.list(root).use { paths -> + paths.asSequence() + .filter { Files.isDirectory(it) && Files.isRegularFile(it.resolve("go.mod")) } + .sortedBy { it.fileName.toString() } + .toList() + } +} diff --git a/core/opentaint-dataflow-core/opentaint-go-dataflow/src/main/kotlin/org/opentaint/dataflow/go/rules/GoTaintConfiguration.kt b/core/opentaint-dataflow-core/opentaint-go-dataflow/src/main/kotlin/org/opentaint/dataflow/go/rules/GoTaintConfiguration.kt index af624f2256..9dc7fb71e9 100644 --- a/core/opentaint-dataflow-core/opentaint-go-dataflow/src/main/kotlin/org/opentaint/dataflow/go/rules/GoTaintConfiguration.kt +++ b/core/opentaint-dataflow-core/opentaint-go-dataflow/src/main/kotlin/org/opentaint/dataflow/go/rules/GoTaintConfiguration.kt @@ -179,7 +179,12 @@ class GoTaintConfiguration : GoTaintRulesProvider { patternRules: List, ruleNameMatcher: R.() -> GoNameMatcher, ): List { - val direct = simpleByName[name].orEmpty() + val genericOriginName = name.substringBefore('[', name) + val direct = if (genericOriginName == name) { + simpleByName[name].orEmpty() + } else { + simpleByName[name].orEmpty() + simpleByName[genericOriginName].orEmpty() + } val patternMatches = patternRules.filter { it.ruleNameMatcher().matches(name) } if (patternMatches.isEmpty()) return direct if (direct.isEmpty()) return patternMatches diff --git a/core/opentaint-dataflow-core/opentaint-go-dataflow/src/test/kotlin/org/opentaint/dataflow/go/rules/GoTaintConfigurationTest.kt b/core/opentaint-dataflow-core/opentaint-go-dataflow/src/test/kotlin/org/opentaint/dataflow/go/rules/GoTaintConfigurationTest.kt new file mode 100644 index 0000000000..7f96c85384 --- /dev/null +++ b/core/opentaint-dataflow-core/opentaint-go-dataflow/src/test/kotlin/org/opentaint/dataflow/go/rules/GoTaintConfigurationTest.kt @@ -0,0 +1,44 @@ +package org.opentaint.dataflow.go.rules + +import org.opentaint.dataflow.configuration.go.serialized.GoNameMatcher +import org.opentaint.dataflow.configuration.go.serialized.GoSerializedPassAction +import org.opentaint.dataflow.configuration.go.serialized.GoSerializedRule +import org.opentaint.dataflow.configuration.go.serialized.GoSerializedTaintConfig +import org.opentaint.dataflow.configuration.jvm.serialized.PositionBase.Argument +import org.opentaint.dataflow.configuration.jvm.serialized.PositionBase.Result +import org.opentaint.dataflow.configuration.jvm.serialized.PositionBaseWithModifiers.BaseOnly +import org.opentaint.dataflow.go.GoFunctionSignature +import org.opentaint.ir.go.type.GoIRBasicType +import org.opentaint.ir.go.type.GoIRBasicTypeKind.STRING +import org.opentaint.ir.go.type.GoIRPointerType +import kotlin.test.Test +import kotlin.test.assertEquals + +class GoTaintConfigurationTest { + @Test + fun `simple rule name matches a generic function instance`() { + val stringType = GoIRBasicType(STRING) + val rule = GoSerializedRule.PassThrough( + pkg = GoNameMatcher.Simple("github.com/google/go-github/v89/github"), + function = GoNameMatcher.Simple("Ptr"), + copy = listOf( + GoSerializedPassAction( + from = BaseOnly(Argument(0)), + to = BaseOnly(Result), + ), + ), + ) + val configuration = GoTaintConfiguration().also { + it.loadConfig(GoSerializedTaintConfig(passThrough = listOf(rule))) + } + val signature = GoFunctionSignature( + name = "github.com/google/go-github/v89/github.Ptr[string]", + receiverType = null, + paramTypes = listOf(stringType), + resultType = GoIRPointerType(stringType), + pkgName = "github.com/google/go-github/v89/github", + ) + + assertEquals(1, configuration.passThroughForFunction(signature).size) + } +} diff --git a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRFunction.kt b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRFunction.kt index 1a8e1131a4..a1081a46bf 100644 --- a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRFunction.kt +++ b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRFunction.kt @@ -42,6 +42,8 @@ interface GoIRFunction: CommonMethod { // Generics val typeParams: List + val originFullName: String? get() = null + val typeArgs: List get() = emptyList() override val parameters: List get() = if (freeVars.isEmpty()) params diff --git a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRLookup.kt b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRLookup.kt new file mode 100644 index 0000000000..99fd46a056 --- /dev/null +++ b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRLookup.kt @@ -0,0 +1,30 @@ +package org.opentaint.ir.go.api + +import org.opentaint.ir.go.type.NamedTypeRef + +/** Symbol resolution in one program. References cache only their own lookup result. */ +interface GoIRLookup { + fun function(reference: GoIrFunctionReference): GoIRFunction? + fun namedType(reference: NamedTypeRef): GoIRNamedType? + fun global(reference: GoIRGlobalReference): GoIRGlobal? +} + +class GoIRProgramLookup(private val program: GoIRProgram) : GoIRLookup { + override fun function(reference: GoIrFunctionReference): GoIRFunction? = + program.findPackage(reference.pkg)?.functions?.firstOrNull { + it.name == reference.name && it.signature == reference.signature + } + override fun namedType(reference: NamedTypeRef): GoIRNamedType? = + program.findPackage(reference.refTypePkg)?.findNamedType(reference.refTypeName) + override fun global(reference: GoIRGlobalReference): GoIRGlobal? = + program.findPackage(reference.pkg)?.findGlobal(reference.name) +} + +data class GoIRGlobalReference(val pkg: String, val name: String) { + lateinit var program: GoIRProgram + val global: GoIRGlobal by lazy { + program.lookup.global(this) ?: error("Could not find global $pkg.$name") + } +} + +fun GoIRGlobal.reference(): GoIRGlobalReference = GoIRGlobalReference(pkg.importPath, name) diff --git a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRModelFunction.kt b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRModelFunction.kt new file mode 100644 index 0000000000..b7ffb06c56 --- /dev/null +++ b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRModelFunction.kt @@ -0,0 +1,9 @@ +package org.opentaint.ir.go.api + +/** A function supplied by a model, including helpers with no original declaration. */ +interface GoIRModelFunction : GoIRFunction + +/** A model implementation of an existing program function. */ +interface GoIRApproximatedFunction : GoIRModelFunction { + val original: GoIRFunction +} diff --git a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRProgram.kt b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRProgram.kt index fdc1f043aa..6a360f0d42 100644 --- a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRProgram.kt +++ b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRProgram.kt @@ -6,6 +6,8 @@ import org.opentaint.ir.go.type.GoIRAnonymousInterfaceType * Top-level entity: a loaded Go program with all its packages. */ interface GoIRProgram { + val lookup: GoIRLookup get() = GoIRProgramLookup(this) + val packages: Map // keyed by import path val anonymousInterfaces: Map diff --git a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRTransformer.kt b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRTransformer.kt new file mode 100644 index 0000000000..9ca8c3b440 --- /dev/null +++ b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIRTransformer.kt @@ -0,0 +1,7 @@ +package org.opentaint.ir.go.api + +/** A package lookup interceptor. Unhandled requests are passed to the next resolver. */ +interface GoIRTransformer { + fun resolvePackage(importPath: String, next: () -> GoIRPackage?): GoIRPackage? = next() + val additionalPackages: List get() = emptyList() +} diff --git a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIrFunctionReference.kt b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIrFunctionReference.kt index b006236caa..c9596b1eb1 100644 --- a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIrFunctionReference.kt +++ b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/api/GoIrFunctionReference.kt @@ -11,14 +11,11 @@ data class GoIrFunctionReference( lateinit var program: GoIRProgram - val function: GoIRFunction - get() { - val p = program.findPackage(pkg) - ?: error("Could not find package $pkg for $fullName") - // Match on name + signature because methods on different named - // types legitimately share short names (e.g. `Dog.Legs` and - // `Snake.Legs`); only the signature's receiver disambiguates them. - return p.functions.firstOrNull { it.name == name && it.signature == signature } - ?: error("Could not find function $fullName with signature $signature") - } + val function: GoIRFunction by lazy { + program.lookup.function(this) + ?: error("Could not find function $fullName with signature $signature") + } } + +fun GoIRFunction.reference(): GoIrFunctionReference = + GoIrFunctionReference(pkg?.importPath.orEmpty(), name, signature) diff --git a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/cfg/GoIRCallInfo.kt b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/cfg/GoIRCallInfo.kt index 10940e2780..d435696623 100644 --- a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/cfg/GoIRCallInfo.kt +++ b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/cfg/GoIRCallInfo.kt @@ -1,6 +1,6 @@ package org.opentaint.ir.go.cfg -import org.opentaint.ir.go.api.GoIRFunction +import org.opentaint.ir.go.api.GoIrFunctionReference import org.opentaint.ir.go.api.GoIRPosition import org.opentaint.ir.go.type.GoIRCallMode import org.opentaint.ir.go.type.GoIRChanDirection @@ -11,8 +11,9 @@ sealed interface GoIRCallTarget { val type: GoIRType val displayName: String - data class Function(val function: GoIRFunction) : GoIRCallTarget { - override val type: GoIRType get() = function.signature + data class Function(val lookup: GoIrFunctionReference) : GoIRCallTarget { + val function get() = lookup.function + override val type: GoIRType get() = lookup.signature override val displayName: String get() = function.fullName } diff --git a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/expr/GoIRExpr.kt b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/expr/GoIRExpr.kt index e6177eda30..43b3178f1b 100644 --- a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/expr/GoIRExpr.kt +++ b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/expr/GoIRExpr.kt @@ -139,9 +139,10 @@ data class GoIRTypeAssertExpr( data class GoIRMakeClosureExpr( override var type: GoIRType, - val fn: GoIRFunction, + val lookup: org.opentaint.ir.go.api.GoIrFunctionReference, val bindings: List, ) : GoIRExpr { + val fn get() = lookup.function override val operands: List get() = bindings override fun accept(visitor: GoIRExprVisitor): T = visitor.visitMakeClosure(this) override fun toString(): String = "makeclosure ${fn.name}[${bindings.joinToString(", ")}]" @@ -310,8 +311,9 @@ data class GoIRExtractExpr( data class GoIRGlobalValueExpr( override var type: GoIRType, - val global: GoIRGlobal, + val lookup: org.opentaint.ir.go.api.GoIRGlobalReference, ) : GoIRExpr { + val global get() = lookup.global override val operands: List get() = emptyList() override fun accept(visitor: GoIRExprVisitor): T = visitor.visitGlobalValue(this) override fun toString(): String = "global ${global.fullName}" @@ -319,8 +321,9 @@ data class GoIRGlobalValueExpr( data class GoIRFunctionValueExpr( override var type: GoIRType, - val function: GoIRFunction, + val lookup: org.opentaint.ir.go.api.GoIrFunctionReference, ) : GoIRExpr { + val function get() = lookup.function override val operands: List get() = emptyList() override fun accept(visitor: GoIRExprVisitor): T = visitor.visitFunctionValue(this) override fun toString(): String = "func ${function.fullName}" diff --git a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/inst/GoIRInst.kt b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/inst/GoIRInst.kt index 8b57957561..cd38e1d117 100644 --- a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/inst/GoIRInst.kt +++ b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/inst/GoIRInst.kt @@ -214,9 +214,10 @@ data class GoIRIndexStore( data class GoIRGlobalStore( override val location: GoInstLocation, - val global: GoIRGlobal, + val lookup: org.opentaint.ir.go.api.GoIRGlobalReference, val value: GoIRValue, ) : GoIRInst { + val global get() = lookup.global override val operands: List get() = listOf(value) override fun accept(visitor: GoIRInstVisitor): T = visitor.visitGlobalStore(this) override fun toString(): String = "global ${global.fullName} = $value" diff --git a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/type/GoIRType.kt b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/type/GoIRType.kt index e76f55adc6..a94c9d1f43 100644 --- a/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/type/GoIRType.kt +++ b/core/opentaint-ir/go/go-ir-api/src/main/kotlin/org/opentaint/ir/go/type/GoIRType.kt @@ -64,7 +64,7 @@ data class NamedTypeRef( val fullRefName: String get() = "$refTypePkg.$refTypeName" val namedType: GoIRNamedType - get() = program.findPackage(refTypePkg)?.findNamedType(refTypeName) + get() = program.lookup.namedType(this) ?: error("Can't find namedType for $fullRefName") override fun toString(): String = "ref($fullRefName)" diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRBodyContext.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRBodyContext.kt new file mode 100644 index 0000000000..be618934cd --- /dev/null +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRBodyContext.kt @@ -0,0 +1,12 @@ +package org.opentaint.ir.go.client + +import org.opentaint.ir.go.api.* +import org.opentaint.ir.go.type.* + +/** Symbol resolution policy applied while a body is decoded, independent of wire decoding. */ +open class GoIRBodyContext { + open fun type(type: GoIRType): GoIRType = type + open fun function(reference: GoIrFunctionReference): GoIrFunctionReference = reference + open fun global(reference: GoIRGlobalReference): GoIRGlobalReference = reference + open fun fieldIndex(type: GoIRType, index: Int, name: String): Int = index +} diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRClient.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRClient.kt index f79c449838..f6258c9c60 100644 --- a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRClient.kt +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRClient.kt @@ -41,6 +41,7 @@ class GoIRClient : AutoCloseable { .addAllProjectModulePaths(config.projectModules) .setInstantiateGenerics(config.instantiateGenerics) .setSanityCheck(config.sanityCheck) + .addAllModelDirs(config.modelDirs.map { it.toAbsolutePath().toString() }) .build() val totalStart = System.nanoTime() @@ -48,7 +49,7 @@ class GoIRClient : AutoCloseable { val deserializeStart = System.nanoTime() val program = try { val responses = stub.buildProgram(request) - deserializer.deserialize(responses) + GoIRModelLoader.apply(deserializer.deserialize(responses)) } catch (e: StatusRuntimeException) { // gRPC only says e.g. "UNAVAILABLE: Network closed" when the server // process dies mid-stream. Attach the server's exit state and stderr diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRDeserializer.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRDeserializer.kt index 0e664b7254..c4a86c87e4 100644 --- a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRDeserializer.kt +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRDeserializer.kt @@ -3,6 +3,9 @@ package org.opentaint.ir.go.client import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap import it.unimi.dsi.fastutil.ints.IntOpenHashSet +import org.opentaint.ir.go.api.GoIRFunction +import org.opentaint.ir.go.api.GoIRGlobalReference +import org.opentaint.ir.go.api.reference import org.opentaint.ir.go.api.GoIRField import org.opentaint.ir.go.api.GoIRFreeVar import org.opentaint.ir.go.api.GoIRInterfaceMethod @@ -10,6 +13,7 @@ import org.opentaint.ir.go.api.GoIRPackage import org.opentaint.ir.go.api.GoIRParameter import org.opentaint.ir.go.api.GoIRPosition import org.opentaint.ir.go.api.GoIRProgram +import org.opentaint.ir.go.api.GoIRTypeParamDecl import org.opentaint.ir.go.api.GoIrFunctionReference import org.opentaint.ir.go.cfg.GoIRCallInfo import org.opentaint.ir.go.cfg.GoIRCallTarget @@ -81,6 +85,7 @@ import org.opentaint.ir.go.proto.ProtoConstValue import org.opentaint.ir.go.proto.ProtoFunction import org.opentaint.ir.go.proto.ProtoFunctionBody import org.opentaint.ir.go.proto.ProtoInstruction +import org.opentaint.ir.go.proto.ProtoModelProgram import org.opentaint.ir.go.proto.ProtoNamedType import org.opentaint.ir.go.proto.ProtoNamedTypeKind import org.opentaint.ir.go.proto.ProtoPackage @@ -134,14 +139,17 @@ class GoIRDeserializer { var serverBuildTimeMs: Long = 0L private set - fun deserialize(responses: Iterator): GoIRProgram { + internal fun deserialize(responses: Iterator): DeserializedGoIRBuild { var program: ProtoProgram? = null + val models = mutableListOf() for (r in responses) { when (r.payloadCase) { BuildProgramResponse.PayloadCase.PROGRAM -> { program = r.program } + BuildProgramResponse.PayloadCase.MODEL_PROGRAM -> models += r.modelProgram + BuildProgramResponse.PayloadCase.SUMMARY -> serverBuildTimeMs = r.summary.buildTimeMs BuildProgramResponse.PayloadCase.ERROR -> handleError(r.error.message, r.error.fatal) BuildProgramResponse.PayloadCase.PAYLOAD_NOT_SET -> {} @@ -152,24 +160,42 @@ class GoIRDeserializer { } check(program != null) { "Unexpected IR build issue" } + return DeserializedGoIRBuild(deserializeProgram(program), models) + } + + internal data class DeserializedGoIRBuild( + val program: GoIRProgramImpl, + val models: List, + ) + + internal fun deserializeProgram( + serializedProgram: ProtoProgram, + ): GoIRProgramImpl { val resultPackages = hashMapOf() val anonymousInterfaces = Int2ObjectOpenHashMap() val result = GoIRProgramImpl(resultPackages, anonymousInterfaces) + deserializedProgram = result - val pkgInfo = deserializePackages(program.packagesList) - packages.filterNotNull().forEach { - resultPackages[it.importPath] = it + val pkgInfo = deserializePackages(serializedProgram.packagesList) + serializedProgram.packagesList.forEach { serializedPackage -> + val deserializedPackage = getPackage(serializedPackage.id) + val importPath = deserializedPackage.importPath + resultPackages.putIfAbsent(importPath, deserializedPackage) } - TypeDeserializationCtx(result, program.typesList, pkgInfo, anonymousInterfaces) + TypeDeserializationCtx(result, serializedProgram.typesList, pkgInfo, anonymousInterfaces) .deserializeTypes() FunctionDeserializationCtx(pkgInfo, result) .deserializeFunctions() - program.packagesList.forEach { deserializePackageMembers(it) } - program.functionBodiesList.forEach { deserializeFunctionBody(it) } + serializedProgram.packagesList.forEach { deserializePackageMembers(it) } + serializedProgram.functionBodiesList.forEach { body -> + getFunction(body.functionId).setBodyReader { owner, context -> + deserializeFunctionBody(body, owner, context) + } + } pkgInfo.namedTypes.forEach { resolveNamedTypeBindings(it) } return result @@ -189,8 +215,11 @@ class GoIRDeserializer { val functionPkgId: Int2IntOpenHashMap, ) - private fun deserializePackages(packagesList: List): PackageInfo { - val deserialized = arrayOfNulls(packagesList.size + 1).also { + private fun deserializePackages( + packagesList: List, + ): PackageInfo { + val maxPackageId = packagesList.maxOfOrNull { it.id } ?: 0 + val deserialized = arrayOfNulls(maxPackageId + 1).also { packages = it } @@ -351,6 +380,13 @@ class GoIRDeserializer { underlying = resolveType(nt.underlyingTypeId), kind = namedTypeKindFromProto(nt.kind), position = positionFromProto(nt.position), + typeParams = nt.typeParamsList.map { typeParam -> + GoIRTypeParamDecl( + typeParam.name, + typeParam.index, + resolveType(typeParam.constraintTypeId), + ) + }, ) for (fd in nt.fieldsList) { @@ -551,6 +587,18 @@ class GoIRDeserializer { declaredHasBody = pf.hasBody, parent = parentFunction, anonymousFunctions = anonymousFunctions, + typeParams = pf.typeParamsList.map { typeParam -> + GoIRTypeParamDecl( + typeParam.name, + typeParam.index, + getType(typeParam.constraintTypeId), + ) + }, + originFullName = pf.originFunctionId.takeIf { it != 0 } + ?.let { originId -> + functions.getOrNull(originId)?.fullName + }, + typeArgs = pf.typeArgIdsList.map(::getType), ) } } @@ -624,8 +672,11 @@ class GoIRDeserializer { // ─── Function Bodies ──────────────────────────────────────────── - private fun deserializeFunctionBody(fb: ProtoFunctionBody) { - val fn = getFunction(fb.functionId) + private fun deserializeFunctionBody( + fb: ProtoFunctionBody, + fn: GoIRFunction, + context: GoIRBodyContext, + ): GoIRBodyImpl { val phiEdgeSyntheticLoads = collectPhiEdgeSyntheticLoads(fb, fn.fullName) val phiEdgeSyntheticLoadsByPred = phiEdgeSyntheticLoads.groupBy { it.predInstProtoIndex } @@ -667,7 +718,7 @@ class GoIRDeserializer { block.instructionsList.forEach { pi -> if (pi.valueId <= 0) return@forEach val newIdx = protoToNew.get(pi.index) - val reg = GoIRRegister(getType(pi.typeId), newIdx, pi.name) + val reg = GoIRRegister(context.type(getType(pi.typeId)), newIdx, pi.name) lazyValueMap.register(pi.valueId, reg) } } @@ -677,7 +728,7 @@ class GoIRDeserializer { for (load in phiEdgeSyntheticLoads) { val predBlockIndex = protoInstBlockIndices[load.predInstProtoIndex] ?: error("No predecessor block for phi edge pred inst ${load.predInstProtoIndex} in ${fn.fullName}") - val (reg, inst) = createSyntheticLoad(load.value, body, predBlockIndex, load.index) + val (reg, inst) = createSyntheticLoad(load.value, body, predBlockIndex, load.index, context) phiEdgeSyntheticValues[load.key] = reg phiEdgeSyntheticInstructions[load.key] = inst } @@ -685,7 +736,7 @@ class GoIRDeserializer { val ctx = InstContext(body, blockIdx = 0, fn = fn, valueMap = lazyValueMap, instructions = mutableListOf(), protoToNew = protoToNew, blockStartProtoToNew = blockStartProtoToNew, - phiEdgeSyntheticValues = phiEdgeSyntheticValues) + phiEdgeSyntheticValues = phiEdgeSyntheticValues, context = context) for ((blockIdx, pb) in fb.blocksList.withIndex()) { val block = blocks[blockIdx] ctx.blockIdx = blockIdx @@ -714,7 +765,7 @@ class GoIRDeserializer { } specializeStores(blocks) - fn.setBody(body) + return body } /** @@ -759,7 +810,7 @@ class GoIRDeserializer { pi: ProtoInstruction, ): GoIRInst { fun ref(vr: ProtoValueRef): GoIRValue = valueRefFromProto(vr, this) - fun type(id: Int): GoIRType = getType(id) + fun type(id: Int): GoIRType = context.type(getType(id)) fun translate(protoIdx: Int): GoIRInstRef = GoIRInstRef( protoToNew.get(protoIdx) ) @@ -807,7 +858,11 @@ class GoIRDeserializer { ) ProtoInstruction.InstCase.MAKE_CLOSURE -> { val closureFn = getFunction(pi.makeClosure.fnId) - assign(GoIRMakeClosureExpr(exprType(), closureFn, pi.makeClosure.bindingsList.map { ref(it) })) + assign(GoIRMakeClosureExpr( + exprType(), + context.function(functionReference(closureFn)), + pi.makeClosure.bindingsList.map { ref(it) }, + )) } ProtoInstruction.InstCase.MAKE_MAP -> assign( GoIRMakeMapExpr(exprType(), if (pi.makeMap.hasReserve) ref(pi.makeMap.reserve) else null) @@ -819,10 +874,18 @@ class GoIRDeserializer { GoIRMakeSliceExpr(exprType(), ref(pi.makeSlice.len), ref(pi.makeSlice.cap)) ) ProtoInstruction.InstCase.FIELD_ADDR -> assign( - GoIRFieldAddrExpr(exprType(), ref(pi.fieldAddr.x), pi.fieldAddr.fieldIndex, pi.fieldAddr.fieldName) + GoIRFieldAddrExpr( + exprType(), ref(pi.fieldAddr.x), + context.fieldIndex(type(pi.fieldAddr.x.typeId), pi.fieldAddr.fieldIndex, pi.fieldAddr.fieldName), + pi.fieldAddr.fieldName, + ) ) ProtoInstruction.InstCase.FIELD -> assign( - GoIRFieldExpr(exprType(), ref(pi.field.x), pi.field.fieldIndex, pi.field.fieldName) + GoIRFieldExpr( + exprType(), ref(pi.field.x), + context.fieldIndex(type(pi.field.x.typeId), pi.field.fieldIndex, pi.field.fieldName), + pi.field.fieldName, + ) ) ProtoInstruction.InstCase.INDEX_ADDR -> assign( GoIRIndexAddrExpr(exprType(), ref(pi.indexAddr.x), ref(pi.indexAddr.index)) @@ -922,7 +985,7 @@ class GoIRDeserializer { when (addr.refCase) { ProtoValueRef.RefCase.GLOBAL_ID -> GoIRGlobalStore( loc, - resolveGlobal(addr.globalId), + context.global(globalReference(addr.globalId)), ref(pi.store.`val`), ) ProtoValueRef.RefCase.FUNCTION_ID, @@ -962,7 +1025,7 @@ class GoIRDeserializer { // ─── Value references ─────────────────────────────────────────── private fun valueRefFromProto(vr: ProtoValueRef, ctx: InstContext): GoIRValue { - val type = getType(vr.typeId) + val type = ctx.context.type(getType(vr.typeId)) return when (vr.refCase) { ProtoValueRef.RefCase.INST_VALUE_ID -> ctx.valueMap[vr.instValueId] @@ -991,15 +1054,15 @@ class GoIRDeserializer { receiver = if (ci.hasReceiver()) valueRefFromProto(ci.receiver, ctx) else null, methodName = ci.methodName.ifEmpty { null }, args = ci.argsList.map { valueRefFromProto(it, ctx) }, - resultType = getType(ci.resultTypeId), + resultType = ctx.context.type(getType(ci.resultTypeId)), ) } private fun callTargetFromProto(vr: ProtoValueRef, ctx: InstContext): GoIRCallTarget { - val type = getType(vr.typeId) + val type = ctx.context.type(getType(vr.typeId)) return when (vr.refCase) { ProtoValueRef.RefCase.FUNCTION_ID -> - GoIRCallTarget.Function(getFunction(vr.functionId)) + GoIRCallTarget.Function(ctx.context.function(functionReference(getFunction(vr.functionId)))) ProtoValueRef.RefCase.BUILTIN_NAME -> GoIRCallTarget.Builtin(vr.builtinName, type) ProtoValueRef.RefCase.GLOBAL_ID -> @@ -1009,34 +1072,42 @@ class GoIRDeserializer { } } + private fun functionReference(function: GoIRFunction): GoIrFunctionReference = + function.reference().also { it.program = deserializedProgram } + + private fun globalReference(id: Int): GoIRGlobalReference = + resolveGlobal(id).reference().also { it.program = deserializedProgram } + + private lateinit var deserializedProgram: GoIRProgram + private fun resolveGlobal(globalId: Int): GoIRGlobalImpl = globalsById.get(globalId) ?: error("Global was not found for $globalId") private fun synthLoadGlobal(globalId: Int, typeId: Int, ctx: InstContext): GoIRRegister { val idx = ctx.take() - val (reg, inst) = createSyntheticLoadGlobal(globalId, typeId, ctx.body, ctx.blockIdx, idx) + val (reg, inst) = createSyntheticLoadGlobal(globalId, typeId, ctx.body, ctx.blockIdx, idx, ctx.context) ctx.instructions.add(inst) return reg } private fun synthLoadFunction(functionId: Int, typeId: Int, ctx: InstContext): GoIRRegister { val idx = ctx.take() - val (reg, inst) = createSyntheticLoadFunction(functionId, typeId, ctx.body, ctx.blockIdx, idx) + val (reg, inst) = createSyntheticLoadFunction(functionId, typeId, ctx.body, ctx.blockIdx, idx, ctx.context) ctx.instructions.add(inst) return reg } private fun synthLoadBuiltin(name: String, typeId: Int, ctx: InstContext): GoIRRegister { val idx = ctx.take() - val (reg, inst) = createSyntheticLoadBuiltin(name, typeId, ctx.body, ctx.blockIdx, idx) + val (reg, inst) = createSyntheticLoadBuiltin(name, typeId, ctx.body, ctx.blockIdx, idx, ctx.context) ctx.instructions.add(inst) return reg } private fun synthLoadFreeVar(freeVarIndex: Int, typeId: Int, ctx: InstContext): GoIRRegister { val idx = ctx.take() - val (reg, inst) = createSyntheticLoadFreeVar(freeVarIndex, typeId, ctx.body, ctx.blockIdx, idx) + val (reg, inst) = createSyntheticLoadFreeVar(freeVarIndex, typeId, ctx.body, ctx.blockIdx, idx, ctx.context) ctx.instructions.add(inst) return reg } @@ -1046,11 +1117,12 @@ class GoIRDeserializer { body: GoIRBodyImpl, blockIndex: Int, index: Int, + context: GoIRBodyContext, ): Pair = when (vr.refCase) { - ProtoValueRef.RefCase.GLOBAL_ID -> createSyntheticLoadGlobal(vr.globalId, vr.typeId, body, blockIndex, index) - ProtoValueRef.RefCase.FUNCTION_ID -> createSyntheticLoadFunction(vr.functionId, vr.typeId, body, blockIndex, index) - ProtoValueRef.RefCase.BUILTIN_NAME -> createSyntheticLoadBuiltin(vr.builtinName, vr.typeId, body, blockIndex, index) - ProtoValueRef.RefCase.FREE_VAR_INDEX -> createSyntheticLoadFreeVar(vr.freeVarIndex, vr.typeId, body, blockIndex, index) + ProtoValueRef.RefCase.GLOBAL_ID -> createSyntheticLoadGlobal(vr.globalId, vr.typeId, body, blockIndex, index, context) + ProtoValueRef.RefCase.FUNCTION_ID -> createSyntheticLoadFunction(vr.functionId, vr.typeId, body, blockIndex, index, context) + ProtoValueRef.RefCase.BUILTIN_NAME -> createSyntheticLoadBuiltin(vr.builtinName, vr.typeId, body, blockIndex, index, context) + ProtoValueRef.RefCase.FREE_VAR_INDEX -> createSyntheticLoadFreeVar(vr.freeVarIndex, vr.typeId, body, blockIndex, index, context) else -> error("${vr.refCase} is not a synthetic-load ref") } @@ -1060,9 +1132,10 @@ class GoIRDeserializer { body: GoIRBodyImpl, blockIndex: Int, index: Int, + context: GoIRBodyContext, ): Pair { - val type = getType(typeId) - val global = resolveGlobal(globalId) + val type = context.type(getType(typeId)) + val global = context.global(globalReference(globalId)) val reg = GoIRRegister(type, index, "_g$index") val expr = GoIRGlobalValueExpr(type, global) val loc = GoInstLocation(body, index, blockIndex, null) @@ -1075,9 +1148,10 @@ class GoIRDeserializer { body: GoIRBodyImpl, blockIndex: Int, index: Int, + context: GoIRBodyContext, ): Pair { - val type = getType(typeId) - val func = getFunction(functionId) + val type = context.type(getType(typeId)) + val func = context.function(functionReference(getFunction(functionId))) val reg = GoIRRegister(type, index, "_f$index") val expr = GoIRFunctionValueExpr(type, func) val loc = GoInstLocation(body, index, blockIndex, null) @@ -1090,8 +1164,9 @@ class GoIRDeserializer { body: GoIRBodyImpl, blockIndex: Int, index: Int, + context: GoIRBodyContext, ): Pair { - val type = getType(typeId) + val type = context.type(getType(typeId)) val reg = GoIRRegister(type, index, "_b$index") val expr = GoIRBuiltinValueExpr(type, name) val loc = GoInstLocation(body, index, blockIndex, null) @@ -1104,8 +1179,9 @@ class GoIRDeserializer { body: GoIRBodyImpl, blockIndex: Int, index: Int, + context: GoIRBodyContext, ): Pair { - val type = getType(typeId) + val type = context.type(getType(typeId)) val freeVarName = body.function.freeVars[freeVarIndex].name val reg = GoIRRegister(type, index, "_v$index") val expr = GoIRFreeVarValueExpr(type, freeVarIndex, freeVarName) @@ -1360,12 +1436,13 @@ private data class PhiEdgeSyntheticLoad( private class InstContext( val body: GoIRBodyImpl, var blockIdx: Int, - val fn: GoIRFunctionImpl, + val fn: GoIRFunction, val valueMap: ValueMap, var instructions: MutableList, val protoToNew: Int2IntOpenHashMap, val blockStartProtoToNew: Int2IntOpenHashMap, val phiEdgeSyntheticValues: Map, + val context: GoIRBodyContext, ) { var nextIndex: Int = 0 fun take(): Int = nextIndex++ diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRLoadConfig.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRLoadConfig.kt index 62dc61bcd7..cbf69ff2eb 100644 --- a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRLoadConfig.kt +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRLoadConfig.kt @@ -1,5 +1,7 @@ package org.opentaint.ir.go.client +import java.nio.file.Path + enum class GoIRLoadMode { FULL, PROJECT } data class GoIRLoadConfig( @@ -8,4 +10,5 @@ data class GoIRLoadConfig( val projectModules: Set = emptySet(), val instantiateGenerics: Boolean = true, val sanityCheck: Boolean = true, + val modelDirs: List = emptyList(), ) diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRModelLoader.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRModelLoader.kt new file mode 100644 index 0000000000..d063838041 --- /dev/null +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRModelLoader.kt @@ -0,0 +1,20 @@ +package org.opentaint.ir.go.client + +import org.opentaint.ir.go.api.GoIRProgram + +/** Model policy and transformer installation, separate from wire deserialization. */ +internal object GoIRModelLoader { + fun apply(build: GoIRDeserializer.DeserializedGoIRBuild): GoIRProgram { + if (build.models.isEmpty()) return build.program + val definitions = build.models.map { model -> + val program = GoIRDeserializer().deserializeProgram(model.program) + try { + GoIRModelDefinition.create(model.source, program) + } catch (error: IllegalArgumentException) { + throw IllegalArgumentException("invalid Go model ${model.source}: ${error.message}", error) + } + } + build.program.install(listOf(GoIRModelTransformer(build.program, definitions))) + return build.program + } +} diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRModelSpecializer.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRModelSpecializer.kt new file mode 100644 index 0000000000..cc81aba59a --- /dev/null +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRModelSpecializer.kt @@ -0,0 +1,176 @@ +package org.opentaint.ir.go.client + +import org.opentaint.ir.go.api.GoIRFunction +import org.opentaint.ir.go.api.GoIRGlobalReference +import org.opentaint.ir.go.api.GoIRPackage +import org.opentaint.ir.go.api.GoIRParameter +import org.opentaint.ir.go.api.GoIRFreeVar +import org.opentaint.ir.go.api.GoIRProgram +import org.opentaint.ir.go.api.GoIRTypeParamDecl +import org.opentaint.ir.go.api.GoIrFunctionReference +import org.opentaint.ir.go.impl.GoIRFunctionImpl +import org.opentaint.ir.go.type.* +import java.util.IdentityHashMap + +/** Substitutes concrete type arguments and closure lookups while reading a generic model body. */ +internal class GoIRModelSpecializer( + private val program: GoIRProgram, + private val pkg: GoIRPackage, + private val model: GoIRBodyContext, + private val symbol: (String) -> String, +) { + /** Returns the specialized root followed by its specialized anonymous functions. */ + fun specialize(modelRoot: GoIRFunction, target: GoIRFunction): List { + require(modelRoot.bodyAvailable) { "Go model function ${modelRoot.fullName} has no body" } + require(modelRoot.typeParams.size == target.typeArgs.size) { + "Go generic instance ${target.fullName} has ${target.typeArgs.size} type arguments, " + + "want ${modelRoot.typeParams.size}" + } + val typeArguments = modelRoot.typeParams.associate { parameter -> + parameter.index to target.typeArgs[parameter.index] + } + val specializer = TypeSpecializer(typeArguments) + fun specializeType(type: GoIRType) = specializer(model.type(type)) + val sourceFunctions = collectAnonymousFunctions(modelRoot) + val suffix = target.name.removePrefix(modelRoot.name).ifEmpty { + "#opentaint-model-${target.fullName.hashCode().toUInt()}" + } + val functionMap = IdentityHashMap() + val anonymousRefs = IdentityHashMap>() + + fun reference(function: GoIRFunction): GoIrFunctionReference = + GoIrFunctionReference(pkg.importPath, function.name, function.signature) + .also { it.program = program } + + val rootRefs = mutableListOf() + val root = modelFunction( + original = target, + name = target.name, + fullName = target.fullName, + pkg = pkg, + signature = target.signature, + params = target.params, + freeVars = target.freeVars, + position = modelRoot.position, + isMethod = target.isMethod, + isPointerReceiver = target.isPointerReceiver, + isExported = target.isExported, + syntheticKind = modelRoot.syntheticKind, + hasBody = true, + parent = target.parent, + anonymousFunctions = rootRefs, + typeParams = target.typeParams, + originFullName = target.originFullName, + typeArgs = target.typeArgs, + ) + functionMap[modelRoot] = root + anonymousRefs[modelRoot] = rootRefs + + sourceFunctions.forEach { source -> + val refs = mutableListOf() + val parent = source.parent?.function?.let(functionMap::get) + ?: error("Go model anonymous function ${source.fullName} has no copied parent") + val copied = modelFunction( + original = null, + name = source.name + suffix, + fullName = symbol(source.fullName) + suffix, + pkg = pkg, + signature = specializeType(source.signature) as GoIRFuncType, + params = source.params.map { GoIRParameter(it.name, specializeType(it.type), it.index) }, + freeVars = source.freeVars.map { GoIRFreeVar(it.name, specializeType(it.type), it.index) }, + position = source.position, + isMethod = source.isMethod, + isPointerReceiver = source.isPointerReceiver, + isExported = source.isExported, + syntheticKind = source.syntheticKind, + hasBody = true, + parent = reference(parent), + anonymousFunctions = refs, + typeParams = source.typeParams.map { + GoIRTypeParamDecl(it.name, it.index, specializeType(it.constraint)) + }, + originFullName = null, + typeArgs = emptyList(), + ) + functionMap[source] = copied + anonymousRefs[source] = refs + } + + (listOf(modelRoot) + sourceFunctions).forEach { source -> + val refs = anonymousRefs.getValue(source) + refs += source.anonymousFunctions.map { child -> + functionMap[child.function]?.let(::reference) ?: model.function(child) + } + } + + val context = object : GoIRBodyContext() { + override fun type(type: GoIRType) = specializeType(type) + override fun function(reference: GoIrFunctionReference): GoIrFunctionReference = + reference.program.lookup.function(reference)?.let(functionMap::get)?.let(::reference) + ?: model.function(reference) + override fun global(reference: GoIRGlobalReference) = model.global(reference) + override fun fieldIndex(type: GoIRType, index: Int, name: String) = + model.fieldIndex(type, index, name) + } + (listOf(modelRoot) + sourceFunctions).forEach { source -> + functionMap.getValue(source).setBodyReader { owner, _ -> + (source as GoIRFunctionImpl).readBody(owner, context) + ?: error("Go model ${source.fullName} has no body") + } + } + return listOf(root) + sourceFunctions.map(functionMap::getValue) + } + + private fun collectAnonymousFunctions(root: GoIRFunction): List { + val result = mutableListOf() + val pending = ArrayDeque(root.anonymousFunctions.map { it.function }) + while (pending.isNotEmpty()) { + val function = pending.removeFirst() + if (result.any { it === function }) continue + result += function + pending += function.anonymousFunctions.map { it.function } + } + return result + } + +} + +private class TypeSpecializer( + private val arguments: Map, +) : (GoIRType) -> GoIRType { + override fun invoke(type: GoIRType): GoIRType { + val result = when (type) { + is GoIRTypeParamType -> arguments[type.paramIndex] ?: type + is GoIRTypeParamRef -> arguments[type.paramIndex] ?: type + is GoIRPointerType -> GoIRPointerType(invoke(type.elem)) + is GoIRArrayType -> GoIRArrayType(invoke(type.elem), type.length) + is GoIRSliceType -> GoIRSliceType(invoke(type.elem)) + is GoIRMapType -> GoIRMapType(invoke(type.key), invoke(type.value)) + is GoIRChanType -> GoIRChanType(invoke(type.elem), type.direction) + is GoIRStructType -> GoIRStructType( + type.fields.map { GoIRStructField(it.name, invoke(it.type), it.isEmbedded, it.tag) }, + type.structTypeRef, + ) + is GoIRNamedTypeRef -> GoIRNamedTypeRef(type.typeRef, type.typeArgs.map(::invoke)) + is GoIRNamedInterfaceType -> GoIRNamedInterfaceType( + type.methods.map { GoIRInterfaceMethodSig(it.name, invoke(it.signature) as GoIRFuncType) }, + type.embeds.map(::invoke), + type.interfaceTypeRef, + ) + is GoIRAnonymousInterfaceType -> GoIRAnonymousInterfaceType( + type.id, + type.methods.map { GoIRInterfaceMethodSig(it.name, invoke(it.signature) as GoIRFuncType) }, + type.embeds.map(::invoke), + ) + is GoIRFuncType -> GoIRFuncType( + type.params.map(::invoke), + type.results.map(::invoke), + type.isVariadic, + type.recv?.let(::invoke), + ) + is GoIRTupleType -> GoIRTupleType(type.elements.map(::invoke)) + else -> type + } + return result + } +} diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRModelTransformer.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRModelTransformer.kt new file mode 100644 index 0000000000..5261460852 --- /dev/null +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/client/GoIRModelTransformer.kt @@ -0,0 +1,464 @@ +package org.opentaint.ir.go.client + +import org.opentaint.ir.go.api.GoIRConst +import org.opentaint.ir.go.api.GoIRFunction +import org.opentaint.ir.go.api.GoIRGlobal +import org.opentaint.ir.go.api.GoIRGlobalReference +import org.opentaint.ir.go.api.GoIRNamedType +import org.opentaint.ir.go.api.GoIRPackage +import org.opentaint.ir.go.api.GoIRProgram +import org.opentaint.ir.go.api.GoIRTransformer +import org.opentaint.ir.go.api.GoIrFunctionReference +import org.opentaint.ir.go.impl.GoIRApproximatedFunctionImpl +import org.opentaint.ir.go.impl.GoIRConstImpl +import org.opentaint.ir.go.impl.GoIRFunctionImpl +import org.opentaint.ir.go.impl.GoIRGlobalImpl +import org.opentaint.ir.go.impl.GoIRModelFunctionImpl +import org.opentaint.ir.go.impl.GoIRNamedTypeImpl +import org.opentaint.ir.go.impl.GoIRPackageImpl +import org.opentaint.ir.go.impl.GoIRProgramImpl +import org.opentaint.ir.go.type.* + +/** A deserialized model module and the target path of each of its model-addressed (`opentaint/`) packages. */ +internal class GoIRModelDefinition private constructor( + val source: String, + val program: GoIRProgram, + val targets: Map, +) { + companion object { + fun create(source: String, program: GoIRProgram): GoIRModelDefinition { + val targets = program.packages.values + .filterNot { it.isDependency || it.isStdlib || it.importPath == UNIVERSE_PACKAGE_PATH } + .associate { pkg -> + require(pkg.importPath.startsWith(GO_MODEL_PREFIX)) { + "Go model package \"${pkg.importPath}\" is not model-addressed, " + + "its import path must be ${GO_MODEL_PREFIX}" + } + val target = pkg.importPath.removePrefix(GO_MODEL_PREFIX) + require(target.isNotEmpty()) { + "Go model package \"${pkg.importPath}\" has an empty target package" + } + pkg.importPath to target + } + require(targets.isNotEmpty()) { + "Go model contains no model-addressed packages, expected at least one package " + + "under $GO_MODEL_PREFIX" + } + return GoIRModelDefinition(source, program, targets) + } + } +} + +/** Applies partial Go models by intercepting the lookup of every modeled package. */ +internal class GoIRModelTransformer( + private val program: GoIRProgramImpl, + definitions: List, +) : GoIRTransformer { + private val scopesByTarget = linkedMapOf() + private val modelDependencies = linkedMapOf() + + init { + val definitionsByTarget = linkedMapOf() + definitions.forEach { definition -> + definition.targets.values.forEach { target -> + val previous = definitionsByTarget.putIfAbsent(target, definition) + require(previous == null) { + "Go package \"$target\" is modeled more than once " + + "(${previous?.source} and ${definition.source})" + } + } + } + // Bundled modules can describe dependencies that this project does not use. + definitionsByTarget.entries.removeIf { (target, _) -> program.declaredPackage(target) == null } + val scopes = definitionsByTarget.values.distinct().associateWith { ModelScope(it) } + definitionsByTarget.forEach { (target, definition) -> scopesByTarget[target] = scopes.getValue(definition) } + scopesByTarget.forEach { (target, scope) -> scope.validatePackage(target) } + scopes.keys.forEach { definition -> + definition.program.packages.forEach { (path, pkg) -> + if (path !in definition.targets && program.declaredPackage(path) == null) { + modelDependencies.putIfAbsent(path, pkg) + } + } + } + } + + override val additionalPackages: List get() = modelDependencies.values.toList() + + override fun resolvePackage(importPath: String, next: () -> GoIRPackage?): GoIRPackage? { + val scope = scopesByTarget[importPath] ?: return next() + val original = next() ?: return null + return scope.createPackage(original) + } + + /** Resolves the symbols of one model module into the program it models. */ + private inner class ModelScope(val definition: GoIRModelDefinition) { + private val modelPaths = definition.targets.keys.sortedByDescending(String::length) + private val modelPackages = definition.targets.entries.associate { (model, target) -> + target to definition.program.findPackage(model)!! + } + + // Anonymous interface IDs are local to each wire program, so model definitions get fresh program IDs. + private val interfaceIds: Map = run { + var nextId = (program.anonymousInterfaces.keys.maxOrNull() ?: -1) + 1 + definition.program.anonymousInterfaces.keys.associateWith { nextId++ } + } + + init { + definition.program.anonymousInterfaces.values.forEach { + program.addAnonymousInterface(type(it) as GoIRAnonymousInterfaceType) + } + } + + private fun path(importPath: String) = definition.targets[importPath] ?: importPath + + private fun symbol(fullName: String) = modelPaths.fold(fullName) { result, modelPath -> + result.replace(modelPath, definition.targets.getValue(modelPath)) + } + + private fun reference(ref: NamedTypeRef) = + ref.copy(refTypePkg = path(ref.refTypePkg)).also { it.program = program } + + fun type(value: GoIRType): GoIRType = when (value) { + is GoIRTypeParamType -> value.copy(constraint = type(value.constraint)) + is GoIRTypeParamRef -> value + is GoIRPointerType -> GoIRPointerType(type(value.elem)) + is GoIRArrayType -> GoIRArrayType(type(value.elem), value.length) + is GoIRSliceType -> GoIRSliceType(type(value.elem)) + is GoIRMapType -> GoIRMapType(type(value.key), type(value.value)) + is GoIRChanType -> GoIRChanType(type(value.elem), value.direction) + is GoIRStructType -> GoIRStructType( + value.fields.map { GoIRStructField(it.name, type(it.type), it.isEmbedded, it.tag) }, + value.structTypeRef?.let(::reference), + ) + is GoIRNamedTypeRef -> GoIRNamedTypeRef(reference(value.typeRef), value.typeArgs.map(::type)) + is GoIRNamedInterfaceType -> GoIRNamedInterfaceType( + value.methods.map { GoIRInterfaceMethodSig(it.name, type(it.signature) as GoIRFuncType) }, + value.embeds.map(::type), + reference(value.interfaceTypeRef), + ) + is GoIRAnonymousInterfaceTypeRef -> GoIRAnonymousInterfaceTypeRef(interfaceIds.getValue(value.id)) + .also { it.program = program } + is GoIRAnonymousInterfaceType -> GoIRAnonymousInterfaceType( + interfaceIds.getValue(value.id), + value.methods.map { GoIRInterfaceMethodSig(it.name, type(it.signature) as GoIRFuncType) }, + value.embeds.map(::type), + ) + is GoIRFuncType -> GoIRFuncType( + value.params.map(::type), + value.results.map(::type), + value.isVariadic, + value.recv?.let(::type), + ) + is GoIRTupleType -> GoIRTupleType(value.elements.map(::type)) + else -> value + } + + /** Decodes model bodies against the modeled program: target paths, program IDs and merged field layouts. */ + val context = object : GoIRBodyContext() { + override fun type(type: GoIRType) = this@ModelScope.type(type) + override fun function(reference: GoIrFunctionReference) = reference + .copy(pkg = path(reference.pkg), signature = type(reference.signature) as GoIRFuncType) + .also { it.program = program } + override fun global(reference: GoIRGlobalReference) = + reference.copy(pkg = path(reference.pkg)).also { it.program = program } + override fun fieldIndex(type: GoIRType, index: Int, name: String): Int { + fun named(type: GoIRType): GoIRNamedType? = when (type) { + is GoIRPointerType -> named(type.elem) + is GoIRNamedTypeRef -> type.namedType + is GoIRStructType -> type.namedType + else -> null + } + return named(type)?.fields?.firstOrNull { it.name == name }?.index ?: index + } + } + + private fun function(model: GoIRFunction, pkg: GoIRPackage, original: GoIRFunction?): GoIRFunction = + modelFunction( + original, model.name, symbol(model.fullName), pkg, type(model.signature) as GoIRFuncType, + model.params.map { it.copy(type = type(it.type)) }, + model.freeVars.map { it.copy(type = type(it.type)) }, + model.position, model.isMethod, model.isPointerReceiver, model.isExported, model.syntheticKind, + model.hasBody, model.parent?.let(context::function), model.anonymousFunctions.map(context::function), + model.typeParams.map { it.copy(constraint = type(it.constraint)) }, + model.originFullName?.let(::symbol), model.typeArgs.map(::type), + ).also { function -> + if (!model.bodyAvailable) return@also + function.setBodyReader { owner, _ -> + (model as GoIRFunctionImpl).readBody(owner, context) + ?: error("Go model ${model.fullName} has no body") + } + } + + fun validatePackage(target: String) { + val original = program.declaredPackage(target)!! + val model = modelPackages[target] + ?: throw IllegalArgumentException( + "invalid Go model ${definition.source}: model package \"$target\" was not found", + ) + require(model.name == original.name) { + "Go model package \"opentaint/$target\" has package name " + + "\"${model.name}\", want \"${original.name}\" (the modeled package name)" + } + + val originalFunctions = original.functions.associateBy { it.fullName } + val initializers = initializerFunctions(model) + model.functions.forEach { function -> + if (function in initializers || function.parent != null) return@forEach + val targetFunction = originalFunctions[symbol(function.fullName)] + if (targetFunction != null) { + require(sameSignature(function, targetFunction)) { + "Go model function ${symbol(function.fullName)} has a different signature from its target" + } + } + } + model.namedTypes.forEach { modelType -> + val targetType = original.findNamedType(modelType.name) ?: return@forEach + val targetFields = targetType.fields.associateBy { it.name } + modelType.fields.forEach { modelField -> + val targetField = targetFields[modelField.name] ?: return@forEach + require(sameType(type(modelField.type), targetField.type)) { + "invalid Go model ${definition.source}: field ${targetType.fullName}.${modelField.name} " + + "has a different type from its target" + } + } + } + } + + fun createPackage(original: GoIRPackage): GoIRPackage { + val model = modelPackages.getValue(original.importPath) + val initializers = initializerFunctions(model) + val modelFunctions = model.functions.filterNot { it in initializers } + val modelByName = modelFunctions + .filter { it.parent == null } + .associateBy { symbol(it.fullName) } + val used = mutableSetOf() + val specializedAnonymousFunctions = mutableListOf() + val result = GoIRPackageImpl(original.importPath, original.name, original.isStdlib, original.isDependency) + val specializer = GoIRModelSpecializer(program, result, context, ::symbol) + original.functions.forEach { target -> + val replacement = modelByName[target.fullName] + val genericModel = target.originFullName?.let(modelByName::get) + result.addFunction( + if (replacement != null && replacement.bodyAvailable && sameSignature(replacement, target)) { + used += replacement + function(replacement, result, target) + } else if (genericModel != null && genericModel.bodyAvailable) { + val specialized = specializer.specialize(genericModel, target) + specializedAnonymousFunctions += specialized.drop(1) + specialized.first() + } else { + target + }, + ) + } + specializedAnonymousFunctions.forEach(result::addFunction) + modelFunctions.filter { it !in used }.forEach { result.addFunction(function(it, result, null)) } + + val functionsByName = buildMap { result.functions.forEach { putIfAbsent(it.fullName, it) } } + val modelTypes = model.namedTypes.associateBy { it.name } + original.namedTypes.forEach { target -> + result.addNamedType(modelTypes[target.name]?.let { namedType(target, it, result, functionsByName) } ?: target) + } + model.namedTypes.filter { original.findNamedType(it.name) == null }.forEach { + result.addNamedType(namedType(null, it, result, functionsByName)) + } + original.globals.forEach(result::addGlobal) + model.globals.filter { result.findGlobal(it.name) == null }.forEach { result.addGlobal(global(it, result)) } + original.constants.forEach(result::addConst) + model.constants.filter { result.findConstant(it.name) == null }.forEach { result.addConst(constant(it, result)) } + result._imports += original.imports + result._imports += model.imports.mapNotNull { modelDependencies[it.importPath] } + .filter { dependency -> result._imports.none { it.importPath == dependency.importPath } } + result.initFunction = original.initFunction + return result + } + + private fun global(model: GoIRGlobal, pkg: GoIRPackage) = + GoIRGlobalImpl(model.name, symbol(model.fullName), type(model.type), pkg, model.isExported, model.position) + + private fun constant(model: GoIRConst, pkg: GoIRPackage) = GoIRConstImpl( + model.name, symbol(model.fullName), type(model.type), model.value, pkg, model.isExported, model.position, + ) + + private fun namedType( + original: GoIRNamedType?, + model: GoIRNamedType, + pkg: GoIRPackage, + functionsByName: Map, + ): GoIRNamedType { + val originalFields = original?.fields.orEmpty() + val originalNames = originalFields.mapTo(mutableSetOf()) { it.name } + val fields = originalFields + model.fields.filter { it.name !in originalNames } + .mapIndexed { index, field -> field.copy(type = type(field.type), index = originalFields.size + index) } + val underlying = when (val declared = original?.underlying ?: type(model.underlying)) { + is GoIRStructType -> GoIRStructType( + fields.map { GoIRStructField(it.name, it.type, it.isEmbedded, it.tag) }, + declared.structTypeRef, + ) + else -> declared + } + val result = GoIRNamedTypeImpl( + model.name, symbol(model.fullName), pkg, underlying, + original?.kind ?: model.kind, + original?.position ?: model.position, + original?.typeParams ?: model.typeParams.map { it.copy(constraint = type(it.constraint)) }, + ) + fields.mapTo(result._fields) { it.copy(enclosingType = result) } + result._methods += methods(original?.methods.orEmpty(), model.methods, functionsByName) + result._pointerMethods += methods(original?.pointerMethods.orEmpty(), model.pointerMethods, functionsByName) + result._interfaceMethods += original?.interfaceMethods ?: model.interfaceMethods.map { + it.copy(signature = type(it.signature) as GoIRFuncType, enclosingInterface = result) + } + result._embeddedInterfaces += original?.embeddedInterfaces ?: model.embeddedInterfaces.map { embedded -> + pkg.findNamedType(embedded.name)?.takeIf { path(embedded.pkg.importPath) == pkg.importPath } ?: embedded + } + return result + } + + private fun methods( + targetMethods: List, + modelMethods: List, + functionsByName: Map, + ): List { + val targetNames = targetMethods.mapTo(mutableSetOf()) { it.fullName } + return targetMethods.map { functionsByName[it.fullName] ?: it } + + modelMethods.map { symbol(it.fullName) }.filter { it !in targetNames }.mapNotNull(functionsByName::get) + } + + private fun sameSignature(model: GoIRFunction, target: GoIRFunction): Boolean = + sameType(type(model.signature), target.signature) && + model.isMethod == target.isMethod && + model.isPointerReceiver == target.isPointerReceiver + } + + private fun sameType(first: GoIRType, second: GoIRType): Boolean = + sameType(first, second, mutableListOf()) + + private fun sameType( + first: GoIRType, + second: GoIRType, + active: MutableList>, + ): Boolean { + if (first === second) return true + if (active.any { (left, right) -> left === first && right === second }) return true + active += first to second + return when { + first is GoIRBasicType && second is GoIRBasicType -> first.kind == second.kind + first is GoIRPointerType && second is GoIRPointerType -> + sameType(first.elem, second.elem, active) + first is GoIRArrayType && second is GoIRArrayType -> + first.length == second.length && sameType(first.elem, second.elem, active) + first is GoIRSliceType && second is GoIRSliceType -> + sameType(first.elem, second.elem, active) + first is GoIRMapType && second is GoIRMapType -> + sameType(first.key, second.key, active) && sameType(first.value, second.value, active) + first is GoIRChanType && second is GoIRChanType -> + first.direction == second.direction && sameType(first.elem, second.elem, active) + first is GoIRStructType && second is GoIRStructType -> + first.structTypeRef?.fullRefName == second.structTypeRef?.fullRefName && + first.fields.size == second.fields.size && + first.fields.zip(second.fields).all { (left, right) -> + left.name == right.name && left.isEmbedded == right.isEmbedded && + left.tag == right.tag && sameType(left.type, right.type, active) + } + first is GoIRNamedTypeRef && second is GoIRNamedTypeRef -> + first.typeRef.fullRefName == second.typeRef.fullRefName && + sameTypes(first.typeArgs, second.typeArgs, active) + first is GoIRFuncType && second is GoIRFuncType -> + first.isVariadic == second.isVariadic && + nullableTypeEqual(first.recv, second.recv, active) && + sameTypes(first.params, second.params, active) && + sameTypes(first.results, second.results, active) + first is GoIRTupleType && second is GoIRTupleType -> + sameTypes(first.elements, second.elements, active) + first is GoIRTypeParamType && second is GoIRTypeParamType -> + first.paramIndex == second.paramIndex && first.name == second.name && + sameType(first.constraint, second.constraint, active) + first is GoIRTypeParamRef && second is GoIRTypeParamRef -> + first.paramIndex == second.paramIndex && first.name == second.name + first is GoIRAnonymousInterfaceTypeRef && second is GoIRAnonymousInterfaceTypeRef -> + sameAnonymousInterface(first.interfaceType, second.interfaceType, active) + first is GoIRAnonymousInterfaceType && second is GoIRAnonymousInterfaceType -> + sameAnonymousInterface(first, second, active) + first is GoIRNamedInterfaceType && second is GoIRNamedInterfaceType -> + first.interfaceTypeRef.fullRefName == second.interfaceTypeRef.fullRefName + else -> false + } + } + + private fun sameAnonymousInterface( + first: GoIRAnonymousInterfaceType, + second: GoIRAnonymousInterfaceType, + active: MutableList>, + ): Boolean = first.methods.size == second.methods.size && + first.methods.zip(second.methods).all { (left, right) -> + left.name == right.name && sameType(left.signature, right.signature, active) + } && sameTypes(first.embeds, second.embeds, active) + + private fun sameTypes( + first: List, + second: List, + active: MutableList>, + ): Boolean = first.size == second.size && + first.zip(second).all { (left, right) -> sameType(left, right, active) } + + private fun nullableTypeEqual( + first: GoIRType?, + second: GoIRType?, + active: MutableList>, + ): Boolean = when { + first == null -> second == null + second == null -> false + else -> sameType(first, second, active) + } + + private fun initializerFunctions(pkg: GoIRPackage): Set { + val result = pkg.functions + .filterTo(mutableSetOf()) { it.name == "init" || it.name.startsWith("init#") } + do { + val oldSize = result.size + pkg.functions.forEach { function -> + val parentName = function.parent?.fullName + if (parentName != null && result.any { it.fullName == parentName }) { + result += function + } + } + } while (result.size != oldSize) + return result + } +} + + +internal fun modelFunction( + original: GoIRFunction?, + name: String, + fullName: String, + pkg: GoIRPackage?, + signature: GoIRFuncType, + params: List, + freeVars: List, + position: org.opentaint.ir.go.api.GoIRPosition?, + isMethod: Boolean, + isPointerReceiver: Boolean, + isExported: Boolean, + syntheticKind: String?, + hasBody: Boolean, + parent: GoIrFunctionReference?, + anonymousFunctions: List, + typeParams: List, + originFullName: String?, + typeArgs: List, +): GoIRFunctionImpl = if (original == null) { + GoIRModelFunctionImpl( + name, fullName, pkg, signature, params, freeVars, position, isMethod, isPointerReceiver, isExported, + syntheticKind, hasBody, parent, anonymousFunctions, typeParams, originFullName, typeArgs, + ) +} else { + GoIRApproximatedFunctionImpl( + original, name, fullName, pkg, signature, params, freeVars, position, isMethod, isPointerReceiver, + isExported, syntheticKind, hasBody, parent, anonymousFunctions, typeParams, originFullName, typeArgs, + ) +} + +private const val GO_MODEL_PREFIX = "opentaint/" +private const val UNIVERSE_PACKAGE_PATH = "" diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRFunctionIdentity.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRFunctionIdentity.kt new file mode 100644 index 0000000000..52cced30c8 --- /dev/null +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRFunctionIdentity.kt @@ -0,0 +1,18 @@ +package org.opentaint.ir.go.impl + +import org.opentaint.ir.go.api.GoIRFunction +import org.opentaint.ir.go.api.GoIRModelFunction + +/** + * Function identity shared by every [GoIRFunction] implementation in this module. + * A model keeps the name of the function it replaces, so model-ness is part of the identity: + * otherwise bodies and instruction locations of a model and its original would compare equal. + */ +internal fun GoIRFunction.functionEquals(other: Any?): Boolean { + if (this === other) return true + if (other !is GoIRFunction) return false + return fullName == other.fullName && (this is GoIRModelFunction) == (other is GoIRModelFunction) +} + +internal fun GoIRFunction.functionHashCode(): Int = + 31 * fullName.hashCode() + if (this is GoIRModelFunction) 1 else 0 diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRFunctionImpl.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRFunctionImpl.kt index 305e9f26ce..80012bea0d 100644 --- a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRFunctionImpl.kt +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRFunctionImpl.kt @@ -1,5 +1,6 @@ package org.opentaint.ir.go.impl +import org.opentaint.ir.go.client.GoIRBodyContext import org.opentaint.ir.go.api.GoIRBody import org.opentaint.ir.go.api.GoIRBodyUnavailableException import org.opentaint.ir.go.api.GoIRFreeVar @@ -10,8 +11,9 @@ import org.opentaint.ir.go.api.GoIRPosition import org.opentaint.ir.go.api.GoIRTypeParamDecl import org.opentaint.ir.go.api.GoIrFunctionReference import org.opentaint.ir.go.type.GoIRFuncType +import org.opentaint.ir.go.type.GoIRType -class GoIRFunctionImpl( +open class GoIRFunctionImpl( override val name: String, override val fullName: String, override val pkg: GoIRPackage?, @@ -27,21 +29,35 @@ class GoIRFunctionImpl( private val declaredHasBody: Boolean = false, override val parent: GoIrFunctionReference?, override val anonymousFunctions: List, + override val typeParams: List = emptyList(), + override val originFullName: String? = null, + override val typeArgs: List = emptyList(), ) : GoIRFunction { + private var _body: GoIRBody? = null + private var reader: ((GoIRFunction, GoIRBodyContext) -> GoIRBody)? = null + + internal fun setBodyReader(reader: (GoIRFunction, GoIRBodyContext) -> GoIRBody) { + this.reader = reader + } + + /** Decodes this body for [owner], resolving its symbols through [context]. */ + internal fun readBody(owner: GoIRFunction, context: GoIRBodyContext): GoIRBody? = + reader?.invoke(owner, context) ?: _body + + private val resolvedBody by lazy { reader?.invoke(this, GoIRBodyContext()) } override val body: GoIRBody? get() { val b = _body if (b != null) return b + resolvedBody?.let { return it } if (!declaredHasBody) return null throw GoIRBodyUnavailableException(fullName) } override val hasBody: Boolean get() = declaredHasBody - override val bodyAvailable: Boolean get() = _body != null - - override val typeParams: List = emptyList() // TODO: implement + override val bodyAvailable: Boolean get() = _body != null || reader != null fun setBody(body: GoIRBody) { this._body = body @@ -49,11 +65,7 @@ class GoIRFunctionImpl( override fun toString(): String = "GoIRFunction($fullName)" - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other !is GoIRFunction) return false - return fullName == other.fullName - } + override fun equals(other: Any?): Boolean = functionEquals(other) - override fun hashCode(): Int = fullName.hashCode() + override fun hashCode(): Int = functionHashCode() } diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRGlobalImpl.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRGlobalImpl.kt index 2bd886354f..c670986ec3 100644 --- a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRGlobalImpl.kt +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRGlobalImpl.kt @@ -12,6 +12,7 @@ class GoIRGlobalImpl( override val isExported: Boolean, override val position: GoIRPosition?, ) : GoIRGlobal { + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is GoIRGlobal) return false @@ -30,6 +31,7 @@ class GoIRConstImpl( override val isExported: Boolean, override val position: GoIRPosition?, ) : GoIRConst { + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is GoIRConst) return false diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRModelFunctionImpl.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRModelFunctionImpl.kt new file mode 100644 index 0000000000..617ef4e064 --- /dev/null +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRModelFunctionImpl.kt @@ -0,0 +1,62 @@ +package org.opentaint.ir.go.impl + +import org.opentaint.ir.go.api.GoIRApproximatedFunction +import org.opentaint.ir.go.api.GoIRFreeVar +import org.opentaint.ir.go.api.GoIRFunction +import org.opentaint.ir.go.api.GoIRModelFunction +import org.opentaint.ir.go.api.GoIRPackage +import org.opentaint.ir.go.api.GoIRParameter +import org.opentaint.ir.go.api.GoIRPosition +import org.opentaint.ir.go.api.GoIRTypeParamDecl +import org.opentaint.ir.go.api.GoIrFunctionReference +import org.opentaint.ir.go.type.GoIRFuncType +import org.opentaint.ir.go.type.GoIRType + +open class GoIRModelFunctionImpl( + name: String, + fullName: String, + pkg: GoIRPackage?, + signature: GoIRFuncType, + params: List, + freeVars: List, + position: GoIRPosition?, + isMethod: Boolean, + isPointerReceiver: Boolean, + isExported: Boolean, + syntheticKind: String?, + declaredHasBody: Boolean, + parent: GoIrFunctionReference?, + anonymousFunctions: List, + typeParams: List, + originFullName: String?, + typeArgs: List, +) : GoIRFunctionImpl( + name, fullName, pkg, signature, params, freeVars, position, isMethod, isPointerReceiver, isExported, + isSynthetic = true, syntheticKind, declaredHasBody, parent, anonymousFunctions, typeParams, originFullName, typeArgs, +), GoIRModelFunction { + override fun toString(): String = "GoIRModelFunction($fullName)" +} + +class GoIRApproximatedFunctionImpl( + override val original: GoIRFunction, + name: String, + fullName: String, + pkg: GoIRPackage?, + signature: GoIRFuncType, + params: List, + freeVars: List, + position: GoIRPosition?, + isMethod: Boolean, + isPointerReceiver: Boolean, + isExported: Boolean, + syntheticKind: String?, + declaredHasBody: Boolean, + parent: GoIrFunctionReference?, + anonymousFunctions: List, + typeParams: List, + originFullName: String?, + typeArgs: List, +) : GoIRModelFunctionImpl( + name, fullName, pkg, signature, params, freeVars, position, isMethod, isPointerReceiver, isExported, + syntheticKind, declaredHasBody, parent, anonymousFunctions, typeParams, originFullName, typeArgs, +), GoIRApproximatedFunction diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRNamedTypeImpl.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRNamedTypeImpl.kt index 32d69d1edf..34ce609648 100644 --- a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRNamedTypeImpl.kt +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRNamedTypeImpl.kt @@ -17,7 +17,9 @@ class GoIRNamedTypeImpl( override val underlying: GoIRType, override val kind: GoIRNamedTypeKind, override val position: GoIRPosition?, + override val typeParams: List = emptyList(), ) : GoIRNamedType { + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is GoIRNamedType) return false @@ -37,7 +39,5 @@ class GoIRNamedTypeImpl( override val pointerMethods: List get() = _pointerMethods override val interfaceMethods: List get() = _interfaceMethods override val embeddedInterfaces: List get() = _embeddedInterfaces - override val typeParams: List = emptyList() - override fun toString(): String = "GoIRNamedType($fullName)" } diff --git a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRProgramImpl.kt b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRProgramImpl.kt index 36b45b5921..1711fa63e7 100644 --- a/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRProgramImpl.kt +++ b/core/opentaint-ir/go/go-ir-client/src/main/kotlin/org/opentaint/ir/go/impl/GoIRProgramImpl.kt @@ -4,9 +4,53 @@ import org.opentaint.ir.go.api.* import org.opentaint.ir.go.type.GoIRAnonymousInterfaceType class GoIRProgramImpl( - override val packages: Map, - override val anonymousInterfaces: Map, + private val declaredPackages: Map, + private val declaredInterfaces: MutableMap, ) : GoIRProgram { + private var transformers: List = emptyList() + + private val resolvedPackages: Lazy>> = lazy { + val additional = transformers.flatMap { it.additionalPackages }.associateBy { it.importPath } + (declaredPackages.keys + additional.keys).associateWith { importPath -> + lazy { + fun resolve(index: Int): GoIRPackage? = if (index < 0) { + declaredPackages[importPath] ?: additional[importPath] + } else { + transformers[index].resolvePackage(importPath) { resolve(index - 1) } + } + resolve(transformers.lastIndex) + } + } + } + + /** Installs package lookup interceptors. Must be called before the first package lookup. */ + fun install(transformers: List) { + check(!resolvedPackages.isInitialized()) { "Go IR transformers must be installed before package lookup" } + this.transformers = transformers + } + + /** The package as it was deserialized, before any transformer. */ + internal fun declaredPackage(importPath: String): GoIRPackage? = declaredPackages[importPath] + + internal fun addAnonymousInterface(type: GoIRAnonymousInterfaceType) { + check(declaredInterfaces.putIfAbsent(type.id, type) == null) { "Anonymous interface ${type.id} already exists" } + } + + override val anonymousInterfaces: Map get() = declaredInterfaces + + override val packages: Map by lazy { + if (transformers.isEmpty()) return@lazy declaredPackages + object : AbstractMap() { + override fun get(key: String): GoIRPackage? = resolvedPackages.value[key]?.value + override fun containsKey(key: String): Boolean = get(key) != null + override val entries: Set> by lazy { + resolvedPackages.value.entries + .mapNotNull { (importPath, pkg) -> pkg.value?.let { java.util.AbstractMap.SimpleEntry(importPath, it) } } + .toSet() + } + } + } + override fun findPackage(importPath: String) = packages[importPath] override fun allFunctions(): List = diff --git a/core/opentaint-ir/go/go-ir-tests/src/test/kotlin/org/opentaint/ir/go/test/features/ModelTests.kt b/core/opentaint-ir/go/go-ir-tests/src/test/kotlin/org/opentaint/ir/go/test/features/ModelTests.kt new file mode 100644 index 0000000000..a9c4337a6e --- /dev/null +++ b/core/opentaint-ir/go/go-ir-tests/src/test/kotlin/org/opentaint/ir/go/test/features/ModelTests.kt @@ -0,0 +1,887 @@ +package org.opentaint.ir.go.test.features + +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.opentaint.ir.go.api.GoIRApproximatedFunction +import org.opentaint.ir.go.api.GoIRModelFunction +import org.opentaint.ir.go.api.GoIRFunction +import org.opentaint.ir.go.api.GoIRPackage +import org.opentaint.ir.go.api.GoIRTransformer +import org.opentaint.ir.go.impl.GoIRPackageImpl +import org.opentaint.ir.go.impl.GoIRProgramImpl +import org.opentaint.ir.go.cfg.GoIRCallTarget +import org.opentaint.ir.go.client.GoIRLoadConfig +import org.opentaint.ir.go.client.GoIRLoadMode +import org.opentaint.ir.go.expr.GoIRFunctionValueExpr +import org.opentaint.ir.go.expr.GoIRFieldAddrExpr +import org.opentaint.ir.go.expr.GoIRMakeClosureExpr +import org.opentaint.ir.go.ext.findExpressions +import org.opentaint.ir.go.ext.findInstructions +import org.opentaint.ir.go.inst.GoIRCall +import org.opentaint.ir.go.inst.GoIRGo +import org.opentaint.ir.go.inst.GoIRDefer +import org.opentaint.ir.go.inst.GoIRGlobalStore +import org.opentaint.ir.go.inst.GoIRReturn +import org.opentaint.ir.go.test.GoIRTestBuilder +import org.opentaint.ir.go.test.GoIRTestExtension +import org.opentaint.ir.go.type.GoIRAnonymousInterfaceType +import org.opentaint.ir.go.type.GoIRNamedTypeRef +import org.opentaint.ir.go.type.GoIRPointerType +import org.opentaint.ir.go.type.GoIRStructType +import org.opentaint.ir.go.value.GoIRConstValue +import org.opentaint.ir.go.value.GoIRConstantValue +import org.opentaint.ir.go.value.GoIRParameterValue +import java.nio.file.Files +import java.nio.file.Path + +@ExtendWith(GoIRTestExtension::class) +class ModelTests { + @Test + fun `model body replaces dependency body after client deserialization`(builder: GoIRTestBuilder) { + val project = Files.createTempDirectory("goir-model-project") + project.resolve("go.mod").write("module example.com/app\ngo 1.22\n") + project.resolve("main.go").write( + """ + package main + import "strings" + func Upper(value string) string { return strings.ToUpper(value) } + func Start(value string) { go strings.ToUpper(value); defer strings.ToUpper(value) } + func Function() func(string) string { return strings.ToUpper } + func Closure(value string) func() string { return func() string { return strings.ToUpper(value) } } + """.trimIndent(), + ) + + val model = Files.createTempDirectory("goir-model") + model.resolve("go.mod").write("module opentaint\ngo 1.25\n") + val modelPackage = model.resolve("strings") + Files.createDirectories(modelPackage) + modelPackage.resolve("model.go").write( + "package strings\nfunc ToUpper(value string) string { return value }\n", + ) + + val program = builder.buildFromDir( + project, + GoIRLoadConfig(mode = GoIRLoadMode.PROJECT, modelDirs = listOf(model)), + ) + + assertThat(program.findPackage("opentaint/strings")).isNull() + val function = program.findPackage("strings")!!.functions.single { it.name == "ToUpper" } + val caller = program.findPackage("example.com/app")!!.functions.single { it.name == "Upper" } + val originalTarget = caller.findInstructions().single().call.target as GoIRCallTarget.Function + assertThat(function.bodyAvailable).isTrue() + assertThat(originalTarget.function).isSameAs(function) + assertThat(function).isInstanceOf(GoIRApproximatedFunction::class.java) + val returned = (function.body!!.blocks.single().terminator as GoIRReturn).results.single() as GoIRParameterValue + assertThat(returned.paramIndex).isZero() + assertThat(returned.type).isEqualTo(function.params.single().type) + val pkg = caller.pkg!! + val start = pkg.functions.single { it.name == "Start" } + val goTarget = start.findInstructions().single().call.target as GoIRCallTarget.Function + val deferTarget = start.findInstructions().single().call.target as GoIRCallTarget.Function + assertThat(goTarget.function).isSameAs(function) + assertThat(deferTarget.function).isSameAs(function) + val value = pkg.functions.single { it.name == "Function" }.findExpressions().single() + assertThat(value.function).isSameAs(function) + val closure = pkg.functions.single { it.name == "Closure" }.anonymousFunctions.single().function + val closureTarget = closure.findInstructions().single().call.target as GoIRCallTarget.Function + assertThat(closureTarget.function).isSameAs(function) + assertThat(function.body!!.function).isSameAs(function) + assertThat(caller.body!!.instructions).allMatch { it.location.method === caller } + val original = (function as GoIRApproximatedFunction).original + assertThat(original).isNotSameAs(function) + assertThat(original.fullName).isEqualTo(function.fullName) + } + + @Test + fun `lazy model lookup keeps the original Go IR visible`(builder: GoIRTestBuilder) { + val project = project( + "example.com/original", + "package original\n" + + "func Identity(value string) string { return \"original\" }\n", + ) + val model = model( + "example.com/original", + "package original\n" + + "func helper(value string) string { return value }\n" + + "func Identity(value string) string { return helper(value) }\n", + ) + + val effectiveProgram = builder.buildFromDir(project, modelConfig(model)) + val effectivePackage = effectiveProgram.findPackage("example.com/original")!! + val effectiveIdentity = effectivePackage.functions.single { it.name == "Identity" } + assertThat(effectiveIdentity).isInstanceOf(GoIRApproximatedFunction::class.java) + val originalIdentity = (effectiveIdentity as GoIRApproximatedFunction).original + val originalPackage = originalIdentity.pkg!! + + assertThat(originalPackage).isNotSameAs(effectivePackage) + assertThat(effectiveIdentity.pkg).isSameAs(effectivePackage) + assertThat(originalPackage.functions.single { it.name == "Identity" }).isSameAs(originalIdentity) + assertThat(effectiveIdentity.findInstructions()) + .anyMatch { it.call.target?.displayName == "example.com/original.helper" } + assertThat(originalIdentity.isSynthetic).isFalse() + assertThat(originalIdentity.syntheticKind).isNull() + assertThat(originalIdentity.body!!.instructions) + .noneMatch { instruction -> instruction is GoIRCall } + assertThat(originalPackage.functions.map { it.name }).doesNotContain("helper") + } + + @Test + fun `model packages keep identity consistent with the original Go IR`(builder: GoIRTestBuilder) { + val project = project( + "example.com/original", + "package original\n" + + "type Box struct { Value string }\n" + + "var Global string\n" + + "const Constant = 1\n" + + "func Identity(value string) string { return \"original\" }\n" + + "func Untouched() {}\n", + ) + val model = model( + "example.com/original", + "package original\n" + + "type Box struct { Value string; extra string }\n" + + "func Identity(value string) string { return value }\n", + ) + + val effectiveProgram = builder.buildFromDir(project, modelConfig(model)) + val effective = effectiveProgram.findPackage("example.com/original")!! + val original = (effective.functions.single { it.name == "Identity" } as GoIRApproximatedFunction).original.pkg!! + fun assertSameIdentity(first: Any, second: Any) { + assertThat(first).isEqualTo(second) + assertThat(second).isEqualTo(first) + assertThat(first.hashCode()).isEqualTo(second.hashCode()) + } + + assertSameIdentity(effective, original) + assertSameIdentity(effective.findNamedType("Box")!!, original.findNamedType("Box")!!) + assertSameIdentity(effective.findGlobal("Global")!!, original.findGlobal("Global")!!) + assertSameIdentity(effective.findConstant("Constant")!!, original.findConstant("Constant")!!) + assertSameIdentity( + effective.functions.single { it.name == "Untouched" }, + original.functions.single { it.name == "Untouched" }, + ) + assertThat(effective.findNamedType("Box")!!.fields.first()) + .isEqualTo(original.findNamedType("Box")!!.fields.first()) + + val approximation = effective.functions.single { it.name == "Identity" } + val originalIdentity = original.functions.single { it.name == "Identity" } + assertThat(approximation).isNotEqualTo(originalIdentity) + assertThat(originalIdentity).isNotEqualTo(approximation) + assertThat(approximation.body).isNotEqualTo(originalIdentity.body) + assertThat(setOf(approximation, originalIdentity)).hasSize(2) + } + + @Test + fun `partial model adds fields and methods after client deserialization`(builder: GoIRTestBuilder) { + val project = Files.createTempDirectory("goir-model-fields-project") + project.resolve("go.mod").write("module example.com/fields\ngo 1.22\n") + project.resolve("fields.go").write( + """ + package fields + type Box struct { Original string } + func (b *Box) Put(value string) { b.Original = "original" } + func (b *Box) Keep(value string) string { return "original" } + """.trimIndent(), + ) + + val model = Files.createTempDirectory("goir-model-fields") + model.resolve("go.mod").write("module opentaint\ngo 1.25\n") + val modelPackage = model.resolve("example.com/fields") + Files.createDirectories(modelPackage) + modelPackage.resolve("model.go").write( + """ + package fields + type Box struct { Shadow string } + func (b *Box) Put(value string) { b.Shadow = value } + func (b *Box) Helper(value string) string { + b.Shadow = value + return b.Shadow + } + """.trimIndent(), + ) + + val program = builder.buildFromDir( + project, + GoIRLoadConfig(mode = GoIRLoadMode.PROJECT, modelDirs = listOf(model)), + ) + + val targetPackage = program.findPackage("example.com/fields")!! + val box = targetPackage.findNamedType("Box")!! + assertThat(box.fields.map { it.name to it.index }) + .containsExactly("Original" to 0, "Shadow" to 1) + assertThat((box.underlying as GoIRStructType).fields.map { it.name }) + .containsExactly("Original", "Shadow") + assertThat(box.pointerMethods.map { it.name }).contains("Put", "Keep", "Helper") + + val put = box.pointerMethods.single { it.name == "Put" } + val receiverType = (put.params.first().type as GoIRPointerType).elem as GoIRNamedTypeRef + assertThat(receiverType.namedType).isSameAs(box) + assertThat(receiverType.namedType.pointerMethods.single { it.name == "Put" }).isSameAs(put) + assertThat((put as GoIRApproximatedFunction).original.pkg!!.findNamedType("Box")!!.fields.map { it.name }) + .containsExactly("Original") + val shadowAccess = put.findExpressions().single { it.fieldName == "Shadow" } + assertThat(shadowAccess.fieldIndex).isEqualTo(1) + assertThat(box.pointerMethods.single { it.name == "Keep" }.bodyAvailable).isTrue() + } + + @Test + fun `model package name must match target package name`(builder: GoIRTestBuilder) { + val project = project( + "example.com/app", + "package main\nimport \"strings\"\nfunc Use(value string) string { return strings.ToUpper(value) }\n", + ) + val model = model( + "strings", + "package modelstrings\nfunc ToUpper(value string) string { return value }\n", + ) + + assertThatThrownBy { + builder.buildFromDir(project, modelConfig(model)) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("package name \"modelstrings\", want \"strings\"") + } + + @Test + fun `model function signature must match target function`(builder: GoIRTestBuilder) { + val project = project( + "example.com/app", + "package main\nimport \"strings\"\nfunc Use(value string) string { return strings.ToUpper(value) }\n", + ) + val model = model( + "strings", + "package strings\nfunc ToUpper(value int) int { return value }\n", + ) + + assertThatThrownBy { + builder.buildFromDir(project, modelConfig(model)) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("strings.ToUpper has a different signature") + } + + @Test + fun `one package cannot have two partial model modules`(builder: GoIRTestBuilder) { + val project = project( + "example.com/app", + "package main\nimport \"strings\"\n" + + "func Upper(value string) string { return strings.ToUpper(value) }\n" + + "func Lower(value string) string { return strings.ToLower(value) }\n", + ) + val firstModel = model( + "strings", + "package strings\nfunc ToUpper(value string) string { return value }\n", + ) + val secondModel = model( + "strings", + "package strings\nfunc ToLower(value string) string { return value }\n", + ) + + assertThatThrownBy { + builder.buildFromDir( + project, + GoIRLoadConfig( + mode = GoIRLoadMode.PROJECT, + modelDirs = listOf(firstModel, secondModel), + ), + ) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("Go package \"strings\" is modeled more than once") + } + + @Test + fun `one package cannot have two generic partial model modules`(builder: GoIRTestBuilder) { + val project = project( + "example.com/app", + """ + package main + import "slices" + func Delete(values []string) []string { + return slices.DeleteFunc(values, func(string) bool { return false }) + } + func Contains(values []string) bool { + return slices.ContainsFunc(values, func(string) bool { return true }) + } + """.trimIndent(), + ) + val firstModel = model( + "slices", + "package slices\nfunc DeleteFunc[S ~[]E, E any](values S, _ func(E) bool) S { return values }\n", + ) + val secondModel = model( + "slices", + "package slices\nfunc ContainsFunc[S ~[]E, E any](_ S, predicate func(E) bool) bool { " + + "var value E; return predicate(value) }\n", + ) + + assertThatThrownBy { + builder.buildFromDir( + project, + GoIRLoadConfig( + mode = GoIRLoadMode.PROJECT, + modelDirs = listOf(firstModel, secondModel), + ), + ) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("Go package \"slices\" is modeled more than once") + } + + @Test + fun `generic model instance body uses concrete target types`(builder: GoIRTestBuilder) { + val project = project( + "example.com/app", + "package main\nimport \"slices\"\nfunc Clone(values []string) []string { return slices.Clone(values) }\n", + ) + val model = model( + "slices", + "package slices\nfunc Clone[S ~[]E, E any](values S) S { return values }\n", + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val clone = program.findPackage("slices")!!.functions.single { + it.fullName.startsWith("slices.Clone[") + } + val returned = (clone.body!!.blocks.single().terminator as GoIRReturn) + .results.single() as GoIRParameterValue + + assertThat(clone.typeParams.map { it.name }).containsExactly("S", "E") + assertThat(returned.type).isEqualTo(clone.params.single().type) + assertThat(returned.type).isEqualTo(clone.signature.results.single()) + } + + @Test + fun `generic model specializes compound body types`(builder: GoIRTestBuilder) { + val project = project( + "example.com/genericmodel", + """ + package genericmodel + type Box[T any] struct { Value T } + func Flow[T any](value T) ( + *T, []T, map[string]T, chan T, + [1]T, struct { Value T }, func(T) T, Box[T], + ) { + var array [1]T + var record struct { Value T } + var callback func(T) T + var box Box[T] + return nil, nil, nil, nil, array, record, callback, box + } + func Use(value string) { Flow[string](value) } + """.trimIndent(), + ) + val model = model( + "example.com/genericmodel", + """ + package genericmodel + type Box[T any] struct { Value T } + func Flow[T any](value T) ( + *T, []T, map[string]T, chan T, + [1]T, struct { Value T }, func(T) T, Box[T], + ) { + pointer := &value + values := []T{value} + index := map[string]T{"value": value} + stream := make(chan T, 1) + stream <- value + array := [1]T{value} + record := struct { Value T }{Value: value} + callback := func(input T) T { return input } + box := Box[T]{Value: value} + return pointer, values, index, stream, array, record, callback, box + } + """.trimIndent(), + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val flow = program.findPackage("example.com/genericmodel")!!.functions.single { + it.fullName.startsWith("example.com/genericmodel.Flow[") + } + val returned = (flow.body!!.blocks.single { block -> + block.terminator is GoIRReturn + }.terminator as GoIRReturn).results + + assertThat(returned.map { it.type }).containsExactlyElementsOf(flow.signature.results) + } + + @Test + fun `generic receiver model instance uses concrete target types`(builder: GoIRTestBuilder) { + val project = project( + "example.com/genericreceiver", + """ + package genericreceiver + type Box[T any] struct { value T } + func (box Box[T]) Get() T { return box.value } + func Use(box Box[string]) string { return box.Get() } + """.trimIndent(), + ) + val model = model( + "example.com/genericreceiver", + """ + package genericreceiver + type Box[T any] struct { value T } + func (box Box[T]) Get() T { return box.value } + """.trimIndent(), + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val functions = program.findPackage("example.com/genericreceiver")!!.functions + val get = functions.single { + it.fullName.contains("Box[string]") && it.name.startsWith("Get[") + } + val returned = (get.body!!.blocks.single().terminator as GoIRReturn).results.single() + + assertThat(returned.type).isEqualTo(get.signature.results.single()) + } + + @Test + fun `generic model closure uses concrete target types`(builder: GoIRTestBuilder) { + val project = project( + "example.com/genericclosure", + """ + package genericclosure + func Apply[T any](value T) T { return value } + func UseString(value string) string { return Apply[string](value) } + func UseInt(value int) int { return Apply[int](value) } + """.trimIndent(), + ) + val model = model( + "example.com/genericclosure", + """ + package genericclosure + func Apply[T any](value T) T { + nested := func(input T) T { + inner := func() T { return value } + return inner() + } + return nested(value) + } + """.trimIndent(), + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val applyInstances = program.findPackage("example.com/genericclosure")!!.functions.filter { + it.fullName.startsWith("example.com/genericclosure.Apply[") + } + assertThat(applyInstances).hasSize(2) + applyInstances.forEach { apply -> + val outer = apply.findExpressions().single().fn + val inner = outer.findExpressions().single().fn + val concreteType = apply.signature.results.single() + + assertThat(outer.parent!!.function).isSameAs(apply) + assertThat(inner.parent!!.function).isSameAs(outer) + assertThat(outer.signature.params.single()).isEqualTo(apply.signature.params.single()) + assertThat(outer.signature.results.single()).isEqualTo(concreteType) + assertThat(inner.signature.results.single()).isEqualTo(concreteType) + val returned = (inner.body!!.blocks.single().terminator as GoIRReturn).results.single() + assertThat(returned.type).isEqualTo(concreteType) + } + assertThat( + applyInstances.map { apply -> + apply.findExpressions().single().fn.fullName + }, + ).doesNotHaveDuplicates() + } + + @Test + fun `one package cannot be modeled twice`(builder: GoIRTestBuilder) { + val project = project( + "example.com/app", + "package main\nimport \"strings\"\nfunc Use(value string) string { return strings.ToUpper(value) }\n", + ) + val source = "package strings\nfunc ToUpper(value string) string { return value }\n" + val firstModel = model("strings", source) + val secondModel = model("strings", source) + + assertThatThrownBy { + builder.buildFromDir( + project, + GoIRLoadConfig( + mode = GoIRLoadMode.PROJECT, + modelDirs = listOf(firstModel, secondModel), + ), + ) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("Go package \"strings\" is modeled more than once") + } + + @Test + fun `model package requires opentaint address prefix`(builder: GoIRTestBuilder) { + val project = project( + "example.com/app", + "package main\nimport \"strings\"\nfunc Use(value string) string { return strings.ToUpper(value) }\n", + ) + val model = model( + targetPath = "strings", + source = "package strings\nfunc ToUpper(value string) string { return value }\n", + module = "models", + ) + + assertThatThrownBy { + builder.buildFromDir(project, modelConfig(model)) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("import path must be opentaint/") + } + + @Test + fun `model initializer and its closures do not replace target initialization`(builder: GoIRTestBuilder) { + val project = project( + "example.com/initmodel", + "package initmodel\nvar Value string\nfunc init() { Value = \"original\" }\n", + ) + val model = model( + "example.com/initmodel", + "package initmodel\nvar Value string\nfunc init() { write := func() { Value = \"model\" }; write() }\n", + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val target = program.findPackage("example.com/initmodel")!! + val initializer = target.functions.single { it.name == "init#1" } + val values = initializer.findInstructions() + .mapNotNull { (it.value as? GoIRConstValue)?.value as? GoIRConstantValue.StringConst } + .map { it.value } + + assertThat(values).contains("original").doesNotContain("model") + assertThat(target.functions.map { it.fullName }).noneMatch { "opentaint" in it } + } + + @Test + fun `existing model field type must match target field type`(builder: GoIRTestBuilder) { + val project = project( + "example.com/fieldtype", + "package fieldtype\ntype Box struct { Value string }\nfunc (b *Box) Put(value string) {}\n", + ) + val model = model( + "example.com/fieldtype", + "package fieldtype\ntype Box struct { Value int }\nfunc (b *Box) Put(value string) { b.Value = len(value) }\n", + ) + + assertThatThrownBy { + builder.buildFromDir(project, modelConfig(model)) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("field example.com/fieldtype.Box.Value has a different type") + } + + @Test + fun `versioned target path can use its declared package name`(builder: GoIRTestBuilder) { + val project = project( + "github.com/acme/widget/v2", + "package widget\nfunc Identity(value string) string { return \"original\" }\n", + ) + val model = model( + "github.com/acme/widget/v2", + "package widget\nfunc Identity(value string) string { return value }\n", + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val identity = program.findPackage("github.com/acme/widget/v2")!! + .functions.single { it.name == "Identity" } + val returned = (identity.body!!.blocks.single().terminator as GoIRReturn).results.single() + + assertThat(returned).isInstanceOf(GoIRParameterValue::class.java) + } + + @Test + fun `model can use the dependency that it models`(builder: GoIRTestBuilder) { + val project = project( + "example.com/dependency", + "package dependency\ntype Value struct { Text string }\n" + + "func Identity(value Value) Value { return Value{Text: \"original\"} }\n", + ) + val model = model( + "example.com/dependency", + "package dependency\nimport target \"example.com/dependency\"\n" + + "func Identity(value target.Value) target.Value { return value }\n", + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val identity = program.findPackage("example.com/dependency")!! + .functions.single { it.name == "Identity" && it.parent == null } + val returned = (identity.body!!.blocks.single().terminator as GoIRReturn).results.single() + + assertThat((returned as GoIRParameterValue).paramIndex).isZero() + } + + @Test + fun `model call into modeled dependency resolves through effective lookup`(builder: GoIRTestBuilder) { + val project = project( + "example.com/dependencycall", + "package dependencycall\n" + + "func Helper(value string) string { return \"original\" }\n" + + "func Identity(value string) string { return \"original\" }\n", + ) + val model = model( + "example.com/dependencycall", + "package dependencycall\n" + + "import target \"example.com/dependencycall\"\n" + + "func Identity(value string) string { return target.Helper(value) }\n", + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val targetPackage = program.findPackage("example.com/dependencycall")!! + val helper = targetPackage.functions.single { it.name == "Helper" && it.parent == null } + val identity = targetPackage.functions.single { it.name == "Identity" && it.parent == null } + val callTarget = identity.findInstructions().single().call.target + + assertThat(callTarget).isInstanceOf(GoIRCallTarget.Function::class.java) + assertThat((callTarget as GoIRCallTarget.Function).function).isSameAs(helper) + } + + @Test + fun `unrelated model does not add packages to the analyzed project`(builder: GoIRTestBuilder) { + val project = project("example.com/unrelated", "package unrelated\nfunc Target() {}\n") + val model = model( + "example.com/absent", + "package absent\nimport \"net/url\"\n" + + "func Identity(value string) string { return url.QueryEscape(value) }\n", + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + + assertThat(program.findPackage("example.com/unrelated")).isNotNull() + assertThat(program.findPackage("example.com/absent")).isNull() + assertThat(program.findPackage("net/url")).isNull() + val withoutModels = builder.buildFromDir(project, GoIRLoadConfig(mode = GoIRLoadMode.PROJECT)) + assertThat(program.packages.keys).containsExactlyInAnyOrderElementsOf(withoutModels.packages.keys) + } + + @Test + fun `model module applies only to packages present in the project`(builder: GoIRTestBuilder) { + val project = project( + "example.com/partial", + "package partial\nfunc Identity(value string) string { return \"original\" }\n", + ) + val model = model( + "example.com/partial", + "package partial\nfunc Identity(value string) string { return value }\n", + ) + val absent = model.resolve("example.com/absent") + Files.createDirectories(absent) + absent.resolve("model.go").write("package absent\nfunc Target() {}\n") + + val program = builder.buildFromDir(project, modelConfig(model)) + val identity = program.findPackage("example.com/partial")!!.functions.single { it.name == "Identity" } + val returned = (identity.body!!.blocks.single().terminator as GoIRReturn).results.single() + + assertThat(returned).isInstanceOf(GoIRParameterValue::class.java) + assertThat(program.findPackage("example.com/absent")).isNull() + } + + @Test + fun `model-only dependency is visible only through effective lookup`(builder: GoIRTestBuilder) { + val project = project( + "example.com/modeldependency", + "package modeldependency\n" + + "func Identity(value string) string { return \"original\" }\n", + ) + val model = model( + "example.com/modeldependency", + "package modeldependency\n" + + "import \"net/url\"\n" + + "func Identity(value string) string { return url.QueryEscape(value) }\n", + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val identity = program.findPackage("example.com/modeldependency")!! + .functions.single { it.name == "Identity" && it.parent == null } + val call = identity.findInstructions().single() + + assertThat(builder.buildFromDir(project, GoIRLoadConfig(mode = GoIRLoadMode.PROJECT)).findPackage("net/url")).isNull() + assertThat(program.findPackage("net/url")).isNotNull() + assertThat(call.call.target?.displayName).isEqualTo("net/url.QueryEscape") + } + + @Test + fun `imported target declarations do not damage generic target types`(builder: GoIRTestBuilder) { + val project = project( + "example.com/genericdependency", + """ + package genericdependency + import _ "net/http" + type Box[T any] struct { Value T } + type Callback func(*Box[string]) + func Register(Callback) {} + """.trimIndent(), + ) + val model = model( + "example.com/genericdependency", + """ + package genericdependency + import _ "net/http" + import target "example.com/genericdependency" + func Register(callback target.Callback) { callback(nil) } + """.trimIndent(), + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val targetPackage = program.findPackage("example.com/genericdependency")!! + val register = targetPackage + .functions.single { it.name == "Register" } + + assertThat(targetPackage.findNamedType("Box")!!.typeParams.map { it.name }) + .containsExactly("T") + assertThat(register.findInstructions()) + .anyMatch { it.call.target is GoIRCallTarget.Dynamic } + } + + @Test + fun `model adds support declarations and keeps helper calls`(builder: GoIRTestBuilder) { + val project = project( + "example.com/support", + "package support\nfunc Target(value string) string { return \"original\" }\n", + ) + val model = model( + "example.com/support", + """ + package support + type Added struct { Text string } + var AddedGlobal string + const AddedConstant = "constant" + func helper(value string) string { return value } + func Target(value string) string { return helper(value) } + """.trimIndent(), + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val targetPackage = program.findPackage("example.com/support")!! + val helper = targetPackage.functions.single { it.name == "helper" } + val target = targetPackage.functions.single { it.name == "Target" } + val calledFunction = target.findInstructions().single().call.target as GoIRCallTarget.Function + + assertThat(targetPackage.findNamedType("Added")).isNotNull() + assertThat(targetPackage.findGlobal("AddedGlobal")).isNotNull() + assertThat(targetPackage.findConstant("AddedConstant")).isNotNull() + assertThat(helper.isSynthetic).isTrue() + assertThat(helper).isInstanceOf(GoIRModelFunction::class.java) + assertThat(helper.bodyAvailable).isTrue() + assertThat(calledFunction.function).isSameAs(helper) + } + + @Test + fun `model body owns its anonymous functions`(builder: GoIRTestBuilder) { + val project = project( + "example.com/closures", + "package closures\nfunc Apply(value string) string { return \"original\" }\n", + ) + val model = model( + "example.com/closures", + "package closures\nfunc Apply(value string) string { " + + "nested := func(input string) string { return input }; return nested(value) }\n", + ) + + val program = builder.buildFromDir(project, modelConfig(model)) + val apply = program.findPackage("example.com/closures")!!.functions.single { it.name == "Apply" } + val closure = apply.anonymousFunctions.single().function + + assertThat(apply.anonymousFunctions).hasSize(1) + assertThat(closure.parent!!.function).isSameAs(apply) + assertThat(closure.bodyAvailable).isTrue() + } + + @Test + fun `one model module can model multiple target packages`(builder: GoIRTestBuilder) { + val project = Files.createTempDirectory("goir-multi-package-project") + project.resolve("go.mod").write("module example.com/multiple\ngo 1.22\n") + listOf("first", "second").forEach { name -> + val packageDir = project.resolve(name) + Files.createDirectories(packageDir) + packageDir.resolve("source.go").write( + "package $name\nfunc Identity(value string) string { return \"original\" }\n", + ) + } + + val model = Files.createTempDirectory("goir-multi-package-model") + model.resolve("go.mod").write("module opentaint\ngo 1.25\n") + listOf("first", "second").forEach { name -> + val packageDir = model.resolve("example.com/multiple/$name") + Files.createDirectories(packageDir) + packageDir.resolve("model.go").write( + "package $name\nfunc Identity(value string) string { return value }\n", + ) + } + + val program = builder.buildFromDir(project, modelConfig(model)) + listOf("first", "second").forEach { name -> + val identity = program.findPackage("example.com/multiple/$name")!! + .functions.single { it.name == "Identity" } + val returned = (identity.body!!.blocks.single().terminator as GoIRReturn).results.single() + assertThat(returned).isInstanceOf(GoIRParameterValue::class.java) + } + } + + @Test + fun `model anonymous interface definitions retain their own lookup scope`(builder: GoIRTestBuilder) { + val project = project( + "example.com/interfaces", + """ + package interfaces + func Original(value interface { Number() int }) int { return value.Number() } + func Target(value string) string { return value } + """.trimIndent(), + ) + val model = model( + "example.com/interfaces", + """ + package interfaces + type Box struct { value string } + func (b Box) Text() string { return b.value } + func helper(input interface { Text() string }) string { return input.Text() } + func Target(value string) string { return helper(Box{value}) } + """.trimIndent(), + ) + val program = builder.buildFromDir(project, modelConfig(model)) + val functions = program.findPackage("example.com/interfaces")!!.functions + val modelType = functions.single { it.name == "helper" }.signature.params.single() as GoIRAnonymousInterfaceType + val originalType = functions.single { it.name == "Original" }.signature.params.single() as GoIRAnonymousInterfaceType + assertThat(modelType.methods.map { it.name }).containsExactly("Text") + assertThat(originalType.methods.map { it.name }).containsExactly("Number") + assertThat(program.anonymousInterfaces[modelType.id]).isEqualTo(modelType) + } + + @Test + fun `package transformers intercept lookup in order and can bypass earlier resolvers`(builder: GoIRTestBuilder) { + val program = builder.buildFromSource("package p\nfunc Target() {}\nfunc Caller() { Target() }\n") as GoIRProgramImpl + val resolved = mutableListOf() + fun recording(name: String) = object : GoIRTransformer { + override fun resolvePackage(importPath: String, next: () -> GoIRPackage?): GoIRPackage? { + if (importPath == "test/p") resolved += name + return next() + } + } + val interceptor = object : GoIRTransformer { + override fun resolvePackage(importPath: String, next: () -> GoIRPackage?): GoIRPackage? { + return if (importPath == "test/p") GoIRPackageImpl(importPath, "p") else next() + } + } + program.install(listOf(recording("first"), interceptor, recording("last"))) + + val pkg = program.findPackage("test/p")!! + assertThat(pkg.functions).isEmpty() + assertThat(program.findPackage("test/p")).isSameAs(pkg) + assertThat(resolved).containsExactly("last") + } + + private fun project(module: String, source: String): Path { + val project = Files.createTempDirectory("goir-model-project") + project.resolve("go.mod").write("module $module\ngo 1.22\n") + project.resolve("source.go").write(source) + return project + } + + private fun model(targetPath: String, source: String, module: String = "opentaint"): Path { + val model = Files.createTempDirectory("goir-model") + model.resolve("go.mod").write("module $module\ngo 1.25\n") + val modelPackage = model.resolve(targetPath) + Files.createDirectories(modelPackage) + modelPackage.resolve("model.go").write(source) + return model + } + + private fun modelConfig(model: Path): GoIRLoadConfig { + return GoIRLoadConfig(mode = GoIRLoadMode.PROJECT, modelDirs = listOf(model)) + } + + private fun Path.write(content: String) { + toFile().writeText(content) + } +} diff --git a/core/opentaint-ir/go/go-ssa-server/build_program_test.go b/core/opentaint-ir/go/go-ssa-server/build_program_test.go index 57cf40a5a3..05d0bec7b6 100644 --- a/core/opentaint-ir/go/go-ssa-server/build_program_test.go +++ b/core/opentaint-ir/go/go-ssa-server/build_program_test.go @@ -6,6 +6,7 @@ import ( "net" "os" "path/filepath" + "strings" "testing" "google.golang.org/grpc" @@ -15,83 +16,130 @@ import ( pb "github.com/opentaint/go-ir/go-ssa-server/proto/goir" ) +type buildResult struct { + program *pb.ProtoProgram + models []*pb.ProtoModelProgram +} + func writeTinyModule(t *testing.T) string { + t.Helper() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), "module example.com/m\ngo 1.22\n") + writeFile( + t, + filepath.Join(dir, "main.go"), + "package main\nimport \"strings\"\nfunc Greet(s string) string { return strings.ToUpper(s) }\nfunc main() { println(Greet(\"x\")) }\n", + ) + return dir +} + +func writeStringsModel(t *testing.T, source string) string { + t.Helper() dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module example.com/m\ngo 1.22\n"), 0o644); err != nil { + writeFile(t, filepath.Join(dir, "go.mod"), "module opentaint\ngo 1.25\n") + packageDir := filepath.Join(dir, "strings") + if err := os.MkdirAll(packageDir, 0o755); err != nil { t.Fatal(err) } - src := "package main\nimport \"strings\"\nfunc Greet(s string) string { return strings.ToUpper(s) }\nfunc main() { println(Greet(\"x\")) }\n" - if err := os.WriteFile(filepath.Join(dir, "main.go"), []byte(src), 0o644); err != nil { + writeFile(t, filepath.Join(packageDir, "model.go"), source) + return dir +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { t.Fatal(err) } - return dir } -func runBuild(t *testing.T, req *pb.BuildProgramRequest) ([]*pb.ProtoPackage, map[int32]bool) { +func startBuild(t *testing.T, req *pb.BuildProgramRequest) pb.GoSSAService_BuildProgramClient { + t.Helper() lis := bufconn.Listen(1 << 20) srv := grpc.NewServer() pb.RegisterGoSSAServiceServer(srv, NewGoSSAServer("test", "test")) go srv.Serve(lis) - defer srv.Stop() + t.Cleanup(srv.Stop) - conn, err := grpc.NewClient("passthrough:///bufnet", - grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }), + conn, err := grpc.NewClient( + "passthrough:///bufnet", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return lis.DialContext(ctx) + }), grpc.WithTransportCredentials(insecure.NewCredentials()), - // The bulk program payload can exceed gRPC's default 4 MB receive limit; - // mirror the production client (GoSsaServerProcess) which allows 256 MB. - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(256*1024*1024))) + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(256*1024*1024)), + ) if err != nil { t.Fatal(err) } - defer conn.Close() + t.Cleanup(func() { _ = conn.Close() }) stream, err := pb.NewGoSSAServiceClient(conn).BuildProgram(context.Background(), req) if err != nil { t.Fatal(err) } - var pkgs []*pb.ProtoPackage - bodies := map[int32]bool{} + return stream +} + +func runBuildResult(t *testing.T, req *pb.BuildProgramRequest) buildResult { + t.Helper() + stream := startBuild(t, req) + result := buildResult{program: &pb.ProtoProgram{}} for { resp, err := stream.Recv() + if err == io.EOF { + return result + } if err != nil { - if err != io.EOF { - t.Fatalf("stream error: %v", err) - } - break + t.Fatalf("stream error: %v", err) } - switch p := resp.Payload.(type) { + switch payload := resp.Payload.(type) { case *pb.BuildProgramResponse_Program: - pkgs = append(pkgs, p.Program.Packages...) - for _, b := range p.Program.FunctionBodies { - bodies[b.FunctionId] = true - } + result.program = payload.Program + case *pb.BuildProgramResponse_ModelProgram: + result.models = append(result.models, payload.ModelProgram) case *pb.BuildProgramResponse_PackageDef: - pkgs = append(pkgs, p.PackageDef) + result.program.Packages = append(result.program.Packages, payload.PackageDef) case *pb.BuildProgramResponse_FunctionBody: - bodies[p.FunctionBody.FunctionId] = true + result.program.FunctionBodies = append(result.program.FunctionBodies, payload.FunctionBody) case *pb.BuildProgramResponse_Error: - if p.Error.Fatal { - t.Fatalf("fatal: %s", p.Error.Message) + if payload.Error.Fatal { + t.Fatalf("fatal: %s", payload.Error.Message) } } } - return pkgs, bodies +} + +func runBuildFatal(t *testing.T, req *pb.BuildProgramRequest) string { + t.Helper() + stream := startBuild(t, req) + for { + resp, err := stream.Recv() + if err == io.EOF { + t.Fatal("expected a fatal Go model build error") + } + if err != nil { + t.Fatalf("expected a Go model build error, got stream error: %v", err) + } + if failure, ok := resp.Payload.(*pb.BuildProgramResponse_Error); ok && failure.Error.Fatal { + return failure.Error.Message + } + } } func findPkg(pkgs []*pb.ProtoPackage, path string) *pb.ProtoPackage { - for _, p := range pkgs { - if p.ImportPath == path { - return p + for _, pkg := range pkgs { + if pkg.ImportPath == path { + return pkg } } return nil } -func bodyIDsOf(p *pb.ProtoPackage, bodies map[int32]bool) (withBody, total int) { - for _, f := range p.Functions { - if f.HasBody { +func bodyIDsOf(pkg *pb.ProtoPackage, bodies map[int32]bool) (withBody, total int) { + for _, function := range pkg.Functions { + if function.HasBody { total++ - if bodies[f.Id] { + if bodies[function.Id] { withBody++ } } @@ -100,37 +148,177 @@ func bodyIDsOf(p *pb.ProtoPackage, bodies map[int32]bool) (withBody, total int) } func TestBuildProgram_ProjectMode_NoStdlibBodies(t *testing.T) { - dir := writeTinyModule(t) - pkgs, bodies := runBuild(t, &pb.BuildProgramRequest{WorkingDir: dir, Patterns: []string{"./..."}, Mode: pb.LoadMode_LOAD_MODE_PROJECT, InstantiateGenerics: true, SanityCheck: true}) + result := runBuildResult(t, &pb.BuildProgramRequest{ + WorkingDir: writeTinyModule(t), Patterns: []string{"./..."}, Mode: pb.LoadMode_LOAD_MODE_PROJECT, + InstantiateGenerics: true, SanityCheck: true, + }) + bodies := make(map[int32]bool, len(result.program.FunctionBodies)) + for _, body := range result.program.FunctionBodies { + bodies[body.FunctionId] = true + } - main := findPkg(pkgs, "example.com/m") - if main == nil || main.IsDependency || main.IsStdlib { - t.Fatalf("main package missing or misclassified: %+v", main) + mainPackage := findPkg(result.program.Packages, "example.com/m") + if mainPackage == nil || mainPackage.IsDependency || mainPackage.IsStdlib { + t.Fatalf("main package missing or misclassified: %+v", mainPackage) } - wb, total := bodyIDsOf(main, bodies) - if total == 0 || wb != total { - t.Fatalf("project bodies: want all %d, got %d", total, wb) + withBody, total := bodyIDsOf(mainPackage, bodies) + if total == 0 || withBody != total { + t.Fatalf("project bodies: want all %d, got %d", total, withBody) } - strs := findPkg(pkgs, "strings") - if strs == nil || !strs.IsStdlib { - t.Fatalf("strings not present/stdlib: %+v", strs) + stringsPackage := findPkg(result.program.Packages, "strings") + if stringsPackage == nil || !stringsPackage.IsStdlib { + t.Fatalf("strings not present or not standard library: %+v", stringsPackage) } - if wb, _ := bodyIDsOf(strs, bodies); wb != 0 { - t.Fatalf("PROJECT mode streamed %d stdlib bodies, want 0", wb) + if withBody, _ := bodyIDsOf(stringsPackage, bodies); withBody != 0 { + t.Fatalf("PROJECT mode streamed %d standard library bodies, want 0", withBody) } } func TestBuildProgram_FullMode_StreamsStdlibBodies(t *testing.T) { - dir := writeTinyModule(t) - pkgs, bodies := runBuild(t, &pb.BuildProgramRequest{WorkingDir: dir, Patterns: []string{"./..."}, Mode: pb.LoadMode_LOAD_MODE_FULL, InstantiateGenerics: true, SanityCheck: true}) - strs := findPkg(pkgs, "strings") - if strs == nil { - t.Fatal("strings package missing in FULL mode") + result := runBuildResult(t, &pb.BuildProgramRequest{ + WorkingDir: writeTinyModule(t), Patterns: []string{"./..."}, Mode: pb.LoadMode_LOAD_MODE_FULL, + InstantiateGenerics: true, SanityCheck: true, + }) + bodies := make(map[int32]bool, len(result.program.FunctionBodies)) + for _, body := range result.program.FunctionBodies { + bodies[body.FunctionId] = true + } + stringsPackage := findPkg(result.program.Packages, "strings") + if stringsPackage == nil || !stringsPackage.IsStdlib { + t.Fatalf("strings package is missing or not standard library: %+v", stringsPackage) } - if !strs.IsStdlib { - t.Fatalf("strings not stdlib: %+v", strs) + if withBody, _ := bodyIDsOf(stringsPackage, bodies); withBody == 0 { + t.Fatal("FULL mode streamed no standard library bodies") + } +} + +func TestBuildProgram_StreamsRawGoModelForKotlinResolution(t *testing.T) { + modelDir := writeStringsModel( + t, + "package strings\nfunc ToUpper(value string) string { return value }\n", + ) + result := runBuildResult(t, &pb.BuildProgramRequest{ + WorkingDir: writeTinyModule(t), Patterns: []string{"./..."}, Mode: pb.LoadMode_LOAD_MODE_PROJECT, + InstantiateGenerics: true, SanityCheck: true, ModelDirs: []string{modelDir}, + }) + + if len(result.models) != 1 { + t.Fatalf("want one raw model program, got %d", len(result.models)) + } + model := result.models[0] + if model.Source != modelDir { + t.Fatalf("model source is %q, want %q", model.Source, modelDir) + } + if findPkg(model.Program.Packages, "opentaint/strings") == nil { + t.Fatal("raw model program has no opentaint/strings package") + } + if findPkg(result.program.Packages, "opentaint/strings") != nil { + t.Fatal("base program contains the raw model package") + } + stringsPackage := findPkg(result.program.Packages, "strings") + if stringsPackage == nil { + t.Fatal("base program has no strings package") + } + var toUpperID int32 + for _, function := range stringsPackage.Functions { + if function.Name == "ToUpper" { + toUpperID = function.Id + } + } + for _, body := range result.program.FunctionBodies { + if body.FunctionId == toUpperID { + t.Fatal("Go server attached the model body to the base program") + } + } +} + +func TestBuildProgram_GoModelMustCompile(t *testing.T) { + modelDir := writeStringsModel( + t, + "package strings\nfunc ToUpper(value string) string { return missing }\n", + ) + message := runBuildFatal(t, &pb.BuildProgramRequest{ + WorkingDir: writeTinyModule(t), Patterns: []string{"./..."}, Mode: pb.LoadMode_LOAD_MODE_PROJECT, + InstantiateGenerics: true, SanityCheck: true, ModelDirs: []string{modelDir}, + }) + if !strings.Contains(message, "Go model does not compile") || !strings.Contains(message, "undefined: missing") { + t.Fatalf("unexpected compilation error: %s", message) + } +} + +func TestBuildProgram_GoModelCanImportTargetProjectModule(t *testing.T) { + projectDir := t.TempDir() + writeFile(t, filepath.Join(projectDir, "go.mod"), "module example.com/dependency\ngo 1.22\n") + writeFile( + t, + filepath.Join(projectDir, "dependency.go"), + "package dependency\ntype Value struct { Text string }\nfunc Identity(value Value) Value { return Value{} }\n", + ) + + modelDir := t.TempDir() + writeFile(t, filepath.Join(modelDir, "go.mod"), "module opentaint\ngo 1.25\n") + modelPackageDir := filepath.Join(modelDir, "example.com", "dependency") + if err := os.MkdirAll(modelPackageDir, 0o755); err != nil { + t.Fatal(err) + } + writeFile( + t, + filepath.Join(modelPackageDir, "model.go"), + "package dependency\nimport target \"example.com/dependency\"\nfunc Identity(value target.Value) target.Value { return value }\n", + ) + + result := runBuildResult(t, &pb.BuildProgramRequest{ + WorkingDir: projectDir, Patterns: []string{"./..."}, Mode: pb.LoadMode_LOAD_MODE_PROJECT, + InstantiateGenerics: true, SanityCheck: true, ModelDirs: []string{modelDir}, + }) + if len(result.models) != 1 { + t.Fatalf("want one raw model program, got %d", len(result.models)) + } + if findPkg(result.models[0].Program.Packages, "example.com/dependency") == nil { + t.Fatal("raw model program has no imported target project package") + } +} + +func TestBuildProgram_GoModelCanImportResolvedProjectDependency(t *testing.T) { + dependencyDir := t.TempDir() + writeFile(t, filepath.Join(dependencyDir, "go.mod"), "module example.com/library\ngo 1.22\n") + writeFile( + t, + filepath.Join(dependencyDir, "library.go"), + "package library\ntype Value struct { Text string }\nfunc Identity(value Value) Value { return Value{} }\n", + ) + + projectDir := t.TempDir() + projectGoMod := "module example.com/app\ngo 1.22\nrequire example.com/library v0.0.0\n" + + "replace example.com/library => " + dependencyDir + "\n" + writeFile(t, filepath.Join(projectDir, "go.mod"), projectGoMod) + writeFile( + t, + filepath.Join(projectDir, "main.go"), + "package main\nimport \"example.com/library\"\nfunc Use(value library.Value) library.Value { return library.Identity(value) }\n", + ) + + modelDir := t.TempDir() + writeFile(t, filepath.Join(modelDir, "go.mod"), "module opentaint\ngo 1.25\n") + modelPackageDir := filepath.Join(modelDir, "example.com", "library") + if err := os.MkdirAll(modelPackageDir, 0o755); err != nil { + t.Fatal(err) + } + writeFile( + t, + filepath.Join(modelPackageDir, "model.go"), + "package library\nimport target \"example.com/library\"\nfunc Identity(value target.Value) target.Value { return value }\n", + ) + + result := runBuildResult(t, &pb.BuildProgramRequest{ + WorkingDir: projectDir, Patterns: []string{"./..."}, Mode: pb.LoadMode_LOAD_MODE_PROJECT, + InstantiateGenerics: true, SanityCheck: true, ModelDirs: []string{modelDir}, + }) + if len(result.models) != 1 { + t.Fatalf("want one raw model program, got %d", len(result.models)) } - if wb, _ := bodyIDsOf(strs, bodies); wb == 0 { - t.Fatal("FULL mode streamed no stdlib bodies") + dependency := findPkg(result.models[0].Program.Packages, "example.com/library") + if dependency == nil || !dependency.IsDependency { + t.Fatalf("raw model has no resolved project dependency: %+v", dependency) } } diff --git a/core/opentaint-ir/go/go-ssa-server/builder.go b/core/opentaint-ir/go/go-ssa-server/builder.go index 41a3a30259..34ee94bd4e 100644 --- a/core/opentaint-ir/go/go-ssa-server/builder.go +++ b/core/opentaint-ir/go/go-ssa-server/builder.go @@ -15,17 +15,35 @@ import ( ) func buildSSA(req *pb.BuildProgramRequest) (*ssa.Program, []*ssa.Package, map[string]*packages.Package, error) { + return buildSSAWithPackageErrors(req, false) +} + +func buildSSAWithPackageErrors(req *pb.BuildProgramRequest, failOnPackageErrors bool) (*ssa.Program, []*ssa.Package, map[string]*packages.Package, error) { + return buildSSAWithPackageErrorsAndEnv(req, failOnPackageErrors, nil) +} + +func buildSSAWithPackageErrorsAndEnv( + req *pb.BuildProgramRequest, + failOnPackageErrors bool, + extraEnv []string, +) (*ssa.Program, []*ssa.Package, map[string]*packages.Package, error) { cfg := &packages.Config{ Mode: packages.NeedFiles | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedImports | packages.NeedDeps | packages.NeedName | packages.NeedModule, Dir: req.WorkingDir, } + if len(extraEnv) > 0 { + cfg.Env = append(os.Environ(), extraEnv...) + } if len(req.BuildTags) > 0 { cfg.BuildFlags = []string{"-tags=" + strings.Join(req.BuildTags, ",")} } if req.Gopath != "" { - cfg.Env = append(os.Environ(), "GOPATH="+req.Gopath) + if cfg.Env == nil { + cfg.Env = os.Environ() + } + cfg.Env = append(cfg.Env, "GOPATH="+req.Gopath) } if req.Goroot != "" { if cfg.Env == nil { @@ -55,6 +73,19 @@ func buildSSA(req *pb.BuildProgramRequest) (*ssa.Program, []*ssa.Package, map[st log.Printf("WARN: package %s: %v", p.PkgPath, e) } } + if failOnPackageErrors { + var loadErrors []string + packages.Visit(loaded, func(p *packages.Package) bool { + for _, e := range p.Errors { + loadErrors = append(loadErrors, fmt.Sprintf("%s: %v", p.PkgPath, e)) + } + return true + }, nil) + if len(loadErrors) > 0 { + sort.Strings(loadErrors) + return nil, nil, nil, fmt.Errorf("Go model does not compile:\n%s", strings.Join(loadErrors, "\n")) + } + } mode := ssa.BuilderMode(0) if req.InstantiateGenerics { @@ -77,7 +108,11 @@ func buildSSA(req *pb.BuildProgramRequest) (*ssa.Program, []*ssa.Package, map[st return prog, all, info, nil } -func classifyPackage(p *packages.Package, projectModulePaths map[string]bool) (isStdlib bool, isDependency bool) { +func classifyPackage( + p *packages.Package, + projectModulePaths map[string]bool, + onlyListedProjectModules bool, +) (isStdlib bool, isDependency bool) { if p == nil { return false, true } @@ -85,6 +120,7 @@ func classifyPackage(p *packages.Package, projectModulePaths map[string]bool) (i if isStdlib { return true, false } - isProject := p.Module != nil && (p.Module.Main || projectModulePaths[p.Module.Path]) + isProject := p.Module != nil && (projectModulePaths[p.Module.Path] || + (!onlyListedProjectModules && p.Module.Main)) return false, !isProject } diff --git a/core/opentaint-ir/go/go-ssa-server/go.mod b/core/opentaint-ir/go/go-ssa-server/go.mod index 99d6862135..4b3492fd69 100644 --- a/core/opentaint-ir/go/go-ssa-server/go.mod +++ b/core/opentaint-ir/go/go-ssa-server/go.mod @@ -3,13 +3,13 @@ module github.com/opentaint/go-ir/go-ssa-server go 1.26.0 require ( + golang.org/x/mod v0.37.0 golang.org/x/tools v0.45.0 google.golang.org/grpc v1.81.1 google.golang.org/protobuf v1.36.11 ) require ( - golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect diff --git a/core/opentaint-ir/go/go-ssa-server/model_workspace.go b/core/opentaint-ir/go/go-ssa-server/model_workspace.go new file mode 100644 index 0000000000..789adefd75 --- /dev/null +++ b/core/opentaint-ir/go/go-ssa-server/model_workspace.go @@ -0,0 +1,116 @@ +package server + +import ( + "fmt" + "go/version" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + + "golang.org/x/mod/modfile" + "golang.org/x/tools/go/packages" +) + +func createModelWorkspace( + modelDir string, + projectInfo map[string]*packages.Package, + projectModulePaths map[string]bool, +) (string, func(), error) { + moduleDirs := make(map[string]bool) + modelDir, err := filepath.Abs(modelDir) + if err != nil { + return "", nil, fmt.Errorf("resolve Go model directory: %w", err) + } + moduleDirs[modelDir] = true + + for _, pkg := range projectInfo { + module := pkg.Module + if module == nil || module.Dir == "" || (!module.Main && !projectModulePaths[module.Path]) { + continue + } + moduleDir, absErr := filepath.Abs(module.Dir) + if absErr != nil { + return "", nil, fmt.Errorf("resolve project module %s: %w", module.Path, absErr) + } + moduleDirs[moduleDir] = true + } + + dirs := make([]string, 0, len(moduleDirs)) + for dir := range moduleDirs { + dirs = append(dirs, dir) + } + sort.Strings(dirs) + + tempDir, err := os.MkdirTemp("", "opentaint-go-model-work-*") + if err != nil { + return "", nil, fmt.Errorf("create Go model workspace: %w", err) + } + cleanup := func() { _ = os.RemoveAll(tempDir) } + + goVersion := runtime.Version() + if !version.IsValid(goVersion) { + goVersion = "go1.25.0" + } + goVersion = newerGoVersion(goVersion, moduleGoVersion(modelDir)) + for _, pkg := range projectInfo { + if pkg.Module != nil && moduleDirs[pkg.Module.Dir] { + goVersion = newerGoVersion(goVersion, pkg.Module.GoVersion) + } + } + + var workspace strings.Builder + fmt.Fprintf(&workspace, "go %s\n\nuse (\n", strings.TrimPrefix(goVersion, "go")) + for _, dir := range dirs { + fmt.Fprintf(&workspace, "\t%s\n", strconv.Quote(filepath.ToSlash(dir))) + } + workspace.WriteString(")\n") + + workspacePath := filepath.Join(tempDir, "go.work") + if err := os.WriteFile(workspacePath, []byte(workspace.String()), 0o600); err != nil { + cleanup() + return "", nil, fmt.Errorf("write Go model workspace: %w", err) + } + return workspacePath, cleanup, nil +} + +func moduleGoVersion(moduleDir string) string { + data, err := os.ReadFile(filepath.Join(moduleDir, "go.mod")) + if err != nil { + return "" + } + parsed, err := modfile.Parse("go.mod", data, nil) + if err != nil || parsed.Go == nil { + return "" + } + return parsed.Go.Version +} + +func newerGoVersion(current, candidate string) string { + candidate = "go" + strings.TrimPrefix(candidate, "go") + if version.IsValid(candidate) && version.Compare(candidate, current) > 0 { + return candidate + } + return current +} + +func modelModulePaths(modelDir string, modelInfo map[string]*packages.Package) map[string]bool { + result := make(map[string]bool) + modelDir, err := filepath.Abs(modelDir) + if err != nil { + return result + } + modelDir = filepath.Clean(modelDir) + for _, pkg := range modelInfo { + if pkg.Module == nil || pkg.Module.Dir == "" { + continue + } + moduleDir, absErr := filepath.Abs(pkg.Module.Dir) + if absErr == nil && filepath.Clean(moduleDir) == modelDir { + result[pkg.Module.Path] = true + } + } + return result +} diff --git a/core/opentaint-ir/go/go-ssa-server/model_workspace_test.go b/core/opentaint-ir/go/go-ssa-server/model_workspace_test.go new file mode 100644 index 0000000000..500190cd33 --- /dev/null +++ b/core/opentaint-ir/go/go-ssa-server/model_workspace_test.go @@ -0,0 +1,40 @@ +package server + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/tools/go/packages" +) + +func TestCreateModelWorkspaceUsesNewestModuleGoVersion(t *testing.T) { + modelDir := t.TempDir() + projectDir := t.TempDir() + writeFile(t, filepath.Join(modelDir, "go.mod"), "module opentaint\ngo 1.98.1\n") + writeFile(t, filepath.Join(projectDir, "go.mod"), "module example.com/project\ngo 1.99.2\n") + projectInfo := map[string]*packages.Package{ + "example.com/project": { + Module: &packages.Module{ + Path: "example.com/project", + GoVersion: "1.99.2", + Main: true, + Dir: projectDir, + }, + }, + } + + workspace, cleanup, err := createModelWorkspace(modelDir, projectInfo, nil) + if err != nil { + t.Fatal(err) + } + defer cleanup() + data, err := os.ReadFile(workspace) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(string(data), "go 1.99.2\n") { + t.Fatalf("workspace does not use the newest Go version:\n%s", data) + } +} diff --git a/core/opentaint-ir/go/go-ssa-server/serializer.go b/core/opentaint-ir/go/go-ssa-server/serializer.go index acaa181e07..43403176c5 100644 --- a/core/opentaint-ir/go/go-ssa-server/serializer.go +++ b/core/opentaint-ir/go/go-ssa-server/serializer.go @@ -67,6 +67,7 @@ type serializer struct { mode pb.LoadMode projectModulePaths map[string]bool + onlyListedModules bool loadInfo map[string]*packages.Package } @@ -92,6 +93,21 @@ func newSerializerForBuild(prog *ssa.Program, pkgs []*ssa.Package, info map[stri return s } +func newSerializerForModelBuild( + prog *ssa.Program, + pkgs []*ssa.Package, + info map[string]*packages.Package, + projectModulePaths map[string]bool, +) *serializer { + s := newSerializerWithIDs(prog, pkgs, newIDAllocator()) + s.mode = pb.LoadMode_LOAD_MODE_PROJECT + s.projectModulePaths = projectModulePaths + s.onlyListedModules = true + s.loadInfo = info + s.collectAll() + return s +} + func (s *serializer) collectAll() { for _, pkg := range s.pkgs { s.ids.packageID(pkg) @@ -151,7 +167,7 @@ func (s *serializer) shouldEmitBody(fn *ssa.Function) bool { if dp == nil { return fn.Package() == nil } - isStdlib, isDep := classifyPackage(s.loadInfo[dp.Pkg.Path()], s.projectModulePaths) + isStdlib, isDep := classifyPackage(s.loadInfo[dp.Pkg.Path()], s.projectModulePaths, s.onlyListedModules) return !isDep && !isStdlib } @@ -168,7 +184,7 @@ func (s *serializer) serializeProgram() (*pb.ProtoProgram, []*pb.ProtoError) { protoPkgBySSA := make(map[*ssa.Package]*pb.ProtoPackage, len(s.pkgs)) for _, pkg := range s.pkgs { pp := s.serializePackage(pkg) - isStdlib, isDep := classifyPackage(s.loadInfo[pkg.Pkg.Path()], s.projectModulePaths) + isStdlib, isDep := classifyPackage(s.loadInfo[pkg.Pkg.Path()], s.projectModulePaths, s.onlyListedModules) pp.IsStdlib = isStdlib pp.IsDependency = isDep protoPkgBySSA[pkg] = pp @@ -338,6 +354,14 @@ func (s *serializer) collectFunction(fn *ssa.Function) { func (s *serializer) collectFunctionSignature(fn *ssa.Function) { s.ids.functionID(fn) s.collectType(fn.Signature) + if typeParams := fn.TypeParams(); typeParams != nil { + for i := 0; i < typeParams.Len(); i++ { + s.collectType(typeParams.At(i)) + } + } + for _, typeArg := range fn.TypeArgs() { + s.collectType(typeArg) + } // Collect types from parameters for _, p := range fn.Params { @@ -503,6 +527,12 @@ func (s *serializer) collectType(t types.Type) { } case *types.Named: s.collectType(ut.Underlying()) + typeParams := ut.TypeParams() + if typeParams != nil { + for i := 0; i < typeParams.Len(); i++ { + s.collectType(typeParams.At(i)) + } + } // Also collect type args for instantiated generics targs := ut.TypeArgs() if targs != nil { @@ -866,6 +896,23 @@ func (s *serializer) serializeFunction(fn *ssa.Function) *pb.ProtoFunction { pf.PackageId = s.ids.packageID(fnPkg) } pf.SignatureTypeId = s.typeID(fn.Signature) + if origin := fn.Origin(); origin != nil && origin != fn { + pf.OriginFunctionId = s.ids.functionID(origin) + } + if typeParams := fn.TypeParams(); typeParams != nil { + for i := 0; i < typeParams.Len(); i++ { + typeParam := typeParams.At(i) + pf.TypeParams = append(pf.TypeParams, &pb.ProtoTypeParamDecl{ + Name: typeParam.Obj().Name(), + Index: int32(typeParam.Index()), + ConstraintTypeId: s.typeID(typeParam.Constraint()), + TypeId: s.typeID(typeParam), + }) + } + } + for _, typeArg := range fn.TypeArgs() { + pf.TypeArgIds = append(pf.TypeArgIds, s.typeID(typeArg)) + } // Method info if recv := fn.Signature.Recv(); recv != nil { @@ -985,6 +1032,17 @@ func (s *serializer) serializeNamedType(named *types.TypeName) *pb.ProtoNamedTyp if namedType == nil { return nt } + if typeParams := namedType.TypeParams(); typeParams != nil { + for i := 0; i < typeParams.Len(); i++ { + typeParam := typeParams.At(i) + nt.TypeParams = append(nt.TypeParams, &pb.ProtoTypeParamDecl{ + Name: typeParam.Obj().Name(), + Index: int32(typeParam.Index()), + ConstraintTypeId: s.typeID(typeParam.Constraint()), + TypeId: s.typeID(typeParam), + }) + } + } mset := s.prog.MethodSets.MethodSet(namedType) for i := 0; i < mset.Len(); i++ { fn := s.prog.MethodValue(mset.At(i)) diff --git a/core/opentaint-ir/go/go-ssa-server/server.go b/core/opentaint-ir/go/go-ssa-server/server.go index 454a3e164a..e584542540 100644 --- a/core/opentaint-ir/go/go-ssa-server/server.go +++ b/core/opentaint-ir/go/go-ssa-server/server.go @@ -90,18 +90,81 @@ func (s *goSSAServer) BuildProgram(req *pb.BuildProgramRequest, stream pb.GoSSAS return err } } - if err := stream.Send(&pb.BuildProgramResponse{Payload: &pb.BuildProgramResponse_Program{Program: program}}); err != nil { return fmt.Errorf("streaming program: %w", err) } + for _, modelDir := range req.ModelDirs { + workspacePath, cleanupWorkspace, workspaceErr := createModelWorkspace( + modelDir, info, projectModulePaths, + ) + if workspaceErr != nil { + return stream.Send(&pb.BuildProgramResponse{Payload: &pb.BuildProgramResponse_Error{Error: &pb.ProtoError{ + Message: fmt.Sprintf("loading Go model %s: %v", modelDir, workspaceErr), + Fatal: true, + }}}) + } + modelReq := &pb.BuildProgramRequest{ + Patterns: []string{"./..."}, + InstantiateGenerics: req.InstantiateGenerics, + SanityCheck: req.SanityCheck, + Gopath: req.Gopath, + Goroot: req.Goroot, + BuildTags: req.BuildTags, + WorkingDir: modelDir, + Mode: pb.LoadMode_LOAD_MODE_PROJECT, + } + modelProg, modelPkgs, modelInfo, modelErr := buildSSAWithPackageErrorsAndEnv( + modelReq, true, []string{"GOWORK=" + workspacePath}, + ) + cleanupWorkspace() + if modelErr != nil { + return stream.Send(&pb.BuildProgramResponse{Payload: &pb.BuildProgramResponse_Error{Error: &pb.ProtoError{ + Message: fmt.Sprintf("loading Go model %s: %v", modelDir, modelErr), + Fatal: true, + }}}) + } + modelSerializer := newSerializerForModelBuild( + modelProg, modelPkgs, modelInfo, modelModulePaths(modelDir, modelInfo), + ) + modelProgram, modelSerializeErrors := modelSerializer.serializeProgram() + for _, modelSerializeErr := range modelSerializeErrors { + if err := stream.Send(&pb.BuildProgramResponse{Payload: &pb.BuildProgramResponse_Error{Error: modelSerializeErr}}); err != nil { + return err + } + } + if err := stream.Send(&pb.BuildProgramResponse{Payload: &pb.BuildProgramResponse_ModelProgram{ + ModelProgram: &pb.ProtoModelProgram{Source: modelDir, Program: modelProgram}, + }}); err != nil { + return fmt.Errorf("streaming Go model %s: %w", modelDir, err) + } + } + return stream.Send(&pb.BuildProgramResponse{ Payload: &pb.BuildProgramResponse_Summary{Summary: &pb.ProtoBuildSummary{ - PackageCount: int32(ser.stats.packageCount), - FunctionCount: int32(ser.stats.functionCount), - TypeCount: int32(ser.stats.typeCount), - InstructionCount: int32(ser.stats.instructionCount), + PackageCount: int32(len(program.Packages)), + FunctionCount: countProgramFunctions(program), + TypeCount: int32(len(program.Types)), + InstructionCount: countProgramInstructions(program), BuildTimeMs: time.Since(startTime).Milliseconds(), }}, }) } + +func countProgramFunctions(program *pb.ProtoProgram) int32 { + var count int32 + for _, pkg := range program.Packages { + count += int32(len(pkg.Functions)) + } + return count +} + +func countProgramInstructions(program *pb.ProtoProgram) int32 { + var count int32 + for _, body := range program.FunctionBodies { + for _, block := range body.Blocks { + count += int32(len(block.Instructions)) + } + } + return count +} diff --git a/core/opentaint-ir/go/proto/goir/service.proto b/core/opentaint-ir/go/proto/goir/service.proto index 8a851d0a7f..05c5647021 100644 --- a/core/opentaint-ir/go/proto/goir/service.proto +++ b/core/opentaint-ir/go/proto/goir/service.proto @@ -36,6 +36,7 @@ message BuildProgramRequest { string working_dir = 8; LoadMode mode = 11; repeated string project_module_paths = 12; + repeated string model_dirs = 13; } message BuildProgramResponse { @@ -46,6 +47,7 @@ message BuildProgramResponse { ProtoBuildSummary summary = 4; ProtoError error = 5; ProtoProgram program = 6; + ProtoModelProgram model_program = 7; } } @@ -75,6 +77,12 @@ message ProtoProgram { repeated ProtoFunctionBody function_bodies = 3; } +// A model program is loaded by the Go server and resolved by the Kotlin client. +message ProtoModelProgram { + string source = 1; + ProtoProgram program = 2; +} + // ─── Types ────────────────────────────────────────────────────────── message ProtoTypeDefinition { @@ -262,6 +270,7 @@ message ProtoTypeParamDecl { string name = 1; int32 index = 2; int32 constraint_type_id = 3; + int32 type_id = 4; } message ProtoFunction { @@ -283,6 +292,8 @@ message ProtoFunction { repeated ProtoTypeParamDecl type_params = 16; bool has_body = 17; ProtoPosition position = 18; + int32 origin_function_id = 19; + repeated int32 type_arg_ids = 20; } message ProtoParam { diff --git a/core/opentaint-ir/opentaint-ir-approximations/src/main/kotlin/org/opentaint/ir/approximation/Approximations.kt b/core/opentaint-ir/opentaint-ir-approximations/src/main/kotlin/org/opentaint/ir/approximation/Approximations.kt index 43c3c4a33d..2eef10ad54 100644 --- a/core/opentaint-ir/opentaint-ir-approximations/src/main/kotlin/org/opentaint/ir/approximation/Approximations.kt +++ b/core/opentaint-ir/opentaint-ir-approximations/src/main/kotlin/org/opentaint/ir/approximation/Approximations.kt @@ -147,7 +147,19 @@ class Approximations( clazz.getCompressed("nameId") to annotationValue.getCompressedBlob("classSymbolId") } - namedApprox + classApprox + val explicitMappings = (namedApprox + classApprox).toList() + val explicitlyMappedClasses = explicitMappings.mapTo(hashSetOf()) { it.first } + val prefixMappings = context.txn.all("Class").mapNotNull { clazz -> + val approximationNameId = clazz.getCompressed("nameId") + ?: return@mapNotNull null + if (approximationNameId in explicitlyMappedClasses) return@mapNotNull null + val originalName = prefixAddressedTarget(persistence.findSymbolName(approximationNameId)) + ?: return@mapNotNull null + + approximationNameId to persistence.findSymbolId(originalName) + } + + explicitMappings.asSequence() + prefixMappings } ).forEach { (approximation, original) -> val approximationClassName = persistence.findSymbolName(approximation!!).toApproximationName() @@ -361,19 +373,28 @@ private class ApproximationIndexer( } override fun index(classNode: ClassNode) { - val annotations = classNode.visibleAnnotations ?: return - - // Check whether the classNode contains an approximation related annotation - val approximationAnnotation = annotations.singleOrNull { - approximationAnnotationClassName in it.desc.className || approximationByNameAnnotationClassName in it.desc.className - } ?: return + val approximationClassName = classNode.name.className + val prefixTarget = prefixAddressedTarget(approximationClassName) + val approximationAnnotations = classNode.visibleAnnotations.orEmpty().filter { + approximationAnnotationClassName in it.desc.className || + approximationByNameAnnotationClassName in it.desc.className + } + if (prefixTarget == null && approximationAnnotations.isEmpty()) return + if (approximationAnnotations.size > 1) { + logger.error { "Approximation $approximationClassName has more than one class-address annotation" } + return + } + val approximationAnnotation = approximationAnnotations.singleOrNull() - if (!checkVersion(approximationAnnotation)) + if (approximationAnnotation != null && !checkVersion(approximationAnnotation)) return val originalClassName = when { + approximationAnnotation == null -> prefixTarget!!.toOriginalName() + approximationByNameAnnotationClassName in approximationAnnotation.desc.className -> { - val target = approximationAnnotation.values[1] as String + val target = annotationStringValueByName(approximationAnnotation.values, "value") + ?: error("unable to find `value` in @$approximationByNameAnnotationClassName") target.toOriginalName() } @@ -386,12 +407,12 @@ private class ApproximationIndexer( else -> error("Impossible") } - val approximationClassName = classNode.name.className.toApproximationName() + val typedApproximationClassName = approximationClassName.toApproximationName() // Ensure that each approximation has one and only one - if (originalToApproximation.getOrDefault(originalClassName, approximationClassName) != approximationClassName) { + if (originalToApproximation.getOrDefault(originalClassName, typedApproximationClassName) != typedApproximationClassName) { logger.error { - "An error occurred during approximations indexing: you tried to add `$approximationClassName` " + + "An error occurred during approximations indexing: you tried to add `$typedApproximationClassName` " + "as an approximation for `$originalClassName`, but the target class is already " + "associated with approximation `${originalToApproximation[originalClassName]}`. " + "Only bijection between classes is allowed." @@ -399,18 +420,18 @@ private class ApproximationIndexer( return } - if (approximationToOriginal.getOrDefault(approximationClassName, originalClassName) != originalClassName) { + if (approximationToOriginal.getOrDefault(typedApproximationClassName, originalClassName) != originalClassName) { logger.error { - "An error occurred during approximations indexing: you tried to add `$approximationClassName` " + + "An error occurred during approximations indexing: you tried to add `$typedApproximationClassName` " + "as an approximation for `$originalClassName`, but this approximation is already used for " + - "`${approximationToOriginal[approximationClassName]}`. " + + "`${approximationToOriginal[typedApproximationClassName]}`. " + "Only bijection between classes is allowed." } return } - originalToApproximation[originalClassName] = approximationClassName - approximationToOriginal[approximationClassName] = originalClassName + originalToApproximation[originalClassName] = typedApproximationClassName + approximationToOriginal[typedApproximationClassName] = originalClassName } override fun flush(context: StorageContext) { @@ -428,6 +449,12 @@ private val approximatedMethodName = ApproximatedMethodName::class.qualifiedName private val approximatedFieldName = ApproximatedFieldName::class.qualifiedName!! private val versionAnnotationClassName = Version::class.qualifiedName!! +private const val approximationPackagePrefix = "opentaint." + +private fun prefixAddressedTarget(approximationClassName: String): String? = + approximationClassName.takeIf { it.startsWith(approximationPackagePrefix) } + ?.removePrefix(approximationPackagePrefix)?.takeIf { it.isNotEmpty() } + @JvmInline value class ApproximationClassName(val className: String) { override fun toString(): String = className diff --git a/core/opentaint-ir/opentaint-ir-approximations/src/test/java/opentaint/java/lang/Integer.java b/core/opentaint-ir/opentaint-ir-approximations/src/test/java/opentaint/java/lang/Integer.java new file mode 100644 index 0000000000..6d094a0fa9 --- /dev/null +++ b/core/opentaint-ir/opentaint-ir-approximations/src/test/java/opentaint/java/lang/Integer.java @@ -0,0 +1,17 @@ +package opentaint.java.lang; + +public class Integer { + private final int value; + + public Integer(int value) { + this.value = value; + } + + public static Integer valueOf(int value) { + return new Integer(value); + } + + public int getValue() { + return value; + } +} diff --git a/core/opentaint-ir/opentaint-ir-approximations/src/test/java/opentaint/org/opentaint/ir/approximations/target/KotlinClassName.java b/core/opentaint-ir/opentaint-ir-approximations/src/test/java/opentaint/org/opentaint/ir/approximations/target/KotlinClassName.java new file mode 100644 index 0000000000..faa32b0790 --- /dev/null +++ b/core/opentaint-ir/opentaint-ir-approximations/src/test/java/opentaint/org/opentaint/ir/approximations/target/KotlinClassName.java @@ -0,0 +1,10 @@ +package opentaint.org.opentaint.ir.approximations.target; + +import org.opentaint.ir.approximation.annotation.ApproximateByName; + +@ApproximateByName("org.opentaint.ir.approximations.target.Kotlin-Class") +public class KotlinClassName { + public int evaluate(int value) { + return value; + } +} diff --git a/core/opentaint-ir/opentaint-ir-approximations/src/test/java/opentaint/org/opentaint/ir/approximations/target/KotlinNames.java b/core/opentaint-ir/opentaint-ir-approximations/src/test/java/opentaint/org/opentaint/ir/approximations/target/KotlinNames.java new file mode 100644 index 0000000000..7fa17c26b7 --- /dev/null +++ b/core/opentaint-ir/opentaint-ir-approximations/src/test/java/opentaint/org/opentaint/ir/approximations/target/KotlinNames.java @@ -0,0 +1,14 @@ +package opentaint.org.opentaint.ir.approximations.target; + +import org.opentaint.ir.approximation.annotation.ApproximatedFieldName; +import org.opentaint.ir.approximation.annotation.ApproximatedMethodName; + +public class KotlinNames { + @ApproximatedFieldName("field-with-dash") + public int fieldWithDash; + + @ApproximatedMethodName("method-with-dash") + public int methodWithDash(int value) { + return value; + } +} diff --git a/core/opentaint-ir/opentaint-ir-approximations/src/test/java/org/opentaint/ir/approximations/approx/IntegerApprox.java b/core/opentaint-ir/opentaint-ir-approximations/src/test/java/org/opentaint/ir/approximations/approx/IntegerApprox.java deleted file mode 100644 index 22a28e859e..0000000000 --- a/core/opentaint-ir/opentaint-ir-approximations/src/test/java/org/opentaint/ir/approximations/approx/IntegerApprox.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.opentaint.ir.approximations.approx; - -import org.opentaint.ir.approximation.annotation.Approximate; - - -@Approximate(java.lang.Integer.class) -public class IntegerApprox { - private final int value; - - public IntegerApprox(int value) { - this.value = value; - } - - public static IntegerApprox valueOf(int value) { - return new IntegerApprox(value); - } - - public int getValue() { - return value; - } -} diff --git a/core/opentaint-ir/opentaint-ir-approximations/src/test/java/org/opentaint/ir/approximations/legacy/LegacyTargetApprox.java b/core/opentaint-ir/opentaint-ir-approximations/src/test/java/org/opentaint/ir/approximations/legacy/LegacyTargetApprox.java new file mode 100644 index 0000000000..978840e742 --- /dev/null +++ b/core/opentaint-ir/opentaint-ir-approximations/src/test/java/org/opentaint/ir/approximations/legacy/LegacyTargetApprox.java @@ -0,0 +1,11 @@ +package org.opentaint.ir.approximations.legacy; + +import org.opentaint.ir.approximation.annotation.Approximate; +import org.opentaint.ir.approximations.target.LegacyTarget; + +@Approximate(LegacyTarget.class) +public class LegacyTargetApprox { + public int identity(int value) { + return value; + } +} diff --git a/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/org/opentaint/ir/approximations/approx/KotlinClassApprox.kt b/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/opentaint/org/opentaint/ir/approximations/target/KotlinClass.kt similarity index 72% rename from core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/org/opentaint/ir/approximations/approx/KotlinClassApprox.kt rename to core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/opentaint/org/opentaint/ir/approximations/target/KotlinClass.kt index 47240c4c9e..d82b7cf254 100644 --- a/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/org/opentaint/ir/approximations/approx/KotlinClassApprox.kt +++ b/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/opentaint/org/opentaint/ir/approximations/target/KotlinClass.kt @@ -1,17 +1,16 @@ -package org.opentaint.ir.approximations.approx +package opentaint.org.opentaint.ir.approximations.target -import org.opentaint.ir.approximation.annotation.Approximate +import opentaint.java.lang.Integer as IntegerModel import org.opentaint.ir.approximations.target.ClassForField -import org.opentaint.ir.approximations.target.KotlinClass +import org.opentaint.ir.approximations.target.KotlinClass as TargetKotlinClass import org.jetbrains.annotations.NotNull -@Approximate(KotlinClass::class) -class KotlinClassApprox { +class KotlinClass { @NotNull private val artificialField: ClassForField = ClassForField() private val fieldToReplace: Int = 3 - private val sameApproximation: KotlinClassApprox? = null - private val anotherApproximation: IntegerApprox? = null + private val sameApproximation: KotlinClass? = null + private val anotherApproximation: IntegerModel? = null fun replaceBehaviour(value: Int): Int = 42 @@ -25,7 +24,7 @@ class KotlinClassApprox { return 0 } - fun useSameApproximationTarget(kotlinClass: KotlinClass): Int { + fun useSameApproximationTarget(kotlinClass: TargetKotlinClass): Int { if (sameApproximation == null) return 0 if (kotlinClass.methodWithoutApproximation() == sameApproximation.artificialMethod()) { @@ -52,4 +51,4 @@ class KotlinClassApprox { return 2 } -} \ No newline at end of file +} diff --git a/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/org/opentaint/ir/approximations/ApproximationsTest.kt b/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/org/opentaint/ir/approximations/ApproximationsTest.kt index c87adc8d5c..008c6d6133 100644 --- a/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/org/opentaint/ir/approximations/ApproximationsTest.kt +++ b/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/org/opentaint/ir/approximations/ApproximationsTest.kt @@ -3,11 +3,13 @@ package org.opentaint.ir.approximations import org.opentaint.ir.api.jvm.JavaVersion import org.opentaint.ir.api.jvm.LocationType import org.opentaint.ir.api.jvm.cfg.JIRAssignInst +import org.opentaint.ir.api.jvm.cfg.JIRArgument import org.opentaint.ir.api.jvm.cfg.JIRCallInst import org.opentaint.ir.api.jvm.cfg.JIRFieldRef import org.opentaint.ir.api.jvm.cfg.JIRRawAssignInst import org.opentaint.ir.api.jvm.cfg.JIRRawCallInst import org.opentaint.ir.api.jvm.cfg.JIRRawFieldRef +import org.opentaint.ir.api.jvm.cfg.JIRReturnInst import org.opentaint.ir.api.jvm.ext.findClass import org.opentaint.ir.api.jvm.ext.findDeclaredFieldOrNull import org.opentaint.ir.approximation.Approximations @@ -16,6 +18,8 @@ import org.opentaint.ir.approximation.JIREnrichedVirtualMethod import org.opentaint.ir.approximation.toApproximationName import org.opentaint.ir.approximation.toOriginalName import org.opentaint.ir.approximations.target.KotlinClass +import org.opentaint.ir.approximations.target.KotlinNames +import org.opentaint.ir.approximations.target.LegacyTarget import org.opentaint.ir.impl.fs.JarLocation import org.opentaint.ir.testing.BaseTest import org.opentaint.ir.testing.WithDb @@ -45,6 +49,7 @@ open class ApproximationsTest : BaseTest() { val approximation = approximations.findApproximationByOriginOrNull(originalClassName) assertNotNull(approximation) + assertEquals("opentaint.org.opentaint.ir.approximations.target.KotlinClass", approximation) assertEquals(classes.name, approximations.findOriginalByApproximationOrNull(approximation!!.toApproximationName())) } @@ -57,9 +62,19 @@ open class ApproximationsTest : BaseTest() { val approximation = approximations.findApproximationByOriginOrNull(originalClassName) assertNotNull(approximation) + assertEquals("opentaint.java.lang.Integer", approximation) assertEquals(classec.name, approximations.findOriginalByApproximationOrNull(approximation!!.toApproximationName())) } + @Test + fun `legacy annotation addressing remains supported`() { + val target = cp.findClass() + val approximation = approximations.findApproximationByOriginOrNull(target.name.toOriginalName()) + + assertEquals("org.opentaint.ir.approximations.legacy.LegacyTargetApprox", approximation) + assertTrue(target.declaredMethods.single { it.name == "identity" } is JIREnrichedVirtualMethod) + } + @Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN") @Test fun `integer approximation`() { @@ -126,6 +141,40 @@ open class ApproximationsTest : BaseTest() { assertEquals(methods, classec.declaredMethods) } + @Test + fun `java approximation patches kotlin names that java cannot express`() { + val clazz = cp.findClass() + + val field = clazz.declaredFields.single { it.name == "field-with-dash" } + assertTrue(field is JIREnrichedVirtualField) + + val method = clazz.declaredMethods.single { it.name == "method-with-dash" } + assertTrue(method is JIREnrichedVirtualMethod) + + val returnInstruction = method.flowGraph().instructions + .flatMap { instruction -> + when (instruction) { + is JIRReturnInst -> listOf(instruction) + else -> emptyList() + } + } + .single() + val returnedArgument = returnInstruction.returnValue as JIRArgument + assertEquals(0, returnedArgument.index) + assertEquals("value", returnedArgument.name) + } + + @Test + fun `java approximation patches kotlin class name that java cannot express`() { + val clazz = cp.findClass("org.opentaint.ir.approximations.target.Kotlin-Class") + val method = clazz.declaredMethods.single { it.name == "evaluate" } + + assertTrue(method is JIREnrichedVirtualMethod) + val returnInstruction = method.flowGraph().instructions.filterIsInstance().single() + val returnedArgument = returnInstruction.returnValue as JIRArgument + assertEquals(0, returnedArgument.index) + } + @Test fun `replace approximations in methodBody`() { val classec = cp.findClass() @@ -136,7 +185,7 @@ open class ApproximationsTest : BaseTest() { val rawInstructions = method.rawInstList assertTrue(method.enclosingClass === classec) - assertTrue("KotlinClassApprox" !in method.description) + assertTrue("opentaint." !in method.description) val types = hashSetOf() diff --git a/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/org/opentaint/ir/approximations/target/KotlinNames.kt b/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/org/opentaint/ir/approximations/target/KotlinNames.kt new file mode 100644 index 0000000000..434cb82af7 --- /dev/null +++ b/core/opentaint-ir/opentaint-ir-approximations/src/test/kotlin/org/opentaint/ir/approximations/target/KotlinNames.kt @@ -0,0 +1,16 @@ +package org.opentaint.ir.approximations.target + +class KotlinNames { + @JvmField + val `field-with-dash`: Int = -1 + + fun `method-with-dash`(value: Int): Int = -value +} + +class `Kotlin-Class` { + fun evaluate(value: Int): Int = -value +} + +class LegacyTarget { + fun identity(value: Int): Int = -value +} diff --git a/core/src/main/kotlin/org/opentaint/go/sast/dataflow/GoUnitResolver.kt b/core/src/main/kotlin/org/opentaint/go/sast/dataflow/GoUnitResolver.kt index d79c3c3083..a432e0782a 100644 --- a/core/src/main/kotlin/org/opentaint/go/sast/dataflow/GoUnitResolver.kt +++ b/core/src/main/kotlin/org/opentaint/go/sast/dataflow/GoUnitResolver.kt @@ -4,12 +4,17 @@ import org.opentaint.dataflow.ifds.UnitResolver import org.opentaint.dataflow.ifds.UnitType import org.opentaint.dataflow.ifds.UnknownUnit import org.opentaint.ir.go.api.GoIRFunction +import org.opentaint.ir.go.api.GoIRModelFunction class GoUnitResolver : UnitResolver { data class GoPackageUnit(val pkgImportPath: String) : UnitType override fun resolve(method: GoIRFunction): UnitType { val pkg = method.pkg ?: return UnknownUnit - return if (pkg.isProject) GoPackageUnit(pkg.importPath) else UnknownUnit + return if (pkg.isProject || method is GoIRModelFunction) { + GoPackageUnit(pkg.importPath) + } else { + UnknownUnit + } } } diff --git a/core/src/main/kotlin/org/opentaint/go/sast/project/AnalysisCtx.kt b/core/src/main/kotlin/org/opentaint/go/sast/project/AnalysisCtx.kt index 8cfb8c73b8..9bf43dec37 100644 --- a/core/src/main/kotlin/org/opentaint/go/sast/project/AnalysisCtx.kt +++ b/core/src/main/kotlin/org/opentaint/go/sast/project/AnalysisCtx.kt @@ -1,22 +1,35 @@ package org.opentaint.go.sast.project import mu.KLogging +import org.opentaint.go.config.GoDefaultModelLoader import org.opentaint.ir.go.api.GoIRProgram import org.opentaint.ir.go.client.GoIRClient import org.opentaint.ir.go.client.GoIRLoadConfig import org.opentaint.ir.go.client.GoIRLoadMode import org.opentaint.project.GoProject +import java.nio.file.Path class AnalysisCtx( private val prj: GoProject, val client: GoIRClient, + private val modelPaths: List = emptyList(), + private val useDefaultModels: Boolean = true, ) : AutoCloseable by client { val cp: GoIRProgram by lazy { logger.info { "Building Go IR for project: ${prj.projectDir}" } - client.buildFromDir(prj.projectDir, GoIRLoadConfig(mode = GoIRLoadMode.PROJECT)).program + client.buildFromDir( + prj.projectDir, + GoIRLoadConfig( + mode = GoIRLoadMode.PROJECT, + modelDirs = defaultModelPaths() + modelPaths, + ), + ).program } + private fun defaultModelPaths(): List = + if (useDefaultModels) GoDefaultModelLoader.modelPaths() else emptyList() + companion object { private val logger = object : KLogging() {}.logger } -} \ No newline at end of file +} diff --git a/core/src/main/kotlin/org/opentaint/go/sast/project/GoProjectAnalysisOptions.kt b/core/src/main/kotlin/org/opentaint/go/sast/project/GoProjectAnalysisOptions.kt index 354919aa61..caf525b871 100644 --- a/core/src/main/kotlin/org/opentaint/go/sast/project/GoProjectAnalysisOptions.kt +++ b/core/src/main/kotlin/org/opentaint/go/sast/project/GoProjectAnalysisOptions.kt @@ -1,7 +1,10 @@ package org.opentaint.go.sast.project import org.opentaint.common.sast.CommonAnalysisOptions +import java.nio.file.Path data class GoProjectAnalysisOptions( val common: CommonAnalysisOptions = CommonAnalysisOptions(), + val modelPaths: List = emptyList(), + val useDefaultModels: Boolean = true, ) diff --git a/core/src/main/kotlin/org/opentaint/go/sast/project/GoProjectAnalyzer.kt b/core/src/main/kotlin/org/opentaint/go/sast/project/GoProjectAnalyzer.kt index d8c026065d..ca771304b2 100644 --- a/core/src/main/kotlin/org/opentaint/go/sast/project/GoProjectAnalyzer.kt +++ b/core/src/main/kotlin/org/opentaint/go/sast/project/GoProjectAnalyzer.kt @@ -30,9 +30,12 @@ class GoProjectAnalyzer( results, goOptions.common ) { + private val modelPaths = goOptions.modelPaths + private val useDefaultModels = goOptions.useDefaultModels + override fun initializeProjectAnalysisContext(): AnalysisCtx { val client = GoIRClient() - return AnalysisCtx(project, client) + return AnalysisCtx(project, client, modelPaths, useDefaultModels) } override fun AnalysisCtx.selectProjectEntryPoints(): List { diff --git a/core/src/main/kotlin/org/opentaint/go/sast/project/GoTestProjectAnalyzer.kt b/core/src/main/kotlin/org/opentaint/go/sast/project/GoTestProjectAnalyzer.kt index bd1d834d66..4c4eaf4b30 100644 --- a/core/src/main/kotlin/org/opentaint/go/sast/project/GoTestProjectAnalyzer.kt +++ b/core/src/main/kotlin/org/opentaint/go/sast/project/GoTestProjectAnalyzer.kt @@ -43,6 +43,8 @@ class GoTestProjectAnalyzer( ) : TestProjectAnalyzerBase( project, results, providedOptions.common.copy(storeSummaries = false), ) { + private val modelPaths = providedOptions.modelPaths + private val useDefaultModels = providedOptions.useDefaultModels private val loadedRules = options.loadSemgrepRules(GoLanguageStrategy()) @Serializable @@ -77,7 +79,7 @@ class GoTestProjectAnalyzer( Yaml().decodeFromStream(it) } - val ctx = AnalysisCtx(project, GoIRClient()) + val ctx = AnalysisCtx(project, GoIRClient(), modelPaths, useDefaultModels) val results = ctx.analyzeTestSamples(ruleTests.tests) ?: return ProjectAnalysisStatus.EXCEPTION diff --git a/core/src/main/kotlin/org/opentaint/jvm/sast/runner/ProjectAnalyzerRunner.kt b/core/src/main/kotlin/org/opentaint/jvm/sast/runner/ProjectAnalyzerRunner.kt index d2f3533df7..c6a69d44e4 100644 --- a/core/src/main/kotlin/org/opentaint/jvm/sast/runner/ProjectAnalyzerRunner.kt +++ b/core/src/main/kotlin/org/opentaint/jvm/sast/runner/ProjectAnalyzerRunner.kt @@ -61,6 +61,13 @@ class ProjectAnalyzerRunner : AbstractAnalyzerRunner() { .directory() .multiple() + private val goModels: List by option(help = "Directory containing opentaint/ Go model sources") + .directory() + .multiple() + + private val disableDefaultGoModels: Boolean by option(help = "Disable bundled Go models") + .flag(default = false) + private val semgrepRuleLoadTrace: Path? by option(help = "Output file for Semgrep rules loader trace") .newFile() @@ -126,7 +133,11 @@ class ProjectAnalyzerRunner : AbstractAnalyzerRunner() { ), ) - override fun goOptions() = GoProjectAnalysisOptions(common = commonOptions) + override fun goOptions() = GoProjectAnalysisOptions( + common = commonOptions, + modelPaths = goModels, + useDefaultModels = !disableDefaultGoModels, + ) private fun collectYamlConfigs(path: Path): List = path.walk() .filter { it.extension in arrayOf("yaml", "yml") } diff --git a/core/src/test/kotlin/org/opentaint/go/sast/dataflow/GoUnitResolverTest.kt b/core/src/test/kotlin/org/opentaint/go/sast/dataflow/GoUnitResolverTest.kt index 0782671cf4..c8cf8bb050 100644 --- a/core/src/test/kotlin/org/opentaint/go/sast/dataflow/GoUnitResolverTest.kt +++ b/core/src/test/kotlin/org/opentaint/go/sast/dataflow/GoUnitResolverTest.kt @@ -2,6 +2,8 @@ package org.opentaint.go.sast.dataflow import org.junit.jupiter.api.Test import org.opentaint.dataflow.ifds.UnknownUnit +import org.opentaint.ir.go.api.GoIRModelFunction +import org.opentaint.ir.go.api.GoIRApproximatedFunction import org.opentaint.ir.go.api.GoIRConst import org.opentaint.ir.go.api.GoIRFreeVar import org.opentaint.ir.go.api.GoIRFunction @@ -42,6 +44,19 @@ class GoUnitResolverTest { assertEquals(UnknownUnit, resolver.resolve(fn)) } + @Test + fun `typed models are analyzed in dependency and standard library packages`() { + val resolver = GoUnitResolver() + val dependency = fakeFunction(fakePackage("example.com/lib", isStdlib = false, isDependency = true)) + val helper = object : GoIRModelFunction, GoIRFunction by dependency {} + assertEquals(GoUnitResolver.GoPackageUnit("example.com/lib"), resolver.resolve(helper)) + val stdlib = fakeFunction(fakePackage("strings", isStdlib = true, isDependency = false)) + val approximation = object : GoIRApproximatedFunction, GoIRFunction by stdlib { + override val original = stdlib + } + assertEquals(GoUnitResolver.GoPackageUnit("strings"), resolver.resolve(approximation)) + } + private fun fakePackage(importPath: String, isStdlib: Boolean, isDependency: Boolean): GoIRPackage = object : GoIRPackage { override val importPath = importPath