ethdebug: Build the semantic debug info side table of a contract

docs/compiler-guide/ethdebug.rst

@@ -170,3 +170,22 @@ Transient storage
 
 Slot arithmetic on literal slots is folded into the literals, so that the members of a struct at slot 3 are addressed as ``0x03``, ``0x04`` and so on rather than as sums.
 The test cases under ``test/libsolidity/ethdebugTests/resources/`` show the complete type and pointer tables for each of these layouts and are the reference for their exact form.
+
+Scope Records
+=============
+
+``ethdebug::semanticDebugDataTable()`` builds the side table described in :doc:`ethdebug_internal_debug_info` from analysis results.
+It holds the scope records of a contract, all with instance 0:
+
+- the contract's own, keyed by its AST ID, with the state variables in storage and transient storage, located as in the program-level context,
+- one for every function and modifier compiled into the contract, keyed by its AST ID, with its parameters and return variables in declaration order.
+
+A scope without variables has no record.
+
+Parameters and return variables live on the stack.
+Their pointer is a ``stack`` region whose slot is a ``$$yulLocal`` expression naming the generated Yul local of the unoptimized IR, as ``IRVariable`` names it.
+A value with several stack slots, such as a calldata array or an external function, is a group of regions named after its stack items, such as ``offset`` and ``length``.
+A value without stack slots is optimized out.
+
+Immutables and local variables of blocks have no records yet.
+The test cases ``test/libsolidity/ethdebugTests/semantic_*.sol`` show the records.

libsolidity/interface/Ethdebug.cpp

@@ -21,16 +21,19 @@
 #include <libsolidity/ast/AST.h>
 #include <libsolidity/ast/TypeProvider.h>
 #include <libsolidity/ast/Types.h>
+#include <libsolidity/codegen/ir/IRVariable.h>
 
 #include <liblangutil/SemanticDebugData.h>
 
 #include <libsolutil/Numeric.h>
 
 #include <algorithm>
+#include <functional>
 #include <memory>
 #include <optional>
 #include <set>
 #include <string>
+#include <tuple>
 #include <utility>
 #include <vector>
 
@@ -783,6 +786,108 @@ void registerCallableTypes(TypeRegistry& _types, CallableDeclaration const& _cal
 				_types.registerType(*returnParameter->annotation().type);
 }
 
+/// Calls @a _visit for every function and modifier compiled into @a _contract:
+/// those of its linearized base contracts, since inherited definitions are
+/// compiled into the most derived contract, and the free functions and the
+/// internal library functions of the source units it references.
+void forEachCompiledCallable(
+	ContractDefinition const& _contract,
+	std::function<void(CallableDeclaration const&)> const& _visit
+)
+{
+	for (ContractDefinition const* contract: _contract.annotation().linearizedBaseContracts)
+	{
+		for (FunctionDefinition const* function: contract->definedFunctions())
+			_visit(*function);
+		for (ModifierDefinition const* modifier: contract->functionModifiers())
+			_visit(*modifier);
+	}
+
+	// A library's external functions are compiled into the library itself.
+	SourceUnit const& sourceUnit = _contract.sourceUnit();
+	std::set<SourceUnit const*> sourceUnits = sourceUnit.referencedSourceUnits(true);
+	sourceUnits.insert(&sourceUnit);
+	for (SourceUnit const* unit: sourceUnits)
+		for (ASTPointer<ASTNode> const& node: unit->nodes())
+		{
+			if (auto const* freeFunction = dynamic_cast<FunctionDefinition const*>(node.get()))
+				_visit(*freeFunction);
+			else if (auto const* library = dynamic_cast<ContractDefinition const*>(node.get()); library && library->isLibrary())
+				for (FunctionDefinition const* function: library->definedFunctions())
+					if (function->visibility() <= Visibility::Internal)
+						_visit(*function);
+		}
+}
+
+/// The pointer of a value on the stack, held in the generated Yul locals of
+/// @a _variable, which stand in for stack slots until Yul is compiled to EVM:
+/// a region of the local for a value in one stack slot, the pointer of its
+/// only stack item for a value with one, otherwise a group of the pointers of
+/// its stack items, named after the items and prefixed with @a _name.
+/// @returns nothing for a value without stack slots.
+std::optional<Pointer> stackPointer(IRVariable const& _variable, std::optional<std::string> const& _name)
+{
+	std::vector<std::tuple<std::string, Type const*>> const& items = _variable.type().stackItems();
+	if (items.empty())
+		return std::nullopt;
+	if (items.size() == 1 && !std::get<1>(items.front()))
+		return region(
+			schema::Pointer::Location::Stack,
+			_name,
+			Expression{schema::Pointer::YulLocal{_variable.name()}}
+		);
+	if (items.size() == 1)
+		return stackPointer(_variable.part(std::get<0>(items.front())), _name);
+
+	std::vector<Pointer> members;
+	for (auto const& [itemName, itemType]: items)
+	{
+		solAssert(itemType && !itemName.empty(), "A stack item of a value in several slots has a name and a type.");
+		std::string const name = _name ? *_name + "-" + itemName : itemName;
+		if (std::optional<Pointer> member = stackPointer(_variable.part(itemName), name))
+			members.emplace_back(std::move(*member));
+	}
+	if (members.empty())
+		return std::nullopt;
+	return group(std::move(members));
+}
+
+/// The record of a parameter or return variable: materialized in the Yul
+/// locals of its stack slots, or optimized out if it has none.
+langutil::SemanticDebugVariable stackVariableRecord(TypeRegistry& _types, VariableDeclaration const& _variable)
+{
+	solAssert(_variable.annotation().type);
+	langutil::SemanticDebugVariable record;
+	if (!_variable.name().empty())
+		record.identifier = _variable.name();
+	record.declarationASTID = _variable.id();
+	record.declarationSourceRange = _types.sourceRange(_variable.location());
+	if (_types.registerType(*_variable.annotation().type))
+		record.typeID = TypeRegistry::canonical(*_variable.annotation().type).identifier();
+	if (std::optional<Pointer> pointer = stackPointer(IRVariable{_variable}, std::nullopt))
+	{
+		record.phase = langutil::SemanticDebugVariablePhase::Materialized;
+		record.pointer = std::move(*pointer);
+	}
+	else
+		record.phase = langutil::SemanticDebugVariablePhase::OptimizedOut;
+	return record;
+}
+
+/// The scope record of a function or modifier: its parameters and return
+/// variables, in that order.
+langutil::SemanticDebugScope callableScope(TypeRegistry& _types, CallableDeclaration const& _callable)
+{
+	langutil::SemanticDebugScope scope;
+	for (ASTPointer<VariableDeclaration> const& parameter: _callable.parameters())
+		scope.variableDefinitions.emplace_back(stackVariableRecord(_types, *parameter));
+	// Modifiers have no return parameter list.
+	if (_callable.returnParameterList())
+		for (ASTPointer<VariableDeclaration> const& returnParameter: _callable.returnParameters())
+			scope.variableDefinitions.emplace_back(stackVariableRecord(_types, *returnParameter));
+	return scope;
+}
+
 }
 
 void ethdebug::Resources::merge(Resources _other)
@@ -905,33 +1010,38 @@ ethdebug::Resources ethdebug::resources(ContractDefinition const& _contract, std
 	for (VariableDeclaration const* variable: contractType->immutableVariables())
 		types.registerType(*variable->annotation().type);
 
-	// Inherited functions and modifiers are compiled into the most derived
-	// contract, so every linearized base contract contributes its types.
-	for (ContractDefinition const* contract: _contract.annotation().linearizedBaseContracts)
-	{
-		for (FunctionDefinition const* function: contract->definedFunctions())
-			registerCallableTypes(types, *function);
-		for (ModifierDefinition const* modifier: contract->functionModifiers())
-			registerCallableTypes(types, *modifier);
-	}
-
-	// Free functions and internal library functions reachable through imports
-	// are compiled into the contract as well.
-	SourceUnit const& sourceUnit = _contract.sourceUnit();
-	std::set<SourceUnit const*> sourceUnits = sourceUnit.referencedSourceUnits(true);
-	sourceUnits.insert(&sourceUnit);
-	for (SourceUnit const* unit: sourceUnits)
-		for (ASTPointer<ASTNode> const& node: unit->nodes())
-		{
-			if (auto const* freeFunction = dynamic_cast<FunctionDefinition const*>(node.get()))
-				registerCallableTypes(types, *freeFunction);
-			else if (auto const* library = dynamic_cast<ContractDefinition const*>(node.get()); library && library->isLibrary())
-				for (FunctionDefinition const* function: library->definedFunctions())
-					if (function->visibility() <= Visibility::Internal)
-						registerCallableTypes(types, *function);
-		}
+	forEachCompiledCallable(_contract, [&](CallableDeclaration const& _callable) {
+		registerCallableTypes(types, _callable);
+	});
 
 	result.types = types.takeDocuments();
 	result.pointers = templates.takeTemplates();
 	return result;
 }
+
+langutil::SemanticDebugDataTable ethdebug::semanticDebugDataTable(
+	ContractDefinition const& _contract,
+	std::map<std::string, unsigned> const& _sourceIndices
+)
+{
+	langutil::SemanticDebugDataTable table;
+	table.setContractName(_contract.name());
+
+	auto const setScope = [&](int64_t _astID, langutil::SemanticDebugScope _scope) {
+		// A scope without variables has nothing to describe.
+		if (!_scope.variableDefinitions.empty())
+			table.set({_astID, 0}, std::make_shared<langutil::SemanticDebugScope const>(std::move(_scope)));
+	};
+	setScope(_contract.id(), stateVariableScope(_contract, _sourceIndices));
+	TypeRegistry types{_sourceIndices};
+	forEachCompiledCallable(_contract, [&](CallableDeclaration const& _callable) {
+		setScope(_callable.id(), callableScope(types, _callable));
+	});
+
+	Resources contractResources = resources(_contract, _sourceIndices);
+	for (auto& [id, document]: contractResources.types)
+		table.setType(id, std::move(document));
+	for (auto& [id, pointerTemplate]: contractResources.pointers)
+		table.setPointerTemplate(id, std::move(pointerTemplate));
+	return table;
+}

libsolidity/interface/Ethdebug.h

@@ -26,6 +26,7 @@
 
 #include <liblangutil/EthdebugSchema.h>
 #include <liblangutil/SemanticDebugData.h>
+#include <liblangutil/SemanticDebugDataTable.h>
 
 #include <map>
 #include <optional>
@@ -84,4 +85,18 @@ std::optional<langutil::ethdebug::schema::program::Context> programContext(
 	Resources const& _resources
 );
 
+/// The semantic debug info side table of @a _contract, as specified in
+/// docs/compiler-guide/ethdebug_internal_debug_info.rst, together with its
+/// resources(). It holds the scope record of the contract, with the state
+/// variables in storage and transient storage as stateVariableScope()
+/// describes them, and the scope record of every function and modifier
+/// compiled into the contract, with its parameters and return variables.
+/// Those live on the stack: their pointers are regions over the generated Yul
+/// locals of the unoptimized IR, which stand in for stack slots until Yul is
+/// compiled to EVM. Scopes without variables have no record.
+langutil::SemanticDebugDataTable semanticDebugDataTable(
+	ContractDefinition const& _contract,
+	std::map<std::string, unsigned> const& _sourceIndices
+);
+
 }

test/libsolidity/EthdebugTest.cpp

@@ -20,7 +20,9 @@
 #include <test/Common.h>
 
 #include <liblangutil/DebugInfoSelection.h>
+#include <liblangutil/SemanticDebugDataSerialization.h>
 
+#include <libsolidity/interface/Ethdebug.h>
 #include <libsolidity/interface/OptimiserSettings.h>
 
 #include <boost/throw_exception.hpp>
@@ -144,6 +146,11 @@ std::optional<Json> EthdebugTest::fetchOutput(
 				return std::nullopt;
 			return creation["contract"];
 		}
+		if (_outputName == "semantic")
+			return langutil::semanticDebugDataToJson(ethdebug::semanticDebugDataTable(
+				compiler().contractDefinition(*resolved),
+				compiler().sourceIndices()
+			));
 	}
 	return std::nullopt;
 }

test/libsolidity/EthdebugTest.h

@@ -39,7 +39,9 @@ namespace solidity::frontend::test
 ///
 /// Scope keys exposed to expectations:
 ///   - Globals: `.resources`, `.compilation`.
-///   - Per contract: `Contract.creation`, `Contract.runtime`, `Contract.contract`.
+///   - Per contract: `Contract.creation`, `Contract.runtime`, `Contract.contract`,
+///     `Contract.semantic` (the internal semantic debug info side table, in the
+///     shape of its sidecar).
 ///   - Source-qualified per contract (when needed to disambiguate same-named
 ///     contracts in different sources): `source.sol:Contract.creation`, etc.
 class EthdebugTest: public JSONExpectationTest

test/libsolidity/ethdebugTests/semantic_compiled_functions.sol

@@ -0,0 +1,37 @@
+function helper(uint256 input) pure returns (uint256 output) {
+    return input;
+}
+
+library L {
+    function internalHelper(uint256 x) internal pure returns (uint256) {
+        return x;
+    }
+
+    function externalHelper(uint256 y) external pure returns (uint256) {
+        return y;
+    }
+}
+
+contract Base {
+    function inherited(uint256 value) public pure virtual returns (uint256 result) {
+        return value;
+    }
+}
+
+contract C is Base {
+    function callHelpers(uint256 value) public pure returns (uint256 result) {
+        return helper(L.internalHelper(value));
+    }
+}
+// ----
+// C.semantic.scopes | keys: ["11","21","42","60"]
+// C.semantic.scopes.11.0.variableDefinitions[0].identifier: input
+// C.semantic.scopes.11.0.variableDefinitions[1].identifier: output
+// C.semantic.scopes.21.0.variableDefinitions[0].identifier: x
+// C.semantic.scopes.21.0.variableDefinitions[1].identifier: <PATH NOT FOUND>
+// C.semantic.scopes.42.0.variableDefinitions[0].identifier: value
+// C.semantic.scopes.42.0.variableDefinitions[1].identifier: result
+// C.semantic.scopes.60.0.variableDefinitions[0].identifier: value
+// C.semantic.scopes.60.0.variableDefinitions[1].identifier: result
+// L.semantic.scopes | keys: ["11","21","31"]
+// L.semantic.scopes.31.0.variableDefinitions[0].identifier: y

test/libsolidity/ethdebugTests/semantic_contract_scope.sol

@@ -0,0 +1,37 @@
+struct Point { uint8 x; uint8 y; }
+
+contract C {
+    uint256 counter;
+    Point origin;
+    mapping(address => uint256) balances;
+    uint256 transient temporary;
+    uint256 immutable created = 1;
+    uint256 constant LIMIT = 2;
+
+    function f() public view returns (uint256) {
+        return created + LIMIT;
+    }
+}
+// ====
+// EVMVersion: >=cancun
+// ----
+// C.semantic.scopes | keys: ["33","34"]
+// C.semantic.scopes.34.0.variableDefinitions | length: 4
+// C.semantic.scopes.34.0.variableDefinitions[0].identifier: counter
+// C.semantic.scopes.34.0.variableDefinitions[0].pointer: {"location":"storage","slot":"0x00"}
+// C.semantic.scopes.34.0.variableDefinitions[1].identifier: origin
+// C.semantic.scopes.34.0.variableDefinitions[1].pointer: {
+//     "define": {
+//         "slot": "0x01"
+//     },
+//     "in": {
+//         "template": "t_struct$_Point_$6_storage"
+//     }
+// }
+// C.semantic.scopes.34.0.variableDefinitions[2].identifier: balances
+// C.semantic.scopes.34.0.variableDefinitions[2].pointer: {"location":"storage","slot":"0x02"}
+// C.semantic.scopes.34.0.variableDefinitions[3].identifier: temporary
+// C.semantic.scopes.34.0.variableDefinitions[3].pointer: {"location":"transient","slot":"0x00"}
+// C.semantic.scopes.33.0.variableDefinitions | length: 1
+// C.semantic.scopes.33.0.variableDefinitions[0].identifier: <PATH NOT FOUND>
+// C.semantic.resources.pointers | keys: ["t_mapping$_t_address_$_t_uint256_$","t_struct$_Point_$6_storage"]