ethdebug: Attach semantic debug info to Yul nodes and survive the text round trip

Changelog.md

@@ -41,6 +41,7 @@ Compiler Features:
 * General: Replace the current greedy stack shuffler in the experimental SSA CFG pipeline with a planning one.
 * SMTChecker: Emit a deprecation warning for the BMC engine.
 * SMTChecker: Support `block.slotnum`.
+* Yul: Add the `@ast-id-instance` debug info annotation, which distinguishes Yul scopes cloned from the same Solidity definition.
 * Yul Optimizer: `LoopInvariantCodeMotion` can now move expressions depending on function parameters out of loops.
 * Yul Optimizer: `UnusedStoreEliminator` can now recognize redundant memory and storage operations whose start offset or length is a function parameter.
 * Yul Optimizer: Remove the ineffective elimination of unused `returndatacopy()` operations in simple cases from UnusedStoreEliminator.

docs/using-the-compiler.rst

@@ -392,7 +392,8 @@ Input Description
           //     The snippet is quoted and follows the corresponding `@src` annotation.
           //     Depends on `location`; selecting `snippet` without it is an error.
           // - `ast-id`: Annotations of the form `@ast-id <id>` over elements that can be mapped back to a definition in the original Solidity file.
-          //   `<id>` is a node ID in the Solidity AST ('ast' output).
+          //   `<id>` is a node ID in the Solidity AST ('ast' output). A Yul scope cloned from the same definition
+          //   additionally carries `@ast-id-instance <n>` to distinguish it from the original.
           // - `ethdebug`: Ethdebug annotations (experimental). Depends on `ast-id`; selecting
           //   `ethdebug` without `ast-id` is an error. Requesting an ethdebug output does not
           //   change this selection; without `ethdebug` in it the `evm.bytecode.ethdebug` and

liblangutil/DebugData.h

@@ -21,34 +21,45 @@
 #include <liblangutil/SourceLocation.h>
 #include <optional>
 #include <memory>
+#include <utility>
 
 namespace solidity::langutil
 {
 
+struct SemanticDebugScope;
+
 struct DebugData
 {
 	typedef typename std::shared_ptr<DebugData const> ConstPtr;
 
 	explicit DebugData(
 		langutil::SourceLocation _nativeLocation = {},
 		langutil::SourceLocation _originLocation = {},
-		std::optional<int64_t> _astID = {}
+		std::optional<int64_t> _astID = {},
+		std::optional<int64_t> _astIDInstance = {},
+		std::shared_ptr<SemanticDebugScope const> _semanticDebugScope = {}
 	):
 		nativeLocation(std::move(_nativeLocation)),
 		originLocation(std::move(_originLocation)),
-		astID(_astID)
+		astID(_astID),
+		astIDInstance(_astIDInstance),
+		semanticDebugScope(std::move(_semanticDebugScope))
 	{}
 
 	static DebugData::ConstPtr create(
 		langutil::SourceLocation _nativeLocation,
 		langutil::SourceLocation _originLocation = {},
-		std::optional<int64_t> _astID = {}
+		std::optional<int64_t> _astID = {},
+		std::optional<int64_t> _astIDInstance = {},
+		std::shared_ptr<SemanticDebugScope const> _semanticDebugScope = {}
 	)
 	{
 		return std::make_shared<DebugData>(
 			std::move(_nativeLocation),
 			std::move(_originLocation),
-			_astID
+			_astID,
+			_astIDInstance,
+			std::move(_semanticDebugScope)
 		);
 	}
 
@@ -58,13 +69,31 @@ struct DebugData
 		return emptyDebugData;
 	}
 
+	/// The locations of @a _debugData without its source-language identity:
+	/// for nodes the compiler generates rather than derives from a source node,
+	/// which must not inherit the AST ID, instance or semantic scope.
+	static DebugData::ConstPtr locationsOnly(DebugData::ConstPtr const& _debugData)
+	{
+		if (!_debugData)
+			return create();
+		return create(_debugData->nativeLocation, _debugData->originLocation);
+	}
+
 	/// Location in the Yul code.
 	langutil::SourceLocation nativeLocation;
 	/// Location in the original source that the Yul code was produced from.
 	/// Optional. Only present if the Yul source contains location annotations.
 	langutil::SourceLocation originLocation;
 	/// ID in the (Solidity) source AST.
 	std::optional<int64_t> astID;
+	/// Discriminator of the generated scope instance among those sharing
+	/// @a astID, carried in Yul text by the `@ast-id-instance` annotation.
+	/// Unset or zero for code that has not been cloned.
+	std::optional<int64_t> astIDInstance;
+	/// Semantic scope record attached to this node. It has no Yul comment
+	/// form and travels in the ethdebug sidecar instead
+	/// (see liblangutil/SemanticDebugData.h).
+	std::shared_ptr<SemanticDebugScope const> semanticDebugScope;
 };
 
 } // namespace solidity::langutil

libsolidity/codegen/ir/IRGenerator.cpp

@@ -123,7 +123,7 @@ std::string IRGenerator::generate(
 	Whiskers t(R"(<?isEthdebugEnabled>/// ethdebug: enabled</isEthdebugEnabled>
 		/// @use-src <useSrcMapCreation>
 		object "<CreationObject>" {
-			code {
+			code <contractASTIDComment>{
 				<sourceLocationCommentCreation>
 				<memoryInitCreation>
 				<callValueCheck>
@@ -137,7 +137,7 @@ std::string IRGenerator::generate(
 			}
 			/// @use-src <useSrcMapDeployed>
 			object "<DeployedObject>" {
-				code {
+				code <contractASTIDComment>{
 					<sourceLocationCommentDeployed>
 					<memoryInitDeployed>
 					<?library>
@@ -158,6 +158,13 @@ std::string IRGenerator::generate(
 		m_context.registerImmutableVariable(*var);
 
 	t("isEthdebugEnabled", m_context.debugInfoSelection().ethdebug);
+	// The semantic debug info of contract-level variables attaches to the
+	// top-level Yul block of each object, so the block carries the contract's
+	// AST ID whenever that info is transported.
+	t(
+		"contractASTIDComment",
+		m_context.debugInfoSelection().ethdebug ? "/** @ast-id " + std::to_string(_contract.id()) + " */ " : ""
+	);
 	t("CreationObject", IRNames::creationObject(_contract));
 	t("sourceLocationCommentCreation", dispenseLocationComment(_contract));
 	t("library", _contract.isLibrary());

libsolidity/interface/CompilerStack.cpp

@@ -810,7 +810,10 @@ void CompilerStack::link()
 	}
 }
 
-YulStack CompilerStack::loadGeneratedIR(std::string const& _ir) const
+YulStack CompilerStack::loadGeneratedIR(
+	std::string const& _ir,
+	SemanticDebugDataTable const* _semanticDebugData
+) const
 {
 	YulStack stack(
 		m_evmVersion,
@@ -831,6 +834,8 @@ YulStack CompilerStack::loadGeneratedIR(std::string const& _ir) const
 			true   // _withErrorIds
 		) + "\n"
 	);
+	if (_semanticDebugData)
+		stack.attachSemanticDebugData(*_semanticDebugData);
 
 	return stack;
 }
@@ -970,6 +975,12 @@ std::optional<std::string> const& CompilerStack::yulIR(std::string const& _contr
 	return contract(_contractName).yulIR;
 }
 
+std::optional<SemanticDebugDataTable> const& CompilerStack::yulSemanticDebugData(std::string const& _contractName) const
+{
+	solAssert(m_stackState == CompilationSuccessful, "Compilation was not successful.");
+	return contract(_contractName).yulSemanticDebugData;
+}
+
 std::optional<Json> CompilerStack::yulIRAst(std::string const& _contractName) const
 {
 	solAssert(m_stackState == CompilationSuccessful, "Compilation was not successful.");
@@ -1590,7 +1601,20 @@ void CompilerStack::generateIR(ContractDefinition const& _contract, bool _unopti
 	);
 
 	yulAssert(compiledContract.yulIR);
-	YulStack stack = loadGeneratedIR(*compiledContract.yulIR);
+	if (m_debugInfoSelection.ethdebug)
+		compiledContract.yulSemanticDebugData = ethdebug::semanticDebugDataTable(_contract, sourceIndices());
+	else
+		compiledContract.yulSemanticDebugData = std::nullopt;
+
+	YulStack stack = loadGeneratedIR(
+		*compiledContract.yulIR,
+		compiledContract.yulSemanticDebugData ? &*compiledContract.yulSemanticDebugData : nullptr
+	);
+	// The side table pairs with the unoptimized IR: attaching checks every
+	// pointer against it, and the table is kept as attached. The optimized IR,
+	// whose variable names differ, is not a supported pairing yet.
+	if (compiledContract.yulSemanticDebugData)
+		compiledContract.yulSemanticDebugData = stack.semanticDebugData();
 	if (!_unoptimizedOnly)
 	{
 		stack.optimize(m_viaSSACFG);
@@ -1612,7 +1636,10 @@ void CompilerStack::generateEVMFromIR(ContractDefinition const& _contract)
 		return;
 
 	// Re-parse the Yul IR in EVM dialect
-	YulStack stack = loadGeneratedIR(*compiledContract.yulIROptimized);
+	YulStack stack = loadGeneratedIR(
+		*compiledContract.yulIROptimized,
+		compiledContract.yulSemanticDebugData ? &*compiledContract.yulSemanticDebugData : nullptr
+	);
 
 	std::string deployedName = IRNames::deployedObject(_contract);
 	solAssert(!deployedName.empty(), "");

libsolidity/interface/CompilerStack.h

@@ -39,6 +39,7 @@
 #include <liblangutil/DebugInfoSelection.h>
 #include <liblangutil/ErrorReporter.h>
 #include <liblangutil/EVMVersion.h>
+#include <liblangutil/SemanticDebugDataTable.h>
 #include <liblangutil/SourceLocation.h>
 
 #include <libevmasm/AbstractAssemblyStack.h>
@@ -325,6 +326,9 @@ class CompilerStack: public langutil::CharStreamProvider, public evmasm::Abstrac
 	/// @returns the IR representation of a contract.
 	std::optional<std::string> const& yulIR(std::string const& _contractName) const;
 
+	/// @returns the semantic debug data sidecar associated with the contract's Yul IR.
+	std::optional<langutil::SemanticDebugDataTable> const& yulSemanticDebugData(std::string const& _contractName) const;
+
 	/// @returns the IR representation of a contract AST in format.
 	std::optional<Json> yulIRAst(std::string const& _contractName) const;
 
@@ -460,6 +464,7 @@ class CompilerStack: public langutil::CharStreamProvider, public evmasm::Abstrac
 		evmasm::LinkerObject runtimeObject; ///< Runtime object.
 		std::optional<std::string> yulIR; ///< Yul IR code straight from the code generator.
 		std::optional<std::string> yulIROptimized; ///< Reparsed and possibly optimized Yul IR code.
+		std::optional<langutil::SemanticDebugDataTable> yulSemanticDebugData;
 		util::LazyInit<std::string const> metadata; ///< The metadata json that will be hashed into the chain.
 		util::LazyInit<Json const> abi;
 		util::LazyInit<Json const> storageLayout;
@@ -538,7 +543,10 @@ class CompilerStack: public langutil::CharStreamProvider, public evmasm::Abstrac
 	/// Parses and analyzes specified Yul source and returns the YulStack that can be used to manipulate it.
 	/// Assumes that the IR was generated from sources loaded currently into CompilerStack, which
 	/// means that it is error-free and uses the same settings.
-	yul::YulStack loadGeneratedIR(std::string const& _ir) const;
+	yul::YulStack loadGeneratedIR(
+		std::string const& _ir,
+		langutil::SemanticDebugDataTable const* _semanticDebugData = nullptr
+	) const;
 
 	/// @returns the contract object for the given @a _contractName.
 	/// Can only be called after state is CompilationSuccessful.

libyul/AsmParser.cpp

@@ -58,6 +58,18 @@ std::optional<int> toInt(std::string const& _value)
 	}
 }
 
+std::optional<int64_t> toInt64(std::string const& _value)
+{
+	try
+	{
+		return stoll(_value);
+	}
+	catch (std::out_of_range const&)
+	{
+		return std::nullopt;
+	}
+}
+
 }
 
 langutil::DebugData::ConstPtr Parser::createDebugData() const
@@ -69,7 +81,12 @@ langutil::DebugData::ConstPtr Parser::createDebugData() const
 		case UseSourceLocationFrom::LocationOverride:
 			return DebugData::create(m_locationOverride, m_locationOverride);
 		case UseSourceLocationFrom::Comments:
-			return DebugData::create(ParserBase::currentLocation(), m_locationFromComment, m_astIDFromComment);
+			return DebugData::create(
+				ParserBase::currentLocation(),
+				m_locationFromComment,
+				m_astIDFromComment,
+				m_astIDInstanceFromComment
+			);
 	}
 	solAssert(false, "");
 }
@@ -156,6 +173,7 @@ void Parser::fetchDebugDataFromComment()
 	if (commentLiteral.empty())
 	{
 		m_astIDFromComment = std::nullopt;
+		m_astIDInstanceFromComment = std::nullopt;
 		return;
 	}
 	static std::regex const tagRegex = std::regex(
@@ -167,7 +185,8 @@ void Parser::fetchDebugDataFromComment()
 
 	langutil::SourceLocation originLocation = m_locationFromComment;
 	// Empty for each new node.
-	std::optional<int> astID;
+	std::optional<int64_t> astID;
+	std::optional<int64_t> astIDInstance;
 
 	while (regex_search(commentLiteral.cbegin(), commentLiteral.cend(), match, tagRegex))
 	{
@@ -188,13 +207,23 @@ void Parser::fetchDebugDataFromComment()
 			else
 				break;
 		}
+		else if (match[1] == "@ast-id-instance")
+		{
+			if (auto parseResult = parseASTIDInstanceComment(commentLiteral, m_scanner->currentCommentLocation()))
+				tie(commentLiteral, astIDInstance) = *parseResult;
+			else
+				break;
+		}
 		else
 			// Ignore unrecognized tags.
 			continue;
 	}
 
 	m_locationFromComment = originLocation;
 	m_astIDFromComment = astID;
+	// An instance without an accompanying @ast-id on the same node is
+	// malformed and ignored.
+	m_astIDInstanceFromComment = astID ? astIDInstance : std::nullopt;
 }
 
 std::optional<std::pair<std::string_view, SourceLocation>> Parser::parseSrcComment(
@@ -297,36 +326,46 @@ std::optional<std::pair<std::string_view, SourceLocation>> Parser::parseSrcComme
 	return {{tail, SourceLocation{}}};
 }
 
-std::optional<std::pair<std::string_view, std::optional<int>>> Parser::parseASTIDComment(
+std::optional<std::pair<std::string_view, std::optional<int64_t>>> Parser::parseASTIDComment(
 	std::string_view _arguments,
 	langutil::SourceLocation const& _commentLocation
 )
+{
+	return parseIntegerArgumentComment(_arguments, _commentLocation, "@ast-id", 1749_error);
+}
+
+std::optional<std::pair<std::string_view, std::optional<int64_t>>> Parser::parseASTIDInstanceComment(
+	std::string_view _arguments,
+	langutil::SourceLocation const& _commentLocation
+)
+{
+	return parseIntegerArgumentComment(_arguments, _commentLocation, "@ast-id-instance", 9913_error);
+}
+
+std::optional<std::pair<std::string_view, std::optional<int64_t>>> Parser::parseIntegerArgumentComment(
+	std::string_view _arguments,
+	langutil::SourceLocation const& _commentLocation,
+	std::string_view _tag,
+	langutil::ErrorId _errorID
+)
 {
 	static std::regex const argRegex = std::regex(
 		R"~~(^(\d+)(?:\s|$))~~",
 		std::regex_constants::ECMAScript | std::regex_constants::optimize
 	);
 	std::match_results<std::string_view::const_iterator> match;
-	std::optional<int> astID;
-	bool matched = regex_search(_arguments.cbegin(), _arguments.cend(), match, argRegex);
-	std::string_view tail = _arguments;
-	if (matched)
+	if (!regex_search(_arguments.cbegin(), _arguments.cend(), match, argRegex))
 	{
-		solAssert(match.size() == 2, "");
-		tail = _arguments.substr(static_cast<size_t>(match.position() + match.length()));
-
-		astID = toInt(match[1].str());
+		m_errorReporter.syntaxError(_errorID, _commentLocation, "Invalid argument for " + std::string(_tag) + ".");
+		return std::nullopt;
 	}
+	solAssert(match.size() == 2, "");
 
-	if (!matched || !astID || *astID < 0 || static_cast<int64_t>(*astID) != *astID)
-	{
-		m_errorReporter.syntaxError(1749_error, _commentLocation, "Invalid argument for @ast-id.");
-		astID = std::nullopt;
-	}
-	if (matched)
-		return {{_arguments, astID}};
-	else
-		return std::nullopt;
+	// AST IDs and instances are 64-bit; anything larger is not an ID of this compiler.
+	std::optional<int64_t> const value = toInt64(match[1].str());
+	if (!value)
+		m_errorReporter.syntaxError(_errorID, _commentLocation, "Invalid argument for " + std::string(_tag) + ".");
+	return {{_arguments, value}};
 }
 
 Block Parser::parseBlock()