Merge pull request #15367 from ipsilon/eof-prep-3

eof: Introduce EOF container format support in `Assembly::assemble` (EIP-3540)

libevmasm/Assembly.h

@@ -49,11 +49,14 @@ class Assembly
 {
 public:
 	Assembly(langutil::EVMVersion _evmVersion, bool _creation, std::optional<uint8_t> _eofVersion, std::string _name):
-	m_evmVersion(_evmVersion),
-	m_creation(_creation),
-	m_eofVersion(_eofVersion),
-	m_name(std::move(_name))
-	{}
+		m_evmVersion(_evmVersion),
+		m_creation(_creation),
+		m_eofVersion(_eofVersion),
+		m_name(std::move(_name))
+	{
+		// Code section number 0 has to be non-returning.
+		m_codeSections.emplace_back(CodeSection{0, 0x80, {}});
+	}
 
 	std::optional<uint8_t> eofVersion() const { return m_eofVersion; }
 	AssemblyItem newTag() { assertThrow(m_usedTags < 0xffffffff, AssemblyException, ""); return AssemblyItem(Tag, m_usedTags++); }
@@ -103,12 +106,6 @@ class Assembly
 	/// Appends @a _data literally to the very end of the bytecode.
 	void appendToAuxiliaryData(bytes const& _data) { m_auxiliaryData += _data; }
 
-	/// Returns the assembly items.
-	AssemblyItems const& items() const { return m_items; }
-
-	/// Returns the mutable assembly items. Use with care!
-	AssemblyItems& items() { return m_items; }
-
 	int deposit() const { return m_deposit; }
 	void adjustDeposit(int _adjustment) { m_deposit += _adjustment; assertThrow(m_deposit >= 0, InvalidDeposit, ""); }
 	void setDeposit(int _deposit) { m_deposit = _deposit; assertThrow(m_deposit >= 0, InvalidDeposit, ""); }
@@ -170,7 +167,8 @@ class Assembly
 	static std::pair<std::shared_ptr<Assembly>, std::vector<std::string>> fromJSON(
 		Json const& _json,
 		std::vector<std::string> const& _sourceList = {},
-		size_t _level = 0
+		size_t _level = 0,
+		std::optional<uint8_t> _eofVersion = std::nullopt
 	);
 
 	/// Mark this assembly as invalid. Calling ``assemble`` on it will throw.
@@ -181,12 +179,30 @@ class Assembly
 
 	bool isCreation() const { return m_creation; }
 
+	struct CodeSection
+	{
+		uint8_t inputs = 0;
+		uint8_t outputs = 0;
+		AssemblyItems items{};
+	};
+
+	std::vector<CodeSection>& codeSections()
+	{
+		return m_codeSections;
+	}
+
+	std::vector<CodeSection> const& codeSections() const
+	{
+		return m_codeSections;
+	}
+
 protected:
 	/// Does the same operations as @a optimise, but should only be applied to a sub and
 	/// returns the replaced tags. Also takes an argument containing the tags of this assembly
 	/// that are referenced in a super-assembly.
 	std::map<u256, u256> const& optimiseInternal(OptimiserSettings const& _settings, std::set<size_t> _tagsReferencedFromOutside);
 
+	/// For EOF and legacy it calculates approximate size of "pure" code without data.
 	unsigned codeSize(unsigned subTagSize) const;
 
 	/// Add all assembly items from given JSON array. This function imports the items by iterating through
@@ -210,6 +226,15 @@ class Assembly
 
 	std::shared_ptr<std::string const> sharedSourceName(std::string const& _name) const;
 
+	/// Returns EOF header bytecode | code section sizes offsets | data section size offset
+	std::tuple<bytes, std::vector<size_t>, size_t> createEOFHeader(std::set<uint16_t> const& _referencedSubIds) const;
+
+	LinkerObject const& assembleLegacy() const;
+	LinkerObject const& assembleEOF() const;
+
+	/// Returns map from m_subs to an index of subcontainer in the final EOF bytecode
+	std::map<uint16_t, uint16_t> findReferencedContainers() const;
+
 protected:
 	/// 0 is reserved for exception
 	unsigned m_usedTags = 1;
@@ -223,11 +248,12 @@ class Assembly
 	};
 
 	std::map<std::string, NamedTagInfo> m_namedTags;
-	AssemblyItems m_items;
 	std::map<util::h256, bytes> m_data;
 	/// Data that is appended to the very end of the contract.
 	bytes m_auxiliaryData;
 	std::vector<std::shared_ptr<Assembly>> m_subs;
+	std::vector<CodeSection> m_codeSections;
+	uint16_t m_currentCodeSection = 0;
 	std::map<util::h256, std::string> m_strings;
 	std::map<util::h256, std::string> m_libraries; ///< Identifiers of libraries to be linked.
 	std::map<util::h256, std::string> m_immutables; ///< Identifiers of immutables.

libevmasm/ConstantOptimiser.cpp

@@ -35,46 +35,49 @@ unsigned ConstantOptimisationMethod::optimiseConstants(
 )
 {
 	// TODO: design the optimiser in a way this is not needed
-	AssemblyItems& _items = _assembly.items();
-
 	unsigned optimisations = 0;
-	std::map<AssemblyItem, size_t> pushes;
-	for (AssemblyItem const& item: _items)
-		if (item.type() == Push)
-			pushes[item]++;
-	std::map<u256, AssemblyItems> pendingReplacements;
-	for (auto it: pushes)
+	for (auto& codeSection: _assembly.codeSections())
 	{
-		AssemblyItem const& item = it.first;
-		if (item.data() < 0x100)
-			continue;
-		Params params;
-		params.multiplicity = it.second;
-		params.isCreation = _isCreation;
-		params.runs = _runs;
-		params.evmVersion = _evmVersion;
-		LiteralMethod lit(params, item.data());
-		bigint literalGas = lit.gasNeeded();
-		CodeCopyMethod copy(params, item.data());
-		bigint copyGas = copy.gasNeeded();
-		ComputeMethod compute(params, item.data());
-		bigint computeGas = compute.gasNeeded();
-		AssemblyItems replacement;
-		if (copyGas < literalGas && copyGas < computeGas)
-		{
-			replacement = copy.execute(_assembly);
-			optimisations++;
-		}
-		else if (computeGas < literalGas && computeGas <= copyGas)
+		AssemblyItems& _items = codeSection.items;
+
+		std::map<AssemblyItem, size_t> pushes;
+		for (AssemblyItem const& item: _items)
+			if (item.type() == Push)
+				pushes[item]++;
+		std::map<u256, AssemblyItems> pendingReplacements;
+		for (auto it: pushes)
 		{
-			replacement = compute.execute(_assembly);
-			optimisations++;
+			AssemblyItem const& item = it.first;
+			if (item.data() < 0x100)
+				continue;
+			Params params;
+			params.multiplicity = it.second;
+			params.isCreation = _isCreation;
+			params.runs = _runs;
+			params.evmVersion = _evmVersion;
+			LiteralMethod lit(params, item.data());
+			bigint literalGas = lit.gasNeeded();
+			CodeCopyMethod copy(params, item.data());
+			bigint copyGas = copy.gasNeeded();
+			ComputeMethod compute(params, item.data());
+			bigint computeGas = compute.gasNeeded();
+			AssemblyItems replacement;
+			if (copyGas < literalGas && copyGas < computeGas)
+			{
+				replacement = copy.execute(_assembly);
+				optimisations++;
+			}
+			else if (computeGas < literalGas && computeGas <= copyGas)
+			{
+				replacement = compute.execute(_assembly);
+				optimisations++;
+			}
+			if (!replacement.empty())
+				pendingReplacements[item.data()] = replacement;
 		}
-		if (!replacement.empty())
-			pendingReplacements[item.data()] = replacement;
+		if (!pendingReplacements.empty())
+			replaceConstants(_items, pendingReplacements);
 	}
-	if (!pendingReplacements.empty())
-		replaceConstants(_items, pendingReplacements);
 	return optimisations;
 }
 

libevmasm/EVMAssemblyStack.cpp

@@ -45,7 +45,7 @@ void EVMAssemblyStack::analyze(std::string const& _sourceName, Json const& _asse
 {
 	solAssert(!m_evmAssembly);
 	m_name = _sourceName;
-	std::tie(m_evmAssembly, m_sourceList) = evmasm::Assembly::fromJSON(_assemblyJson);
+	std::tie(m_evmAssembly, m_sourceList) = evmasm::Assembly::fromJSON(_assemblyJson, {}, 0, m_eofVersion);
 	solRequire(m_evmAssembly != nullptr, AssemblyImportException, "Could not create evm assembly object.");
 }
 
@@ -56,12 +56,16 @@ void EVMAssemblyStack::assemble()
 	solAssert(!m_evmRuntimeAssembly);
 
 	m_object = m_evmAssembly->assemble();
-	m_sourceMapping = AssemblyItem::computeSourceMapping(m_evmAssembly->items(), sourceIndices());
+	// TODO: Check for EOF
+	solAssert(m_evmAssembly->codeSections().size() == 1);
+	m_sourceMapping = AssemblyItem::computeSourceMapping(m_evmAssembly->codeSections().front().items, sourceIndices());
 	if (m_evmAssembly->numSubs() > 0)
 	{
 		m_evmRuntimeAssembly = std::make_shared<evmasm::Assembly>(m_evmAssembly->sub(0));
 		solAssert(m_evmRuntimeAssembly && !m_evmRuntimeAssembly->isCreation());
-		m_runtimeSourceMapping = AssemblyItem::computeSourceMapping(m_evmRuntimeAssembly->items(), sourceIndices());
+		// TODO: Check for EOF
+		solAssert(m_evmRuntimeAssembly->codeSections().size() == 1);
+		m_runtimeSourceMapping = AssemblyItem::computeSourceMapping(m_evmRuntimeAssembly->codeSections().front().items, sourceIndices());
 		m_runtimeObject = m_evmRuntimeAssembly->assemble();
 	}
 }

libevmasm/EVMAssemblyStack.h

@@ -33,7 +33,8 @@ namespace solidity::evmasm
 class EVMAssemblyStack: public AbstractAssemblyStack
 {
 public:
-	explicit EVMAssemblyStack(langutil::EVMVersion _evmVersion): m_evmVersion(_evmVersion) {}
+	explicit EVMAssemblyStack(langutil::EVMVersion _evmVersion, std::optional<uint8_t> _eofVersion):
+		m_evmVersion(_evmVersion), m_eofVersion(_eofVersion) {}
 
 	/// Runs parsing and analysis steps.
 	/// Multiple calls overwrite the previous state.
@@ -76,6 +77,7 @@ class EVMAssemblyStack: public AbstractAssemblyStack
 
 private:
 	langutil::EVMVersion m_evmVersion;
+	std::optional<uint8_t> m_eofVersion;
 	std::string m_name;
 	std::shared_ptr<evmasm::Assembly> m_evmAssembly;
 	std::shared_ptr<evmasm::Assembly> m_evmRuntimeAssembly;

libsolidity/interface/CompilerStack.cpp

@@ -808,15 +808,24 @@ evmasm::AssemblyItems const* CompilerStack::assemblyItems(std::string const& _co
 	solAssert(m_stackState == CompilationSuccessful, "Compilation was not successful.");
 
 	Contract const& currentContract = contract(_contractName);
-	return currentContract.evmAssembly ? &currentContract.evmAssembly->items() : nullptr;
+	if (!currentContract.evmAssembly)
+		return nullptr;
+	solUnimplementedAssert(!m_eofVersion.has_value(), "EVM assembly output not implemented for EOF yet.");
+	solAssert(currentContract.evmAssembly->codeSections().size() == 1, "Expected a single code section in legacy codegen.");
+	return &currentContract.evmAssembly->codeSections().front().items;
 }
 
 evmasm::AssemblyItems const* CompilerStack::runtimeAssemblyItems(std::string const& _contractName) const
 {
 	solAssert(m_stackState == CompilationSuccessful, "Compilation was not successful.");
 
 	Contract const& currentContract = contract(_contractName);
-	return currentContract.evmRuntimeAssembly ? &currentContract.evmRuntimeAssembly->items() : nullptr;
+
+	if (!currentContract.evmRuntimeAssembly)
+		return nullptr;
+	solUnimplementedAssert(!m_eofVersion.has_value(), "EVM assembly output not implemented for EOF yet.");
+	solAssert(currentContract.evmRuntimeAssembly->codeSections().size() == 1, "Expected a single code section in legacy codegen.");
+	return &currentContract.evmRuntimeAssembly->codeSections().front().items;
 }
 
 Json CompilerStack::generatedSources(std::string const& _contractName, bool _runtime) const

libsolidity/interface/StandardCompiler.cpp

@@ -1204,7 +1204,7 @@ Json StandardCompiler::importEVMAssembly(StandardCompiler::InputsAndSettings _in
 	if (!isBinaryRequested(_inputsAndSettings.outputSelection))
 		return Json::object();
 
-	evmasm::EVMAssemblyStack stack(_inputsAndSettings.evmVersion);
+	evmasm::EVMAssemblyStack stack(_inputsAndSettings.evmVersion, _inputsAndSettings.eofVersion);
 	std::string const& sourceName = _inputsAndSettings.jsonSources.begin()->first; // result of structured binding can only be used within lambda from C++20 on.
 	Json const& sourceJson = _inputsAndSettings.jsonSources.begin()->second;
 	try

libyul/YulStack.cpp

@@ -262,9 +262,12 @@ YulStack::assembleWithDeployed(std::optional<std::string_view> _deployName)
 		creationObject.bytecode = std::make_shared<evmasm::LinkerObject>(creationAssembly->assemble());
 		yulAssert(creationObject.bytecode->immutableReferences.empty(), "Leftover immutables.");
 		creationObject.assembly = creationAssembly;
+		solUnimplementedAssert(!m_eofVersion.has_value(), "EVM assembly output not implemented for EOF yet.");
+		solAssert(creationAssembly->codeSections().size() == 1);
 		creationObject.sourceMappings = std::make_unique<std::string>(
+			// TODO: fix for EOF
 			evmasm::AssemblyItem::computeSourceMapping(
-				creationAssembly->items(),
+				creationAssembly->codeSections().front().items,
 				{{m_charStream->name(), 0}}
 			)
 		);
@@ -273,11 +276,12 @@ YulStack::assembleWithDeployed(std::optional<std::string_view> _deployName)
 		{
 			deployedObject.bytecode = std::make_shared<evmasm::LinkerObject>(deployedAssembly->assemble());
 			deployedObject.assembly = deployedAssembly;
+			solAssert(deployedAssembly->codeSections().size() == 1);
 			deployedObject.sourceMappings = std::make_unique<std::string>(
 				evmasm::AssemblyItem::computeSourceMapping(
-					deployedAssembly->items(),
+					deployedAssembly->codeSections().front().items,
 					{{m_charStream->name(), 0}}
-				)
+					)
 			);
 		}
 	}

solc/CommandLineInterface.cpp

@@ -843,7 +843,7 @@ void CommandLineInterface::assembleFromEVMAssemblyJSON()
 	solAssert(m_fileReader.sourceUnits().size() == 1);
 	auto&& [sourceUnitName, source] = *m_fileReader.sourceUnits().begin();
 
-	auto evmAssemblyStack = std::make_unique<evmasm::EVMAssemblyStack>(m_options.output.evmVersion);
+	auto evmAssemblyStack = std::make_unique<evmasm::EVMAssemblyStack>(m_options.output.evmVersion, m_options.output.eofVersion);
 	try
 	{
 		evmAssemblyStack->parseAndAnalyze(sourceUnitName, source);

test/cmdlineTests/strict_asm_eof_container_prague/args

@@ -0,0 +1 @@
+ --strict-assembly --experimental-eof-version 1 --evm-version prague --bin

test/cmdlineTests/strict_asm_eof_container_prague/input.yul

@@ -0,0 +1,36 @@
+object "object" {
+    code {
+        revert(0, 0)
+    }
+
+    object "sub0" {
+        code {
+            mstore(0, 0)
+            revert(0, 32)
+        }
+    }
+    object "sub1" {
+        code {
+            mstore(0, 1)
+            revert(0, 32)
+        }
+    }
+    object "sub2" {
+        code {
+            mstore(0, 2)
+            revert(0, 32)
+        }
+        object "sub20" {
+            code {
+                mstore(0, 0x20)
+                revert(0, 32)
+            }
+        }
+    }
+    object "sub3" {
+        code {
+            mstore(0, 3)
+            revert(0, 32)
+        }
+    }
+}

test/cmdlineTests/strict_asm_eof_container_prague/output

@@ -0,0 +1,5 @@
+
+======= strict_asm_eof_container_prague/input.yul (EVM) =======
+
+Binary representation:
+ef00010100040200010003030004001a001b003b001b040000000080ffff5f80fdef00010100040200010007040000000080ffff5f805260205ffdef00010100040200010008040000000080ffff60015f5260205ffdef00010100040200010008030001001b040000000080ffff60025f5260205ffdef00010100040200010008040000000080ffff60205f5260205ffdef00010100040200010008040000000080ffff60035f5260205ffd

test/libevmasm/Assembler.cpp

@@ -270,7 +270,8 @@ BOOST_AUTO_TEST_CASE(immutables_and_its_source_maps)
 
 			checkCompilation(assembly);
 
-			std::string const sourceMappings = AssemblyItem::computeSourceMapping(assembly.items(), indices);
+			BOOST_REQUIRE(assembly.codeSections().size() == 1);
+			std::string const sourceMappings = AssemblyItem::computeSourceMapping(assembly.codeSections().at(0).items, indices);
 			auto const numberOfMappings = std::count(sourceMappings.begin(), sourceMappings.end(), ';');
 
 			LinkerObject const& obj = assembly.assemble();

test/libevmasm/Optimiser.cpp

@@ -1381,16 +1381,18 @@ BOOST_AUTO_TEST_CASE(jumpdest_removal_subassemblies)
 		t1.toSubAssemblyTag(subId).pushTag(),
 		u256(8)
 	};
+	BOOST_REQUIRE(main.codeSections().size() == 1);
 	BOOST_CHECK_EQUAL_COLLECTIONS(
-		main.items().begin(), main.items().end(),
+		main.codeSections().at(0).items.begin(),main.codeSections().at(0).items.end(),
 		expectationMain.begin(), expectationMain.end()
 	);
 
 	AssemblyItems expectationSub{
 		u256(1), t1.tag(), u256(2), Instruction::JUMP, t4.tag(), u256(7), t4.pushTag(), Instruction::JUMP
 	};
+	BOOST_REQUIRE(sub->codeSections().size() == 1);
 	BOOST_CHECK_EQUAL_COLLECTIONS(
-		sub->items().begin(), sub->items().end(),
+		sub->codeSections().at(0).items.begin(), sub->codeSections().at(0).items.end(),
 		expectationSub.begin(), expectationSub.end()
 	);
 }

test/libsolidity/Assembly.cpp

@@ -91,7 +91,8 @@ evmasm::AssemblyItems compileContract(std::shared_ptr<CharStream> _sourceCode)
 			);
 			compiler.compileContract(*contract, std::map<ContractDefinition const*, std::shared_ptr<Compiler const>>{}, bytes());
 
-			return compiler.runtimeAssembly().items();
+			BOOST_REQUIRE(compiler.runtimeAssembly().codeSections().size() == 1);
+			return compiler.runtimeAssembly().codeSections().at(0).items;
 		}
 	BOOST_FAIL("No contract found in source.");
 	return AssemblyItems();