add the ability to declare multiple impl blocks

prdoc/pr_6795.prdoc

@@ -0,0 +1,21 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: ...
+
+doc:
+  - audience: Runtime Dev
+    description: |
+      Makes possible to split internals of `sp_api::impl_runtime_apis!` into multiple blocks.
+      to do this user is required to use `sp_api::impl_runtime_apis_ext` proc-macro attribute on a 
+      module that defines additional `impl` items that will be included in `impl_runtime_apis!` through `use <path>::*` declarations.
+
+
+crates: 
+- name: sp-api
+  bump: major
+- name: sp-api-proc-macro
+  bump: major
+- name: frame-support
+  bump: major
+

substrate/frame/support/test/tests/runtime_metadata.rs

@@ -82,7 +82,7 @@ sp_api::decl_runtime_apis! {
 
 // Module to emulate having the implementation in a different file.
 mod apis {
-	use super::{Block, BlockT, Runtime};
+	use super::{ext, Block, Runtime};
 
 	sp_api::impl_runtime_apis! {
 		impl crate::Api<Block> for Runtime {
@@ -102,17 +102,24 @@ mod apis {
 
 			fn wild_card(_: u32) {}
 		}
+		pub use ext::*;
+	}
+}
 
-		impl sp_api::Core<Block> for Runtime {
-			fn version() -> sp_version::RuntimeVersion {
-				unimplemented!()
-			}
-			fn execute_block(_: Block) {
-				unimplemented!()
-			}
-			fn initialize_block(_: &<Block as BlockT>::Header) -> sp_runtime::ExtrinsicInclusionMode {
-				unimplemented!()
-			}
+#[sp_api::impl_runtime_apis_ext]
+mod ext {
+	use super::*;
+	use crate::apis::RuntimeApiImpl;
+
+	impl sp_api::Core<Block> for Runtime {
+		fn version() -> sp_version::RuntimeVersion {
+			unimplemented!()
+		}
+		fn execute_block(_: Block) {
+			unimplemented!()
+		}
+		fn initialize_block(_: &<Block as BlockT>::Header) -> sp_runtime::ExtrinsicInclusionMode {
+			unimplemented!()
 		}
 	}
 }

substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs

@@ -15,11 +15,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::utils::{
-	extract_api_version, extract_block_type_from_trait_path, extract_impl_trait,
-	extract_parameter_names_types_and_borrows, generate_crate_access,
-	generate_runtime_mod_name_for_trait, prefix_function_with_trait, versioned_trait_name,
-	AllowSelfRefInParameters, ApiVersion, RequireQualifiedTraitPath,
+use crate::{
+	runtime_metadata::Kind,
+	utils::{
+		extract_api_version, extract_block_type_from_trait_path, extract_impl_trait,
+		extract_parameter_names_types_and_borrows, generate_crate_access,
+		generate_runtime_mod_name_for_trait, prefix_function_with_trait, versioned_trait_name,
+		AllowSelfRefInParameters, ApiVersion, RequireQualifiedTraitPath,
+	},
 };
 
 use proc_macro2::{Span, TokenStream};
@@ -32,28 +35,35 @@ use syn::{
 	parse_macro_input, parse_quote,
 	spanned::Spanned,
 	visit_mut::{self, VisitMut},
-	Attribute, Ident, ImplItem, ItemImpl, Path, Signature, Type, TypePath,
+	Attribute, Ident, ImplItem, ItemImpl, ItemMod, ItemUse, Path, Signature, Type, TypePath,
+	UseTree,
 };
 
 use std::collections::HashMap;
 
 /// The structure used for parsing the runtime api implementations.
 struct RuntimeApiImpls {
 	impls: Vec<ItemImpl>,
+	uses: Vec<ItemUse>,
 }
 
 impl Parse for RuntimeApiImpls {
 	fn parse(input: ParseStream) -> Result<Self> {
 		let mut impls = Vec::new();
-
+		let mut uses = Vec::new();
 		while !input.is_empty() {
-			impls.push(ItemImpl::parse(input)?);
+			if input.peek(syn::Token![use]) || input.peek2(syn::Token![use]) {
+				let item = input.parse::<ItemUse>()?;
+				uses.push(item);
+			} else {
+				impls.push(input.parse::<ItemImpl>()?);
+			}
 		}
 
 		if impls.is_empty() {
 			Err(Error::new(Span::call_site(), "No api implementation given!"))
 		} else {
-			Ok(Self { impls })
+			Ok(Self { impls, uses })
 		}
 	}
 }
@@ -192,7 +202,7 @@ fn generate_impl_calls(
 }
 
 /// Generate the dispatch function that is used in native to call into the runtime.
-fn generate_dispatch_function(impls: &[ItemImpl]) -> Result<TokenStream> {
+fn generate_dispatch_function(impls: &[ItemImpl], kind: Kind) -> Result<TokenStream> {
 	let data = Ident::new("_sp_api_input_data_", Span::call_site());
 	let c = generate_crate_access();
 	let impl_calls =
@@ -205,13 +215,33 @@ fn generate_dispatch_function(impls: &[ItemImpl]) -> Result<TokenStream> {
 					#name => Some(#c::Encode::encode(&{ #impl_ })),
 				)
 			});
-
+	let other_branches = match kind {
+		Kind::Main(paths) => {
+			let items = paths
+				.iter()
+				.map(|x| {
+					quote::quote! {
+						if let Some(res) = #x::api::dispatch(method, #data) {
+							return Some(res);
+						}
+					}
+				})
+				.collect::<Vec<TokenStream>>();
+			quote::quote! {
+				#(#items)*
+				None
+			}
+		},
+		Kind::Ext => quote::quote! { None },
+	};
 	Ok(quote!(
 		#c::std_enabled! {
 			pub fn dispatch(method: &str, mut #data: &[u8]) -> Option<Vec<u8>> {
 				match method {
 					#( #impl_calls )*
-					_ => None,
+					_ => {
+						#other_branches
+					},
 				}
 			}
 		}
@@ -264,13 +294,13 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 		#crate_::std_enabled! {
 			/// Implements all runtime apis for the client side.
 			pub struct RuntimeApiImpl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block> + 'static> {
-				call: &'static C,
-				transaction_depth: std::cell::RefCell<u16>,
-				changes: std::cell::RefCell<#crate_::OverlayedChanges<#crate_::HashingFor<Block>>>,
-				recorder: std::option::Option<#crate_::ProofRecorder<Block>>,
-				call_context: #crate_::CallContext,
-				extensions: std::cell::RefCell<#crate_::Extensions>,
-				extensions_generated_for: std::cell::RefCell<std::option::Option<Block::Hash>>,
+				pub(crate) call: &'static C,
+				pub(crate) transaction_depth: std::cell::RefCell<u16>,
+				pub(crate) changes: std::cell::RefCell<#crate_::OverlayedChanges<#crate_::HashingFor<Block>>>,
+				pub(crate) recorder: std::option::Option<#crate_::ProofRecorder<Block>>,
+				pub(crate) call_context: #crate_::CallContext,
+				pub(crate) extensions: std::cell::RefCell<#crate_::Extensions>,
+				pub(crate) extensions_generated_for: std::cell::RefCell<std::option::Option<Block::Hash>>,
 			}
 
 			#[automatically_derived]
@@ -374,7 +404,7 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 			{
 				type RuntimeApi = RuntimeApiImpl<Block, C>;
 
-				fn construct_runtime_api<'a>(
+			fn construct_runtime_api<'a>(
 					call: &'a C,
 				) -> #crate_::ApiRef<'a, Self::RuntimeApi> {
 					RuntimeApiImpl {
@@ -391,7 +421,7 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 
 			#[automatically_derived]
 			impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> RuntimeApiImpl<Block, C> {
-				fn commit_or_rollback_transaction(&self, commit: bool) {
+				pub(crate) fn commit_or_rollback_transaction(&self, commit: bool) {
 					let proof = "\
                     We only close a transaction when we opened one ourself.
                     Other parts of the runtime that make use of transactions (state-machine)
@@ -431,7 +461,7 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					std::result::Result::expect(res, proof);
 				}
 
-				fn start_transaction(&self) {
+				pub(crate) fn start_transaction(&self) {
 					#crate_::OverlayedChanges::start_transaction(
 						&mut std::cell::RefCell::borrow_mut(&self.changes)
 					);
@@ -818,26 +848,70 @@ fn rename_self_in_trait_impls(impls: &mut [ItemImpl]) {
 /// The implementation of the `impl_runtime_apis!` macro.
 pub fn impl_runtime_apis_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
 	// Parse all impl blocks
-	let RuntimeApiImpls { impls: mut api_impls } = parse_macro_input!(input as RuntimeApiImpls);
+	let RuntimeApiImpls { impls: mut api_impls, uses } =
+		parse_macro_input!(input as RuntimeApiImpls);
 
-	impl_runtime_apis_impl_inner(&mut api_impls)
+	impl_runtime_apis_impl_inner(&mut api_impls, &uses)
 		.unwrap_or_else(|e| e.to_compile_error())
 		.into()
 }
 
-fn impl_runtime_apis_impl_inner(api_impls: &mut [ItemImpl]) -> Result<TokenStream> {
+fn impl_runtime_apis_impl_inner(
+	api_impls: &mut [ItemImpl],
+	uses: &[ItemUse],
+) -> Result<TokenStream> {
 	rename_self_in_trait_impls(api_impls);
 
-	let dispatch_impl = generate_dispatch_function(api_impls)?;
 	let api_impls_for_runtime = generate_api_impl_for_runtime(api_impls)?;
 	let base_runtime_api = generate_runtime_api_base_structures()?;
 	let runtime_api_versions = generate_runtime_api_versions(api_impls)?;
 	let wasm_interface = generate_wasm_interface(api_impls)?;
 	let api_impls_for_runtime_api = generate_api_impl_for_runtime_api(api_impls)?;
+	let modules = match uses
+		.iter()
+		.map(|item| {
+			let mut path: Vec<Ident> = vec![];
+			fn call(path: &mut Vec<Ident>, item: &UseTree) -> Result<()> {
+				match &item {
+					syn::UseTree::Path(use_path) => {
+						path.push(use_path.ident.clone());
+						call(path, use_path.tree.as_ref())
+					},
+					syn::UseTree::Glob(_) => Ok(()),
+					syn::UseTree::Name(_) | syn::UseTree::Rename(_) | syn::UseTree::Group(_) => {
+						let error = Error::new(
+							item.span(),
+							"Unsupported syntax used to import api implementaions from an extension module. \
+							Try using `pub use <path>::*` or `use <path>::*`",
+						);
+						return Err(error)
+					},
+				}
+			}
+			call(&mut path, &item.tree)?;
+			let tok = quote::quote! {#(#path)::*};
+			Ok(syn::parse::<TypePath>(tok.into())
+				.expect("Can't happen, a valid TypePath was used in the `UseTree`"))
+		})
+		.collect::<Result<Vec<TypePath>>>()
+		.map_err(|e| e.into_compile_error())
+	{
+		Ok(items) => items,
+		Err(e) => return Ok(e),
+	};
+	let dispatch_impl = generate_dispatch_function(api_impls, Kind::Main(&modules))?;
 
-	let runtime_metadata = crate::runtime_metadata::generate_impl_runtime_metadata(api_impls)?;
+	let runtime_metadata =
+		crate::runtime_metadata::generate_impl_runtime_metadata(api_impls, Kind::Main(&modules))?;
+
+	let c = generate_crate_access();
 
 	let impl_ = quote!(
+		#(
+			#[allow(unused_imports)]
+			#uses
+		)*
+
 		#base_runtime_api
 
 		#api_impls_for_runtime
@@ -848,6 +922,11 @@ fn impl_runtime_apis_impl_inner(api_impls: &mut [ItemImpl]) -> Result<TokenStrea
 
 		#runtime_metadata
 
+		pub fn runtime_api_versions() -> #c::ApisVec {
+			let api = #c::vec::Vec::from([RUNTIME_API_VERSIONS.into_owned(), #(#modules::RUNTIME_API_VERSIONS.into_owned()),*]).concat();
+			api.into()
+		}
+
 		pub mod api {
 			use super::*;
 
@@ -866,6 +945,66 @@ fn impl_runtime_apis_impl_inner(api_impls: &mut [ItemImpl]) -> Result<TokenStrea
 	Ok(impl_)
 }
 
+/// The implementation of the `impl_runtime_apis_ext` macro.
+pub fn impl_runtime_apis_impl_ext(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	let module = parse_macro_input!(input as ItemMod);
+	let (_, items) = module.content.unwrap();
+	let contents = quote::quote! { #(#items)* };
+	let contents: proc_macro::TokenStream = contents.into();
+	// Parse all impl blocks
+	let RuntimeApiImpls { impls: mut api_impls, uses } =
+		parse_macro_input!(contents as RuntimeApiImpls);
+
+	impl_runtime_apis_impl_inner_ext(module.ident, &mut api_impls, &uses)
+		.unwrap_or_else(|e| e.to_compile_error())
+		.into()
+}
+
+fn impl_runtime_apis_impl_inner_ext(
+	module: Ident,
+	api_impls: &mut [ItemImpl],
+	uses: &[ItemUse],
+) -> Result<TokenStream> {
+	rename_self_in_trait_impls(api_impls);
+	let dispatch_impl = generate_dispatch_function(api_impls, Kind::Ext)?;
+	let api_impls_for_runtime = generate_api_impl_for_runtime(api_impls)?;
+	let runtime_api_versions = generate_runtime_api_versions(api_impls)?;
+	let wasm_interface = generate_wasm_interface(api_impls)?;
+	let api_impls_for_runtime_api = generate_api_impl_for_runtime_api(api_impls)?;
+
+	let runtime_metadata =
+		crate::runtime_metadata::generate_impl_runtime_metadata(api_impls, Kind::Ext)?;
+	let impl_ = quote!(
+			pub mod #module {
+				#(#uses)*
+
+				#api_impls_for_runtime
+
+				#api_impls_for_runtime_api
+
+				#runtime_api_versions
+
+				#runtime_metadata
+
+				pub mod api {
+					use super::*;
+
+					#dispatch_impl
+
+					#wasm_interface
+				}
+			}
+	);
+
+	let impl_ = expander::Expander::new("impl_runtime_apis_ext")
+		.dry(std::env::var("EXPAND_MACROS").is_err())
+		.verbose(true)
+		.write_to_out_dir(impl_)
+		.expect("Does not fail because of IO in OUT_DIR; qed");
+
+	Ok(impl_)
+}
+
 // Filters all attributes except the cfg ones.
 fn filter_cfg_attrs(attrs: &[Attribute]) -> Vec<Attribute> {
 	attrs.iter().filter(|a| a.path().is_ident("cfg")).cloned().collect()
@@ -914,7 +1053,7 @@ mod tests {
 		};
 
 		// Parse the items
-		let RuntimeApiImpls { impls: mut api_impls } =
+		let RuntimeApiImpls { impls: mut api_impls, uses: _ } =
 			syn::parse2::<RuntimeApiImpls>(code).unwrap();
 
 		// Run the renamer which is being run first in the `impl_runtime_apis!` macro.

substrate/primitives/api/proc-macro/src/lib.rs

@@ -42,3 +42,8 @@ pub fn mock_impl_runtime_apis(input: TokenStream) -> TokenStream {
 pub fn decl_runtime_apis(input: TokenStream) -> TokenStream {
 	decl_runtime_apis::decl_runtime_apis_impl(input)
 }
+
+#[proc_macro_attribute]
+pub fn impl_runtime_apis_ext(_attrs: TokenStream, tokens: TokenStream) -> TokenStream {
+	impl_runtime_apis::impl_runtime_apis_impl_ext(tokens)
+}