Serialize additional data for procedural macros

Split off from #62855

This PR deerializes the declaration `Span` and attributes for all
procedural macros from their underlying function definitions.
This allows Rustdoc to properly render doc comments
and source links when inlining procedural macros across crates

Author: Aaron1011

src/libproc_macro/bridge/client.rs

@@ -468,6 +468,14 @@ pub enum ProcMacro {
 }
 
 impl ProcMacro {
+    pub fn name(&self) -> &'static str {
+        match self {
+            ProcMacro::CustomDerive { trait_name, .. } => trait_name,
+            ProcMacro::Attr { name, .. } => name,
+            ProcMacro::Bang { name, ..} => name
+        }
+    }
+
     pub const fn custom_derive(
         trait_name: &'static str,
         attributes: &'static [&'static str],

src/librustc/hir/def_id.rs

@@ -1,5 +1,4 @@
 use crate::ty::{self, TyCtxt};
-use crate::hir::map::definitions::FIRST_FREE_DEF_INDEX;
 use rustc_data_structures::indexed_vec::Idx;
 use std::fmt;
 use std::u32;
@@ -102,31 +101,6 @@ newtype_index! {
     }
 }
 
-impl DefIndex {
-    // Proc macros from a proc-macro crate have a kind of virtual DefIndex. This
-    // function maps the index of the macro within the crate (which is also the
-    // index of the macro in the CrateMetadata::proc_macros array) to the
-    // corresponding DefIndex.
-    pub fn from_proc_macro_index(proc_macro_index: usize) -> DefIndex {
-        // DefIndex for proc macros start from FIRST_FREE_DEF_INDEX,
-        // because the first FIRST_FREE_DEF_INDEX indexes are reserved
-        // for internal use.
-        let def_index = DefIndex::from(
-            proc_macro_index.checked_add(FIRST_FREE_DEF_INDEX)
-                .expect("integer overflow adding `proc_macro_index`"));
-        assert!(def_index != CRATE_DEF_INDEX);
-        def_index
-    }
-
-    // This function is the reverse of from_proc_macro_index() above.
-    pub fn to_proc_macro_index(self: DefIndex) -> usize {
-        self.index().checked_sub(FIRST_FREE_DEF_INDEX)
-            .unwrap_or_else(|| {
-                bug!("using local index {:?} as proc-macro index", self)
-            })
-    }
-}
-
 impl rustc_serialize::UseSpecializedEncodable for DefIndex {}
 impl rustc_serialize::UseSpecializedDecodable for DefIndex {}
 

src/librustc/hir/map/definitions.rs

@@ -411,10 +411,6 @@ impl Definitions {
     }
 
     /// Adds a root definition (no parent) and a few other reserved definitions.
-    ///
-    /// After the initial definitions are created the first `FIRST_FREE_DEF_INDEX` indexes
-    /// are taken, so the "user" indexes will be allocated starting with `FIRST_FREE_DEF_INDEX`
-    /// in ascending order.
     pub fn create_root_def(&mut self,
                            crate_name: &str,
                            crate_disambiguator: CrateDisambiguator)
@@ -589,19 +585,6 @@ impl DefPathData {
     }
 }
 
-/// Evaluates to the number of tokens passed to it.
-///
-/// Logarithmic counting: every one or two recursive expansions, the number of
-/// tokens to count is divided by two, instead of being reduced by one.
-/// Therefore, the recursion depth is the binary logarithm of the number of
-/// tokens to count, and the expanded tree is likewise very small.
-macro_rules! count {
-    ()                     => (0usize);
-    ($one:tt)              => (1usize);
-    ($($pairs:tt $_p:tt)*) => (count!($($pairs)*) << 1usize);
-    ($odd:tt $($rest:tt)*) => (count!($($rest)*) | 1usize);
-}
-
 // We define the GlobalMetaDataKind enum with this macro because we want to
 // make sure that we exhaustively iterate over all variants when registering
 // the corresponding DefIndices in the DefTable.
@@ -614,8 +597,6 @@ macro_rules! define_global_metadata_kind {
             $($variant),*
         }
 
-        pub const FIRST_FREE_DEF_INDEX: usize = 1 + count!($($variant)*);
-
         impl GlobalMetaDataKind {
             fn allocate_def_indices(definitions: &mut Definitions) {
                 $({

src/librustc_metadata/creader.rs

@@ -2,8 +2,7 @@
 
 use crate::cstore::{self, CStore, CrateSource, MetadataBlob};
 use crate::locator::{self, CratePaths};
-use crate::decoder::proc_macro_def_path_table;
-use crate::schema::CrateRoot;
+use crate::schema::{CrateRoot};
 use rustc_data_structures::sync::{Lrc, RwLock, Lock};
 
 use rustc::hir::def_id::CrateNum;
@@ -26,11 +25,11 @@ use std::{cmp, fs};
 use syntax::ast;
 use syntax::attr;
 use syntax::ext::allocator::{global_allocator_spans, AllocatorKind};
-use syntax::ext::base::{SyntaxExtension, SyntaxExtensionKind};
 use syntax::symbol::{Symbol, sym};
 use syntax::{span_err, span_fatal};
 use syntax_pos::{Span, DUMMY_SP};
 use log::{debug, info, log_enabled};
+use proc_macro::bridge::client::ProcMacro;
 
 pub struct Library {
     pub dylib: Option<(PathBuf, PathKind)>,
@@ -230,24 +229,13 @@ impl<'a> CrateLoader<'a> {
 
         let dependencies: Vec<CrateNum> = cnum_map.iter().cloned().collect();
 
-        let proc_macros = crate_root.proc_macro_decls_static.map(|_| {
+        let raw_proc_macros =  crate_root.proc_macro_data.map(|_| {
             if self.sess.opts.debugging_opts.dual_proc_macros {
-                let host_lib = host_lib.unwrap();
-                self.load_derive_macros(
-                    &host_lib.metadata.get_root(),
-                    host_lib.dylib.map(|p| p.0),
-                    span
-                )
+                let host_lib = host_lib.as_ref().unwrap();
+                self.dlsym_proc_macros(host_lib.dylib.as_ref().map(|p| p.0.clone()),
+                                       &host_lib.metadata.get_root(), span)
             } else {
-                self.load_derive_macros(&crate_root, dylib.clone().map(|p| p.0), span)
-            }
-        });
-
-        let def_path_table = record_time(&self.sess.perf_stats.decode_def_path_tables_time, || {
-            if let Some(proc_macros) = &proc_macros {
-                proc_macro_def_path_table(&crate_root, proc_macros)
-            } else {
-                crate_root.def_path_table.decode((&metadata, self.sess))
+                self.dlsym_proc_macros(dylib.clone().map(|p| p.0), &crate_root, span)
             }
         });
 
@@ -260,13 +248,16 @@ impl<'a> CrateLoader<'a> {
             .map(|trait_impls| (trait_impls.trait_id, trait_impls.impls))
             .collect();
 
+        let def_path_table = record_time(&self.sess.perf_stats.decode_def_path_tables_time, || {
+            crate_root.def_path_table.decode((&metadata, self.sess))
+        });
+
         let cmeta = cstore::CrateMetadata {
             name: crate_root.name,
             imported_name: ident,
             extern_crate: Lock::new(None),
             def_path_table: Lrc::new(def_path_table),
             trait_impls,
-            proc_macros,
             root: crate_root,
             blob: metadata,
             cnum_map,
@@ -280,7 +271,10 @@ impl<'a> CrateLoader<'a> {
                 rlib,
                 rmeta,
             },
-            private_dep
+            private_dep,
+            span,
+            host_lib,
+            raw_proc_macros
         };
 
         let cmeta = Lrc::new(cmeta);
@@ -389,7 +383,7 @@ impl<'a> CrateLoader<'a> {
         match result {
             (LoadResult::Previous(cnum), None) => {
                 let data = self.cstore.get_crate_data(cnum);
-                if data.root.proc_macro_decls_static.is_some() {
+                if data.root.proc_macro_data.is_some() {
                     dep_kind = DepKind::UnexportedMacrosOnly;
                 }
                 data.dep_kind.with_lock(|data_dep_kind| {
@@ -482,7 +476,7 @@ impl<'a> CrateLoader<'a> {
                           dep_kind: DepKind)
                           -> cstore::CrateNumMap {
         debug!("resolving deps of external crate");
-        if crate_root.proc_macro_decls_static.is_some() {
+        if crate_root.proc_macro_data.is_some() {
             return cstore::CrateNumMap::new();
         }
 
@@ -574,19 +568,13 @@ impl<'a> CrateLoader<'a> {
         }
     }
 
-    /// Loads custom derive macros.
-    ///
-    /// Note that this is intentionally similar to how we load plugins today,
-    /// but also intentionally separate. Plugins are likely always going to be
-    /// implemented as dynamic libraries, but we have a possible future where
-    /// custom derive (and other macro-1.1 style features) are implemented via
-    /// executables and custom IPC.
-    fn load_derive_macros(&mut self, root: &CrateRoot<'_>, dylib: Option<PathBuf>, span: Span)
-                          -> Vec<(ast::Name, Lrc<SyntaxExtension>)> {
-        use std::{env, mem};
+    fn dlsym_proc_macros(&self,
+                         dylib: Option<PathBuf>,
+                         root: &CrateRoot<'_>,
+                         span: Span
+    ) -> &'static [ProcMacro] {
+        use std::env;
         use crate::dynamic_lib::DynamicLibrary;
-        use proc_macro::bridge::client::ProcMacro;
-        use syntax::ext::proc_macro::{BangProcMacro, AttrProcMacro, ProcMacroDerive};
 
         let path = match dylib {
             Some(dylib) => dylib,
@@ -608,38 +596,11 @@ impl<'a> CrateLoader<'a> {
             *(sym as *const &[ProcMacro])
         };
 
-        let extensions = decls.iter().map(|&decl| {
-            let (name, kind, helper_attrs) = match decl {
-                ProcMacro::CustomDerive { trait_name, attributes, client } => {
-                    let helper_attrs =
-                        attributes.iter().cloned().map(Symbol::intern).collect::<Vec<_>>();
-                    (
-                        trait_name,
-                        SyntaxExtensionKind::Derive(Box::new(ProcMacroDerive {
-                            client, attrs: helper_attrs.clone()
-                        })),
-                        helper_attrs,
-                    )
-                }
-                ProcMacro::Attr { name, client } => (
-                    name, SyntaxExtensionKind::Attr(Box::new(AttrProcMacro { client })), Vec::new()
-                ),
-                ProcMacro::Bang { name, client } => (
-                    name, SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client })), Vec::new()
-                )
-            };
-
-            (Symbol::intern(name), Lrc::new(SyntaxExtension {
-                helper_attrs,
-                ..SyntaxExtension::default(kind, root.edition)
-            }))
-        }).collect();
-
         // Intentionally leak the dynamic library. We can't ever unload it
         // since the library can make things that will live arbitrarily long.
-        mem::forget(lib);
+        std::mem::forget(lib);
 
-        extensions
+        decls
     }
 
     /// Look for a plugin registrar. Returns library path, crate

src/librustc_metadata/cstore.rs

@@ -28,6 +28,9 @@ pub use crate::cstore_impl::{provide, provide_extern};
 pub type CrateNumMap = IndexVec<CrateNum, CrateNum>;
 
 pub use rustc_data_structures::sync::MetadataRef;
+use crate::creader::Library;
+use syntax_pos::Span;
+use proc_macro::bridge::client::ProcMacro;
 
 pub struct MetadataBlob(pub MetadataRef);
 
@@ -82,11 +85,19 @@ pub struct CrateMetadata {
     pub dep_kind: Lock<DepKind>,
     pub source: CrateSource,
 
-    pub proc_macros: Option<Vec<(ast::Name, Lrc<SyntaxExtension>)>>,
-
     /// Whether or not this crate should be consider a private dependency
     /// for purposes of the 'exported_private_dependencies' lint
-    pub private_dep: bool
+    pub private_dep: bool,
+
+    pub host_lib: Option<Library>,
+    pub span: Span,
+
+    pub raw_proc_macros: Option<&'static [ProcMacro]>,
+}
+
+pub struct FullProcMacro {
+    pub name: ast::Name,
+    pub ext: Lrc<SyntaxExtension>
 }
 
 pub struct CStore {

src/librustc_metadata/cstore_impl.rs

@@ -426,8 +426,8 @@ impl cstore::CStore {
 
     pub fn load_macro_untracked(&self, id: DefId, sess: &Session) -> LoadedMacro {
         let data = self.get_crate_data(id.krate);
-        if let Some(ref proc_macros) = data.proc_macros {
-            return LoadedMacro::ProcMacro(proc_macros[id.index.to_proc_macro_index()].1.clone());
+        if data.is_proc_macro_crate() {
+            return LoadedMacro::ProcMacro(data.get_proc_macro(id.index, sess).ext);
         } else if data.name == sym::proc_macro && data.item_name(id.index) == sym::quote {
             let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote);
             let kind = SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client }));
@@ -439,7 +439,8 @@ impl cstore::CStore {
         }
 
         let def = data.get_macro(id.index);
-        let macro_full_name = data.def_path(id.index).to_string_friendly(|_| data.imported_name);
+        let macro_full_name = data.def_path(id.index)
+            .to_string_friendly(|_| data.imported_name);
         let source_name = FileName::Macros(macro_full_name);
 
         let source_file = sess.parse_sess.source_map().new_source_file(source_name, def.body);