Finish and benchmark inline/mem2reg improvements.

crates/rustc_codegen_spirv/src/linker/duplicates.rs

@@ -283,7 +283,20 @@ pub fn remove_duplicate_debuginfo(module: &mut Module) {
         })
         .map(|inst| inst.result_id.unwrap());
 
+    let deduper = DebuginfoDeduplicator {
+        custom_ext_inst_set_import,
+    };
     for func in &mut module.functions {
+        deduper.remove_duplicate_debuginfo_in_function(func);
+    }
+}
+
+pub struct DebuginfoDeduplicator {
+    pub custom_ext_inst_set_import: Option<Word>,
+}
+
+impl DebuginfoDeduplicator {
+    pub fn remove_duplicate_debuginfo_in_function(&self, func: &mut rspirv::dr::Function) {
         for block in &mut func.blocks {
             // Ignore the terminator, it's effectively "outside" debuginfo.
             let (_, insts) = block.instructions.split_last_mut().unwrap();
@@ -339,7 +352,8 @@ pub fn remove_duplicate_debuginfo(module: &mut Module) {
                 let inst = &insts[inst_idx];
                 let custom_op = match inst.class.opcode {
                     Op::ExtInst
-                        if Some(inst.operands[0].unwrap_id_ref()) == custom_ext_inst_set_import =>
+                        if Some(inst.operands[0].unwrap_id_ref())
+                            == self.custom_ext_inst_set_import =>
                     {
                         Some(CustomOp::decode_from_ext_inst(inst))
                     }

crates/rustc_codegen_spirv/src/linker/inline.rs

@@ -133,12 +133,57 @@ pub fn inline(sess: &Session, module: &mut Module) -> super::Result<()> {
         .map(Ok)
         .collect();
 
-    // Inline functions in post-order (aka inside-out aka bottom-out) - that is,
+    let mut mem2reg_pointer_to_pointee = FxHashMap::default();
+    let mut mem2reg_constants = FxHashMap::default();
+    {
+        let mut u32 = None;
+        for inst in &module.types_global_values {
+            match inst.class.opcode {
+                Op::TypePointer => {
+                    mem2reg_pointer_to_pointee
+                        .insert(inst.result_id.unwrap(), inst.operands[1].unwrap_id_ref());
+                }
+                Op::TypeInt
+                    if inst.operands[0].unwrap_literal_bit32() == 32
+                        && inst.operands[1].unwrap_literal_bit32() == 0 =>
+                {
+                    assert!(u32.is_none());
+                    u32 = Some(inst.result_id.unwrap());
+                }
+                Op::Constant if u32.is_some() && inst.result_type == u32 => {
+                    let value = inst.operands[0].unwrap_literal_bit32();
+                    mem2reg_constants.insert(inst.result_id.unwrap(), value);
+                }
+                _ => {}
+            }
+        }
+    }
+
+    // Inline functions in post-order (aka inside-out aka bottom-up) - that is,
     // callees are processed before their callers, to avoid duplicating work.
     for func_idx in call_graph.post_order() {
         let mut function = mem::replace(&mut functions[func_idx], Err(FuncIsBeingInlined)).unwrap();
         inliner.inline_fn(&mut function, &functions);
         fuse_trivial_branches(&mut function);
+
+        super::duplicates::DebuginfoDeduplicator {
+            custom_ext_inst_set_import,
+        }
+        .remove_duplicate_debuginfo_in_function(&mut function);
+
+        {
+            super::simple_passes::block_ordering_pass(&mut function);
+            // Note: mem2reg requires functions to be in RPO order (i.e. block_ordering_pass)
+            super::mem2reg::mem2reg(
+                inliner.header,
+                &mut module.types_global_values,
+                &mem2reg_pointer_to_pointee,
+                &mem2reg_constants,
+                &mut function,
+            );
+            super::destructure_composites::destructure_composites(&mut function);
+        }
+
         functions[func_idx] = Ok(function);
     }
 
@@ -411,7 +456,7 @@ fn should_inline(
         }
 
         // If the call isn't passing a legal pointer argument (a "memory object",
-        // i.e. an `OpVariable` or one of the caller's `OpFunctionParameters),
+        // i.e. an `OpVariable` or one of the caller's `OpFunctionParameter`s),
         // then inlining is required to have a chance at producing legal SPIR-V.
         //
         // FIXME(eddyb) rewriting away the pointer could be another alternative.
@@ -826,7 +871,7 @@ impl Inliner<'_, '_> {
                 }
 
                 // `vars_and_debuginfo_range.end` indicates where `OpVariable`s
-                // end and other instructions start (module debuginfo), but to
+                // end and other instructions start (modulo debuginfo), but to
                 // split the block in two, both sides of the "cut" need "repair":
                 // - the variables are missing "inlined call frames" pops, that
                 //   may happen later in the block, and have to be synthesized

crates/rustc_codegen_spirv/src/linker/mem2reg.rs

@@ -193,6 +193,8 @@ fn insert_phis_all(
     for (var_map, _) in &var_maps_and_types {
         split_copy_memory(header, blocks, var_map);
     }
+
+    let mut rewrite_rules = FxHashMap::default();
     for &(ref var_map, base_var_type) in &var_maps_and_types {
         let blocks_with_phi = insert_phis(blocks, dominance_frontier, var_map);
         let mut renamer = Renamer {
@@ -205,16 +207,15 @@ fn insert_phis_all(
             phi_defs: FxHashSet::default(),
             visited: FxHashSet::default(),
             stack: Vec::new(),
-            rewrite_rules: FxHashMap::default(),
+            rewrite_rules: &mut rewrite_rules,
         };
         renamer.rename(0, None);
-        // FIXME(eddyb) shouldn't this full rescan of the function be done once?
-        apply_rewrite_rules(
-            &renamer.rewrite_rules,
-            blocks.values_mut().map(|block| &mut **block),
-        );
-        remove_nops(blocks);
     }
+    apply_rewrite_rules(
+        &rewrite_rules,
+        blocks.values_mut().map(|block| &mut **block),
+    );
+    remove_nops(blocks);
     remove_old_variables(blocks, &var_maps_and_types);
     true
 }
@@ -443,7 +444,7 @@ struct Renamer<'a, 'b> {
     phi_defs: FxHashSet<Word>,
     visited: FxHashSet<usize>,
     stack: Vec<Word>,
-    rewrite_rules: FxHashMap<Word, Word>,
+    rewrite_rules: &'a mut FxHashMap<Word, Word>,
 }
 
 impl Renamer<'_, '_> {

crates/rustc_codegen_spirv/src/linker/mod.rs

@@ -104,7 +104,7 @@ fn apply_rewrite_rules<'a>(
                 )
         });
     for id in all_ids_mut {
-        if let Some(&rewrite) = rewrite_rules.get(id) {
+        while let Some(&rewrite) = rewrite_rules.get(id) {
             *id = rewrite;
         }
     }