Auto merge of #57173 - Zoxc:hir-lifetime, r=<try>

[WIP] Allocate HIR on an arena

Blocked on #56864

Author: bors

Cargo.lock

@@ -47,6 +47,7 @@ name = "arena"
 version = "0.0.0"
 dependencies = [
  "rustc_data_structures 0.0.0",
+ "smallvec 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -2081,6 +2082,7 @@ dependencies = [
  "rustc_data_structures 0.0.0",
  "rustc_errors 0.0.0",
  "rustc_fs_util 0.0.0",
+ "rustc_macros 0.1.0",
  "rustc_target 0.0.0",
  "scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "serialize 0.0.0",
@@ -2328,6 +2330,7 @@ dependencies = [
  "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-demangle 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc_llvm 0.0.0",
+ "rustc_macros 0.1.0",
 ]
 
 [[package]]
@@ -2499,6 +2502,16 @@ dependencies = [
  "core 0.0.0",
 ]
 
+[[package]]
+name = "rustc_macros"
+version = "0.1.0"
+dependencies = [
+ "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 0.15.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rustc_metadata"
 version = "0.0.0"
@@ -2700,6 +2713,7 @@ dependencies = [
  "minifier 0.0.20 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc_macros 0.1.0",
  "tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 

src/bootstrap/bin/rustc.rs

@@ -258,13 +258,6 @@ fn main() {
             }
         }
 
-        // Force all crates compiled by this compiler to (a) be unstable and (b)
-        // allow the `rustc_private` feature to link to other unstable crates
-        // also in the sysroot.
-        if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() {
-            cmd.arg("-Z").arg("force-unstable-if-unmarked");
-        }
-
         if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") {
             cmd.arg("--remap-path-prefix").arg(&map);
         }
@@ -284,6 +277,13 @@ fn main() {
         }
     }
 
+    // Force all crates compiled by this compiler to (a) be unstable and (b)
+    // allow the `rustc_private` feature to link to other unstable crates
+    // also in the sysroot.
+    if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() {
+        cmd.arg("-Z").arg("force-unstable-if-unmarked");
+    }
+
     if env::var_os("RUSTC_PARALLEL_QUERIES").is_some() {
         cmd.arg("--cfg").arg("parallel_queries");
     }

src/bootstrap/check.rs

@@ -42,7 +42,7 @@ impl Step for Std {
                   true);
 
         let libdir = builder.sysroot_libdir(compiler, target);
-        add_to_sysroot(&builder, &libdir, &libstd_stamp(builder, compiler, target));
+        add_to_sysroot(&builder, &libdir, &libstd_stamp(builder, compiler, target), None);
     }
 }
 
@@ -87,8 +87,14 @@ impl Step for Rustc {
                   &librustc_stamp(builder, compiler, target),
                   true);
 
+        let stage_out = builder.build.stage_out(compiler, Mode::Rustc);
         let libdir = builder.sysroot_libdir(compiler, target);
-        add_to_sysroot(&builder, &libdir, &librustc_stamp(builder, compiler, target));
+        add_to_sysroot(
+            &builder,
+            &libdir,
+            &librustc_stamp(builder, compiler, target),
+            Some(&stage_out),
+        );
     }
 }
 
@@ -175,7 +181,7 @@ impl Step for Test {
                   true);
 
         let libdir = builder.sysroot_libdir(compiler, target);
-        add_to_sysroot(builder, &libdir, &libtest_stamp(builder, compiler, target));
+        add_to_sysroot(builder, &libdir, &libtest_stamp(builder, compiler, target), None);
     }
 }
 
@@ -222,7 +228,7 @@ impl Step for Rustdoc {
                   true);
 
         let libdir = builder.sysroot_libdir(compiler, target);
-        add_to_sysroot(&builder, &libdir, &rustdoc_stamp(builder, compiler, target));
+        add_to_sysroot(&builder, &libdir, &rustdoc_stamp(builder, compiler, target), None);
         builder.cargo(compiler, Mode::ToolRustc, target, "clean");
     }
 }

src/bootstrap/compile.rs

@@ -211,7 +211,7 @@ impl Step for StdLink {
                 target_compiler.host,
                 target));
         let libdir = builder.sysroot_libdir(target_compiler, target);
-        add_to_sysroot(builder, &libdir, &libstd_stamp(builder, compiler, target));
+        add_to_sysroot(builder, &libdir, &libstd_stamp(builder, compiler, target), None);
 
         if builder.config.sanitizers && compiler.stage != 0 && target == "x86_64-apple-darwin" {
             // The sanitizers are only built in stage1 or above, so the dylibs will
@@ -414,7 +414,7 @@ impl Step for TestLink {
                 target_compiler.host,
                 target));
         add_to_sysroot(builder, &builder.sysroot_libdir(target_compiler, target),
-                    &libtest_stamp(builder, compiler, target));
+                    &libtest_stamp(builder, compiler, target), None);
 
         builder.cargo(target_compiler, Mode::ToolTest, target, "clean");
     }
@@ -574,8 +574,13 @@ impl Step for RustcLink {
                  &compiler.host,
                  target_compiler.host,
                  target));
-        add_to_sysroot(builder, &builder.sysroot_libdir(target_compiler, target),
-                       &librustc_stamp(builder, compiler, target));
+        let stage_out = builder.build.stage_out(target_compiler, Mode::Rustc);
+        add_to_sysroot(
+            builder,
+            &builder.sysroot_libdir(target_compiler, target),
+            &librustc_stamp(builder, compiler, target),
+            if compiler.stage == 0 { Some(&stage_out) } else { None },
+        );
         builder.cargo(target_compiler, Mode::ToolRustc, target, "clean");
     }
 }
@@ -982,10 +987,26 @@ impl Step for Assemble {
 ///
 /// For a particular stage this will link the file listed in `stamp` into the
 /// `sysroot_dst` provided.
-pub fn add_to_sysroot(builder: &Builder, sysroot_dst: &Path, stamp: &Path) {
+pub fn add_to_sysroot(
+    builder: &Builder,
+    sysroot_dst: &Path,
+    stamp: &Path,
+    stage_out: Option<&Path>) {
     t!(fs::create_dir_all(&sysroot_dst));
     for path in builder.read_stamp_file(stamp) {
-        builder.copy(&path, &sysroot_dst.join(path.file_name().unwrap()));
+        let file_dir = path.parent().unwrap()  // chop off base name
+                           .parent().unwrap()  // chop off `release`
+                           .parent().unwrap(); // chop off `release`
+        if stage_out == Some(file_dir) {
+            // We are copying a build file. We need to add the build triple to it
+            let rustlib_dir = sysroot_dst.parent().unwrap()  // chop off `lib`
+                                         .parent().unwrap(); // chop off `$target`
+            let build_dir = rustlib_dir.join(builder.build.build).join("lib");
+            t!(fs::create_dir_all(&build_dir));
+            builder.copy(&path, &build_dir.join(path.file_name().unwrap()));
+        } else {
+            builder.copy(&path, &sysroot_dst.join(path.file_name().unwrap()));
+        }
     }
 }
 
@@ -1014,8 +1035,12 @@ pub fn run_cargo(builder: &Builder,
     let mut deps = Vec::new();
     let mut toplevel = Vec::new();
     let ok = stream_cargo(builder, cargo, &mut |msg| {
-        let filenames = match msg {
-            CargoMessage::CompilerArtifact { filenames, .. } => filenames,
+        let (filenames, package_id) = match msg {
+            CargoMessage::CompilerArtifact {
+                filenames,
+                package_id,
+                ..
+            } => (filenames, package_id),
             _ => return,
         };
         for filename in filenames {
@@ -1030,8 +1055,12 @@ pub fn run_cargo(builder: &Builder,
             let filename = Path::new(&*filename);
 
             // If this was an output file in the "host dir" we don't actually
-            // worry about it, it's not relevant for us.
+            // worry about it, it's not relevant for us
             if filename.starts_with(&host_root_dir) {
+                // Unless it's a proc macro used in the compiler
+                if package_id.starts_with("rustc_macros ") {
+                    deps.push(filename.to_path_buf());
+                }
                 continue;
             }
 

src/libarena/Cargo.toml

@@ -9,4 +9,5 @@ path = "lib.rs"
 crate-type = ["dylib"]
 
 [dependencies]
-rustc_data_structures = { path = "../librustc_data_structures" }
+rustc_data_structures = { path = "../librustc_data_structures" }
+smallvec = { version = "0.6.7", features = ["union", "may_dangle"] }

src/libarena/lib.rs

@@ -23,9 +23,12 @@
 #![allow(deprecated)]
 
 extern crate alloc;
+extern crate smallvec;
 extern crate rustc_data_structures;
 
 use rustc_data_structures::sync::MTLock;
+#[allow(unused_imports)]
+use smallvec::SmallVec;
 
 use std::cell::{Cell, RefCell};
 use std::cmp;
@@ -151,6 +154,27 @@ impl<T> TypedArena<T> {
         }
     }
 
+    #[inline]
+    fn can_allocate(&self, len: usize) -> bool {
+        let available_capacity_bytes = self.end.get() as usize - self.ptr.get() as usize;
+        let at_least_bytes = len * mem::size_of::<T>();
+        available_capacity_bytes >= at_least_bytes
+    }
+
+    #[inline]
+    fn alloc_raw_slice(&self, len: usize) -> *mut T {
+        assert!(mem::size_of::<T>() != 0);
+        assert!(len != 0);
+
+        if !self.can_allocate(len) {
+            self.grow(len);
+        }
+
+        let start_ptr = self.ptr.get();
+        self.ptr.set(unsafe { start_ptr.add(len) });
+        start_ptr
+    }
+
     /// Allocates a slice of objects that are copied into the `TypedArena`, returning a mutable
     /// reference to it. Will panic if passed a zero-sized types.
     ///
@@ -163,21 +187,40 @@ impl<T> TypedArena<T> {
     where
         T: Copy,
     {
-        assert!(mem::size_of::<T>() != 0);
-        assert!(slice.len() != 0);
+        unsafe {
+            let len = slice.len();
+            let start_ptr = self.alloc_raw_slice(len);
+            slice.as_ptr().copy_to_nonoverlapping(start_ptr, len);
+            slice::from_raw_parts_mut(start_ptr, len)
+        }
+    }
 
-        let available_capacity_bytes = self.end.get() as usize - self.ptr.get() as usize;
-        let at_least_bytes = slice.len() * mem::size_of::<T>();
-        if available_capacity_bytes < at_least_bytes {
-            self.grow(slice.len());
+    #[allow(unreachable_code)]
+    pub fn alloc_from_iter<I: IntoIterator<Item=T>>(&self, iter: I) -> &[T] where T: Clone {
+        assert!(mem::size_of::<T>() != 0);
+        let vec: Vec<_> = iter.into_iter().collect();
+        if vec.is_empty() {
+            return &[]
+        }
+        let vec2 = vec.clone();
+        for a in vec {
+            self.alloc(a);
+        }
+        let result = &vec2[..] as *const [T];
+        mem::forget(vec2);// FIXME: Remove
+        unsafe {
+            return &*result;
         }
 
+        // Move the content to the arena by copying and them forgetting the content of the SmallVec
         unsafe {
-            let start_ptr = self.ptr.get();
-            let arena_slice = slice::from_raw_parts_mut(start_ptr, slice.len());
-            self.ptr.set(start_ptr.add(arena_slice.len()));
-            arena_slice.copy_from_slice(slice);
-            arena_slice
+            let len = vec.len();
+            let start_ptr = self.alloc_raw_slice(len);
+            vec.as_ptr().copy_to_nonoverlapping(start_ptr, len);
+            mem::forget(vec);// FIXME: Remove
+            //mem::forget(vec.drain());
+
+            slice::from_raw_parts_mut(start_ptr, len)
         }
     }
 

src/librustc/Cargo.toml

@@ -22,6 +22,7 @@ rustc-rayon = "0.1.1"
 rustc-rayon-core = "0.1.1"
 rustc_apfloat = { path = "../librustc_apfloat" }
 rustc_target = { path = "../librustc_target" }
+rustc_macros = { path = "../librustc_macros" }
 rustc_data_structures = { path = "../librustc_data_structures" }
 rustc_errors = { path = "../librustc_errors" }
 serialize = { path = "../libserialize" }