clippy_dev: Keep a source map when parsing files.

Author: Jarcho

clippy_dev/Cargo.toml

@@ -9,6 +9,7 @@ chrono = { version = "0.4.38", default-features = false, features = ["clock"] }
 clap = { version = "4.4", features = ["derive"] }
 indoc = "1.0"
 itertools = "0.12"
+memchr = "2.7.5"
 opener = "0.7"
 walkdir = "2.3"
 

clippy_dev/src/deprecate_lint.rs

@@ -1,8 +1,7 @@
-use crate::parse::{ActiveLint, DeprecatedLint, Lint, LintList};
+use crate::parse::{ActiveLint, DeprecatedLint, Lint, ParsedData};
 use crate::update_lints::generate_lint_files;
 use crate::utils::{FileUpdater, UpdateMode, UpdateStatus, Version, delete_dir_if_exists, delete_file_if_exists};
 use core::mem;
-use rustc_data_structures::fx::FxHashMap;
 use std::ffi::OsStr;
 use std::path::Path;
 
@@ -20,8 +19,8 @@ pub fn deprecate(clippy_version: Version, name: &str, reason: &str) {
         panic!("`{name}` should not contain the `{prefix}` prefix");
     }
 
-    let mut list = LintList::collect();
-    let Some(entry) = list.lints.get_mut(name) else {
+    let mut data = ParsedData::collect();
+    let Some(entry) = data.lints.get_mut(name) else {
         eprintln!("error: failed to find lint `{name}`");
         return;
     };
@@ -36,13 +35,13 @@ pub fn deprecate(clippy_version: Version, name: &str, reason: &str) {
         return;
     };
 
-    remove_lint_declaration(name, &lint, &list.lints);
-    generate_lint_files(UpdateMode::Change, &list);
+    remove_lint_declaration(name, &lint, &data);
+    generate_lint_files(UpdateMode::Change, &data);
     println!("info: `{name}` has successfully been deprecated");
     println!("note: you must run `cargo uitest` to update the test results");
 }
 
-fn remove_lint_declaration(name: &str, lint: &ActiveLint, lints: &FxHashMap<String, Lint>) {
+fn remove_lint_declaration(name: &str, lint: &ActiveLint, data: &ParsedData) {
     fn remove_test_assets(name: &str) {
         let test_file_stem = format!("tests/ui/{name}");
         let path = Path::new(&test_file_stem);
@@ -84,24 +83,26 @@ fn remove_lint_declaration(name: &str, lint: &ActiveLint, lints: &FxHashMap<Stri
         }
     }
 
-    if lints.values().any(|l| {
+    let lint_file = &data.source_map.files[lint.span.file];
+    if data.lints.values().any(|l| {
         if let Lint::Active(l) = l {
-            l.krate == lint.krate && l.module.starts_with(&lint.module)
+            let other_file = &data.source_map.files[l.span.file];
+            other_file.krate == lint_file.krate && other_file.module.starts_with(&lint_file.module)
         } else {
             false
         }
     }) {
         // Try to delete a sub-module that matches the lint's name
-        let removed_mod = if lint.path.file_name().map(OsStr::as_encoded_bytes) == Some(b"mod.rs") {
-            let mut path = lint.path.to_path_buf();
+        let removed_mod = if lint_file.path.file_name().map(OsStr::as_encoded_bytes) == Some(b"mod.rs") {
+            let mut path = lint_file.path.to_path_buf();
             path.set_file_name(name);
             path.set_extension("rs");
             delete_file_if_exists(&path)
         } else {
             false
         };
 
-        FileUpdater::default().update_file(&lint.path, &mut |_, src, dst| {
+        FileUpdater::default().update_file(&lint_file.path, &mut |_, src, dst| {
             let (a, b, c, d) = if removed_mod
                 && let mod_decl = format!("\nmod {name};")
                 && let Some(mod_start) = src.find(&mod_decl)
@@ -132,7 +133,7 @@ fn remove_lint_declaration(name: &str, lint: &ActiveLint, lints: &FxHashMap<Stri
         });
     } else {
         // No other lint in the same module or a sub-module.
-        delete_file_if_exists(&lint.path);
+        delete_file_if_exists(&lint_file.path);
     }
     remove_test_assets(name);
 }

clippy_dev/src/lib.rs

@@ -21,6 +21,7 @@
 extern crate rustc_driver;
 
 extern crate rustc_data_structures;
+extern crate rustc_index;
 extern crate rustc_lexer;
 extern crate rustc_literal_escaper;
 
@@ -38,3 +39,4 @@ pub mod update_lints;
 pub mod utils;
 
 mod parse;
+mod source_map;

clippy_dev/src/parse.rs

@@ -1,10 +1,11 @@
-use crate::utils::{ErrAction, File, expect_action, walk_dir_no_dot_or_target};
+use crate::source_map::{SourceFile, SourceMap, Span};
+use crate::utils::{ErrAction, expect_action, walk_dir_no_dot_or_target};
 use core::ops::Range;
 use core::slice;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_lexer::{self as lexer, FrontmatterAllowed};
 use std::fs;
-use std::path::{self, Path, PathBuf};
+use std::path::{self, Path};
 
 #[derive(Clone, Copy)]
 pub enum Token<'a> {
@@ -184,11 +185,8 @@ impl<'txt> RustSearcher<'txt> {
 }
 
 pub struct ActiveLint {
-    pub krate: String,
-    pub module: String,
     pub group: String,
-    pub path: PathBuf,
-    pub span: Range<u32>,
+    pub span: Span,
 }
 
 pub struct DeprecatedLint {
@@ -207,24 +205,26 @@ pub enum Lint {
     Renamed(RenamedLint),
 }
 
-pub struct LintList {
+pub struct ParsedData {
+    pub source_map: SourceMap,
     pub lints: FxHashMap<String, Lint>,
     pub deprecated_span: Range<u32>,
     pub renamed_span: Range<u32>,
 }
-impl LintList {
+impl ParsedData {
     pub fn collect() -> Self {
         // 2025-05: Initial capacities should fit everything without reallocating.
         let mut parser = Parser {
+            source_map: SourceMap::with_capacity(8, 1000),
             lints: FxHashMap::with_capacity_and_hasher(1000, Default::default()),
             deprecated_span: 0..0,
             renamed_span: 0..0,
-            contents: String::with_capacity(1024 * 128 * 3),
         };
         parser.parse_src_files();
         parser.parse_deprecated_lints();
 
-        LintList {
+        ParsedData {
+            source_map: parser.source_map,
             lints: parser.lints,
             deprecated_span: parser.deprecated_span,
             renamed_span: parser.renamed_span,
@@ -233,10 +233,10 @@ impl LintList {
 }
 
 struct Parser {
+    source_map: SourceMap,
     lints: FxHashMap<String, Lint>,
     deprecated_span: Range<u32>,
     renamed_span: Range<u32>,
-    contents: String,
 }
 impl Parser {
     /// Parses all source files looking for lint declarations (`declare_clippy_lint! { .. }`).
@@ -250,8 +250,9 @@ impl Parser {
                 continue;
             };
             if crate_path.starts_with("clippy_lints") && crate_path != "clippy_lints_internal" {
-                crate_path.push_str("/src");
-                let krate = &crate_path[..crate_path.len() - 4];
+                let krate = self.source_map.add_new_crate(&crate_path);
+                crate_path.push(path::MAIN_SEPARATOR);
+                crate_path.push_str("src");
                 for e in walk_dir_no_dot_or_target(&crate_path) {
                     let e = expect_action(e, ErrAction::Read, &crate_path);
                     if let Some(path) = e.path().to_str()
@@ -270,15 +271,16 @@ impl Parser {
                             };
                             path.replace(path::MAIN_SEPARATOR, "::")
                         };
-                        self.parse_src_file(e.path(), krate, &module);
+                        let file = self.source_map.load_new_file(e.path(), krate, module);
+                        self.parse_src_file(file);
                     }
                 }
             }
         }
     }
 
     /// Parse a source file looking for `declare_clippy_lint` macro invocations.
-    fn parse_src_file(&mut self, path: &Path, krate: &str, module: &str) {
+    fn parse_src_file(&mut self, file: SourceFile) {
         #[allow(clippy::enum_glob_use)]
         use Token::*;
         #[rustfmt::skip]
@@ -291,20 +293,20 @@ impl Parser {
             Ident("pub"), CaptureIdent, Comma, AnyComment, CaptureIdent, Comma,
         ];
 
-        File::open_read_to_cleared_string(path, &mut self.contents);
-        let mut searcher = RustSearcher::new(&self.contents);
+        let mut searcher = RustSearcher::new(&self.source_map.files[file].contents);
         while searcher.find_token(Ident("declare_clippy_lint")) {
             let start = searcher.pos() - "declare_clippy_lint".len() as u32;
             let (mut name, mut group) = ("", "");
             if searcher.match_tokens(DECL_TOKENS, &mut [&mut name, &mut group]) && searcher.find_token(CloseBrace) {
                 self.lints.insert(
                     name.to_ascii_lowercase(),
                     Lint::Active(ActiveLint {
-                        krate: krate.into(),
-                        module: module.into(),
                         group: group.into(),
-                        path: path.into(),
-                        span: start..searcher.pos(),
+                        span: Span {
+                            file,
+                            start,
+                            end: searcher.pos(),
+                        },
                     }),
                 );
             }
@@ -332,10 +334,15 @@ impl Parser {
             Bang, OpenBrace, Ident("RENAMED"), OpenParen, Ident("RENAMED_VERSION"), CloseParen, Eq, OpenBracket,
         ];
 
-        let path = Path::new("clippy_lints/src/deprecated_lints.rs");
-        File::open_read_to_cleared_string(path, &mut self.contents);
+        let krate = self.source_map.add_crate("clippy_lints");
+        let file = self.source_map.load_file(
+            Path::new("clippy_lints/src/deprecated_lints.rs"),
+            krate,
+            "deprecated_lints",
+        );
+        let file = &self.source_map.files[file];
 
-        let mut searcher = RustSearcher::new(&self.contents);
+        let mut searcher = RustSearcher::new(&file.contents);
         // First instance is the macro definition.
         assert!(
             searcher.find_token(Ident("declare_with_version")),
@@ -350,10 +357,10 @@ impl Parser {
             let mut reason = "";
             while searcher.match_tokens(DECL_TOKENS, &mut [&mut version, &mut name, &mut reason]) {
                 self.lints.insert(
-                    parse_clippy_lint_name(path, name),
+                    parse_clippy_lint_name(&file.path, name),
                     Lint::Deprecated(DeprecatedLint {
-                        reason: parse_str_single_line(path, reason),
-                        version: parse_str_single_line(path, version),
+                        reason: parse_str_single_line(&file.path, reason),
+                        version: parse_str_single_line(&file.path, version),
                     }),
                 );
                 end = searcher.pos();
@@ -371,10 +378,10 @@ impl Parser {
             let mut new_name = "";
             while searcher.match_tokens(DECL_TOKENS, &mut [&mut version, &mut old_name, &mut new_name]) {
                 self.lints.insert(
-                    parse_clippy_lint_name(path, old_name),
+                    parse_clippy_lint_name(&file.path, old_name),
                     Lint::Renamed(RenamedLint {
-                        new_name: parse_maybe_clippy_lint_name(path, new_name),
-                        version: parse_str_single_line(path, version),
+                        new_name: parse_maybe_clippy_lint_name(&file.path, new_name),
+                        version: parse_str_single_line(&file.path, version),
                     }),
                 );
                 end = searcher.pos();

clippy_dev/src/rename_lint.rs

@@ -1,4 +1,4 @@
-use crate::parse::{Lint, LintList, RenamedLint, RustSearcher, Token};
+use crate::parse::{Lint, ParsedData, RenamedLint, RustSearcher, Token};
 use crate::update_lints::generate_lint_files;
 use crate::utils::{
     ErrAction, FileUpdater, UpdateMode, UpdateStatus, Version, delete_dir_if_exists, delete_file_if_exists,
@@ -36,15 +36,15 @@ pub fn rename(clippy_version: Version, old_name: &str, new_name: &str, uplift: b
     }
 
     let mut updater = FileUpdater::default();
-    let mut list = LintList::collect();
+    let mut data = ParsedData::collect();
 
     // Update any existing renames
     let new_name_prefixed = if uplift {
         new_name.to_owned()
     } else {
         String::from_iter(["clippy::", new_name])
     };
-    for lint in list.lints.values_mut() {
+    for lint in data.lints.values_mut() {
         if let Lint::Renamed(lint) = lint
             && lint.new_name.strip_prefix("clippy::") == Some(old_name)
         {
@@ -53,11 +53,11 @@ pub fn rename(clippy_version: Version, old_name: &str, new_name: &str, uplift: b
     }
 
     // Mark the lint as renamed
-    let Some(old_entry) = list.lints.get_mut(old_name) else {
+    let Some(old_entry) = data.lints.get_mut(old_name) else {
         eprintln!("error: failed to find lint `{old_name}`");
         return;
     };
-    let Lint::Active(mut lint) = mem::replace(
+    let Lint::Active(lint) = mem::replace(
         old_entry,
         Lint::Renamed(RenamedLint {
             new_name: new_name_prefixed,
@@ -70,19 +70,20 @@ pub fn rename(clippy_version: Version, old_name: &str, new_name: &str, uplift: b
 
     let mut mod_edit = ModEdit::None;
     if uplift {
-        let is_unique_mod = list.lints.values().any(|x| {
+        let lint_file = &data.source_map.files[lint.span.file];
+        let is_unique_mod = data.lints.values().any(|x| {
             if let Lint::Active(x) = x {
-                x.module == lint.module
+                data.source_map.files[x.span.file].module == lint_file.module
             } else {
                 false
             }
         });
         if is_unique_mod {
-            if delete_file_if_exists(lint.path.as_ref()) {
+            if delete_file_if_exists(lint_file.path.as_ref()) {
                 mod_edit = ModEdit::Delete;
             }
         } else {
-            updater.update_file(&lint.path, &mut |_, src, dst| -> UpdateStatus {
+            updater.update_file(&lint_file.path, &mut |_, src, dst| -> UpdateStatus {
                 let mut start = &src[..lint.span.start as usize];
                 if start.ends_with("\n\n") {
                     start = &start[..start.len() - 1];
@@ -96,27 +97,28 @@ pub fn rename(clippy_version: Version, old_name: &str, new_name: &str, uplift: b
                 UpdateStatus::Changed
             });
         }
-        delete_test_files(old_name, &list.lints);
-    } else if let StdEntry::Vacant(entry) = list.lints.entry(new_name.to_owned()) {
-        if lint.module.ends_with(old_name)
-            && lint
+        delete_test_files(old_name, &data.lints);
+    } else if let StdEntry::Vacant(entry) = data.lints.entry(new_name.to_owned()) {
+        let lint_file = &mut data.source_map.files[lint.span.file];
+        if lint_file.module.ends_with(old_name)
+            && lint_file
                 .path
                 .file_stem()
                 .is_some_and(|x| x.as_encoded_bytes() == old_name.as_bytes())
         {
-            let mut new_path = lint.path.with_file_name(new_name).into_os_string();
+            let mut new_path = lint_file.path.with_file_name(new_name).into_os_string();
             new_path.push(".rs");
-            if try_rename_file(lint.path.as_ref(), new_path.as_ref()) {
-                lint.path = new_path.into();
+            if try_rename_file(lint_file.path.as_ref(), new_path.as_ref()) {
+                lint_file.path = new_path.into();
                 mod_edit = ModEdit::Rename;
-            }
 
-            let mod_len = lint.module.len();
-            lint.module.truncate(mod_len - old_name.len());
-            lint.module.push_str(new_name);
+                let mod_len = lint_file.module.len();
+                lint_file.module.truncate(mod_len - old_name.len());
+                lint_file.module.push_str(new_name);
+            }
         }
         entry.insert(Lint::Active(lint));
-        rename_test_files(old_name, new_name, &list.lints);
+        rename_test_files(old_name, new_name, &data.lints);
     } else {
         println!("Renamed `clippy::{old_name}` to `clippy::{new_name}`");
         println!("Since `{new_name}` already exists the existing code has not been changed");
@@ -130,7 +132,7 @@ pub fn rename(clippy_version: Version, old_name: &str, new_name: &str, uplift: b
             updater.update_file(e.path(), &mut update_fn);
         }
     }
-    generate_lint_files(UpdateMode::Change, &list);
+    generate_lint_files(UpdateMode::Change, &data);
 
     if uplift {
         println!("Uplifted `clippy::{old_name}` as `{new_name}`");