Add support for targets::tar_source() and sourceDir()

crates/ark/src/lsp/diagnostics.rs

@@ -1000,7 +1000,7 @@ fn handle_package_attach_call(node: Node, context: &mut DiagnosticContext) -> an
     let package_name = package_node.get_identifier_or_string_text(context.contents())?;
     let attach_pos = node.end_position();
 
-    insert_package_exports(package_name, attach_pos, context)?;
+    insert_package_exports(package_name, attach_pos, context);
 
     // Also attach packages from `Depends` field, if any
     if let Some(package_names) = context
@@ -1009,7 +1009,7 @@ fn handle_package_attach_call(node: Node, context: &mut DiagnosticContext) -> an
         .and_then(|package| package.depends(context.db).as_ref())
     {
         for package_name in package_names.iter() {
-            insert_package_exports(package_name, attach_pos, context)?;
+            insert_package_exports(package_name, attach_pos, context);
         }
     }
 
@@ -1055,21 +1055,16 @@ fn handle_package_attach_call(node: Node, context: &mut DiagnosticContext) -> an
         _ => vec![],
     };
     for package_name in attach_field {
-        insert_package_exports(package_name, attach_pos, context)?;
+        insert_package_exports(package_name, attach_pos, context);
     }
 
     Ok(())
 }
 
-fn insert_package_exports(
-    package_name: &str,
-    attach_pos: Point,
-    context: &mut DiagnosticContext,
-) -> anyhow::Result<()> {
+fn insert_package_exports(package_name: &str, attach_pos: Point, context: &mut DiagnosticContext) {
     let Some(package) = context.db.package_by_name(package_name) else {
-        return Err(anyhow::anyhow!(
-            "Can't get exports from package {package_name} because it is not installed."
-        ));
+        // Package is not installed. This is linted via another path.
+        return;
     };
 
     // Start from explicit `NAMESPACE` exports
@@ -1088,8 +1083,6 @@ fn insert_package_exports(
         .entry(attach_pos)
         .or_default()
         .extend(exports);
-
-    Ok(())
 }
 
 fn recurse_subset_or_subset2(

crates/oak_db/src/db.rs

@@ -209,6 +209,17 @@ pub fn workspace_files(db: &dyn Db) -> Vec<File> {
     files
 }
 
+/// The scripts held directly by workspace roots, in root order.
+/// Like [`workspace_files`] but without package files.
+#[salsa::tracked(returns(ref))]
+pub(crate) fn workspace_scripts(db: &dyn Db) -> Vec<File> {
+    db.workspace_roots()
+        .roots(db)
+        .iter()
+        .flat_map(|root| root.scripts(db).iter().copied())
+        .collect()
+}
+
 fn collect_root_files(db: &dyn Db, files: &mut Vec<File>, r: Root) {
     let owned = |f: File| root_by_file(db, f) == Some(r);
     files.extend(r.scripts(db).iter().copied().filter(|&f| owned(f)));

crates/oak_db/src/file_imports.rs

@@ -10,6 +10,7 @@ use oak_semantic::semantic_index::SemanticCallKind;
 use oak_semantic::semantic_index::SemanticIndex;
 use rustc_hash::FxHashSet;
 
+use crate::db::workspace_scripts;
 use crate::Db;
 use crate::File;
 use crate::Package;
@@ -375,35 +376,18 @@ impl File {
         package.files(db).contains(&self)
     }
 
-    /// The collation members of `self`'s own `R/` directory, in load order:
-    /// sorted by basename, ASCII case-insensitively, the same order a
-    /// package `R/` with no `Collate:` gets from
-    /// `oak_scan::packages::order_alphabetically`.
+    /// The collation members of `self`'s own `R/` directory, in load order.
     ///
     /// Path-based only. The scan-time resolver
     /// ([`SalsaImportsResolver`](crate::imports::SalsaImportsResolver)) calls
     /// `cross_file_layers` while `self`'s own semantic index is still being
     /// built. The query can't recurse into the index.
-    ///
-    /// Gathers candidates from workspace roots only (`root.scripts(db)` for
-    /// each). `OrphanRoot`, library roots, and `StaleRoot` don't contribute
-    /// collation siblings.
     #[salsa::tracked(returns(ref))]
     pub(crate) fn collation_siblings(self, db: &dyn Db) -> Vec<File> {
         let Some(dir) = self.path(db).as_path().and_then(Utf8Path::parent) else {
             return Vec::new();
         };
-
-        let mut siblings: Vec<File> = db
-            .workspace_roots()
-            .roots(db)
-            .iter()
-            .flat_map(|root| root.scripts(db).iter().copied())
-            .filter(|file| file.path(db).as_path().and_then(Utf8Path::parent) == Some(dir))
-            .collect();
-
-        siblings.sort_by_cached_key(|file| collation_basename_key(*file, db));
-        siblings
+        files_in_directory(db, dir)
     }
 }
 
@@ -463,7 +447,18 @@ fn build_inherited_layers(
         None => ImportLayer::File(source_site),
     };
 
-    let mut above = vec![source_layer];
+    // One Source effect can load several files (`sourceDir()`, `tar_source()`).
+    // Keep track of already sourced files, since their eager top-level context
+    // can see what's already been sourced.
+    let mut above: Vec<ImportLayer> = match offset {
+        Some(offset) => loaded_before(db, source_site, file, offset)
+            .into_iter()
+            .map(ImportLayer::File)
+            .collect(),
+        None => Vec::new(),
+    };
+
+    above.push(source_layer);
     above.extend(own_cross.above.iter().cloned());
     above.extend(
         grandparents
@@ -498,6 +493,22 @@ fn source_offset(db: &dyn Db, source_file: File, file: File) -> Option<TextSize>
         .max()
 }
 
+/// The files the call at `offset` loaded before `file`, latest first.
+fn loaded_before(db: &dyn Db, source_file: File, file: File, offset: TextSize) -> Vec<File> {
+    let sites = source_file.source_sites(db);
+    let Some(own) = sites.iter().rposition(|site| site.target() == Some(file)) else {
+        return Vec::new();
+    };
+
+    sites[..own]
+        .iter()
+        .rev()
+        .take_while(|site| site.offset() == offset)
+        .filter_map(|site| site.target())
+        .filter(|target| *target != file)
+        .collect()
+}
+
 fn package_load_layers(
     file: File,
     db: &dyn Db,
@@ -721,7 +732,37 @@ fn testthat_support_key(file: File, db: &dyn Db) -> Cow<'_, str> {
     file.path(db).file_name().unwrap_or_default()
 }
 
-/// Case-insensitive basename sort key for `collation_siblings`, matching
+/// The workspace files sitting directly in `dir`, in load order: sorted by
+/// basename, ASCII case-insensitively, the same order a package `R/` with no
+/// `Collate:` gets from `oak_scan::packages::order_alphabetically`.
+pub(crate) fn files_in_directory(db: &dyn Db, dir: &Utf8Path) -> Vec<File> {
+    let mut files: Vec<File> = workspace_scripts(db)
+        .iter()
+        .copied()
+        .filter(|file| file.path(db).as_path().and_then(Utf8Path::parent) == Some(dir))
+        .collect();
+
+    files.sort_by_cached_key(|file| collation_basename_key(*file, db));
+    files
+}
+
+/// The workspace files anywhere under `dir`, approximating the order returned by
+/// `list.files(dir, recursive = TRUE)`.
+pub(crate) fn files_in_directory_recursive(db: &dyn Db, dir: &Utf8Path) -> Vec<File> {
+    let mut keyed: Vec<(String, File)> = workspace_scripts(db)
+        .iter()
+        .copied()
+        .filter_map(|file| {
+            let relative = file.path(db).as_path()?.strip_prefix(dir).ok()?;
+            Some((relative.as_str().to_ascii_lowercase(), file))
+        })
+        .collect();
+
+    keyed.sort_by(|(left, _), (right, _)| left.cmp(right));
+    keyed.into_iter().map(|(_, file)| file).collect()
+}
+
+/// Case-insensitive basename sort key for [`files_in_directory`], matching
 /// `oak_scan::packages::order_alphabetically`'s `basename_key` so a
 /// non-package `R/` collates the same way as a package `R/` with no
 /// `Collate:`.

crates/oak_db/src/imports.rs

@@ -11,6 +11,7 @@ use oak_semantic::SourceResolution;
 use rustc_hash::FxHashMap;
 use url::Url;
 
+use crate::file_imports::files_in_directory_recursive;
 use crate::file_imports::CollationView;
 use crate::file_imports::ImportLayer;
 use crate::Db;
@@ -56,6 +57,53 @@ impl<'db> SalsaImportsResolver<'db> {
             cache: EffectsCache::default(),
         }
     }
+
+    /// What sourcing `file` brings in. The two reads this makes are the ones
+    /// described on [`SalsaImportsResolver`].
+    fn source_resolution(&self, file: File) -> SourceResolution {
+        let names: Vec<String> = file
+            .exports(self.db)
+            .iter()
+            .map(|(name, _)| name.to_string())
+            .collect();
+
+        let packages: Vec<String> = file
+            .attached_packages(self.db)
+            .iter()
+            .map(|name| name.text(self.db).to_string())
+            .collect();
+
+        SourceResolution {
+            url: file.path(self.db).to_url(),
+            names,
+            packages,
+        }
+    }
+}
+
+/// The workspace files a `SourceTarget::Dir` path names, in load order. Empty
+/// when the path doesn't anchor.
+///
+/// Tracked to give the directory listing a backdating point, the role
+/// [`File::collation_siblings`] plays for the `R/` convention. The listing
+/// filters every root's scripts, so without a memo here a file appearing
+/// anywhere in the workspace would re-run the whole `semantic_index` of every
+/// file holding a directory source, `_targets.R` being the one that hurts.
+///
+/// Reads only inputs, so it's safe to call while `file`'s own index is being
+/// built.
+#[salsa::tracked(returns(ref))]
+pub(crate) fn source_dir_scripts(db: &dyn Db, file: File, path: String) -> Vec<File> {
+    let Some(anchor) = anchor_dir(db, file) else {
+        return Vec::new();
+    };
+    let Some(target_path) = resolve_relative_to(&anchor, &path) else {
+        return Vec::new();
+    };
+    let Some(dir) = target_path.as_path() else {
+        return Vec::new();
+    };
+    files_in_directory_recursive(db, dir)
 }
 
 /// Per-build memo for `resolve_effects`, keyed on `(name, attached)`.
@@ -97,24 +145,17 @@ impl<'db> ImportsResolver for SalsaImportsResolver<'db> {
         // TODO(diagnostics): Until we support out-of-workspace sourced files,
         // should we at least lint so user knows that we can't analyse the file?
         let file = self.db.file_by_path(&target_path)?;
+        Some(self.source_resolution(file))
+    }
 
-        let names: Vec<String> = file
-            .exports(self.db)
-            .iter()
-            .map(|(name, _)| name.to_string())
-            .collect();
-
-        let packages: Vec<String> = file
-            .attached_packages(self.db)
+    fn resolve_source_dir(&mut self, path: &str) -> Vec<SourceResolution> {
+        source_dir_scripts(self.db, self.file, path.to_string())
             .iter()
-            .map(|name| name.text(self.db).to_string())
-            .collect();
-
-        Some(SourceResolution {
-            url: target_path.to_url(),
-            names,
-            packages,
-        })
+            .copied()
+            // Exclude sourcing file
+            .filter(|file| *file != self.file)
+            .map(|file| self.source_resolution(file))
+            .collect()
     }
 
     fn resolve_effects(&mut self, name: &str, attached: &[String]) -> Option<EffectsHandlers> {