RProject improvements, Config Specialization, Bugfixes

package.json

@@ -233,6 +233,7 @@
     "rotating-file-stream": "^3.2.8",
     "seedrandom": "^3.0.5",
     "semver": "^7.7.4",
+    "smol-toml": "^1.7.0",
     "tar": "^7.4.3",
     "tmp": "^0.2.3",
     "ts-essentials": "^10.1.1",

scripts/checkup-parallel.ts

@@ -60,7 +60,38 @@ const runningVitest = Math.max(1, jobs.filter(j => vitestIds.has(j.id)).length);
 const perVitest = Math.max(1, Math.floor((cores - 1) / runningVitest));
 jobs = jobs.map(j => vitestIds.has(j.id) ? { ...j, argv: [...j.argv, `--maxWorkers=${perVitest}`] } : j);
 
-const logDir = fs.mkdtempSync(path.join(os.tmpdir(), 'flowr-checkup-'));
+const logPrefix = 'flowr-checkup-';
+
+/** log dirs stay around to be inspected after a run, so prune the older ones to not pile up in tmp */
+function pruneOldLogDirs(keep: number): void {
+	let dirs: { path: string, mtime: number }[];
+	try {
+		dirs = fs.readdirSync(os.tmpdir())
+			.filter(n => n.startsWith(logPrefix))
+			.map(n => path.join(os.tmpdir(), n))
+			.flatMap(p => {
+				try {
+					const stat = fs.statSync(p);
+					return stat.isDirectory() ? [{ path: p, mtime: stat.mtimeMs }] : [];
+				} catch{
+					return [];
+				}
+			});
+	} catch{
+		return;
+	}
+	for(const { path: p } of dirs.sort((a, b) => b.mtime - a.mtime).slice(keep)) {
+		try {
+			fs.rmSync(p, { recursive: true, force: true });
+		} catch{
+			// a concurrent checkup may own it, leave it be
+		}
+	}
+}
+
+// keep the previous run, so that with this run's dir at most two remain
+pruneOldLogDirs(1);
+const logDir = fs.mkdtempSync(path.join(os.tmpdir(), logPrefix));
 const bold = (s: string) => `\x1b[1m${s}\x1b[0m`;
 const green = (s: string) => `\x1b[32m${s}\x1b[0m`;
 const red = (s: string) => `\x1b[31m${s}\x1b[0m`;

src/benchmark/slicer.ts

@@ -9,7 +9,7 @@ import { log, LogLevel } from '../util/log';
 import type { MergeableRecord } from '../util/objects';
 import type { DataflowInformation } from '../dataflow/info';
 import type { SliceResult } from '../slicing/static/slicer-types';
-import type { ReconstructionResult } from '../reconstruct/reconstruct';
+import type { InlineFull, ReconstructionResult } from '../reconstruct/reconstruct';
 import type { PipelineExecutor } from '../core/pipeline-executor';
 import { guard } from '../util/assert';
 import { withoutWhitespace } from '../util/text/strings';
@@ -137,7 +137,8 @@ export class BenchmarkSlicer {
 	 * Can only be called once for each instance.
 	 */
 	public async init(request: RParseRequestFromFile | RParseRequestFromText, config: FlowrConfig,
-		autoSelectIf?: AutoSelectPredicate, threshold?: number, inlineSources?: boolean, includeCallees?: boolean) {
+		autoSelectIf?: AutoSelectPredicate, threshold?: number, inlineSources?: boolean, includeCallees?: boolean,
+		inlineFull?: InlineFull) {
 		guard(this.stats === undefined, 'cannot initialize the slicer twice');
 
 		// we know these are in sync so we just cast to one of them
@@ -159,6 +160,7 @@ export class BenchmarkSlicer {
 			threshold,
 			inlineSources,
 			includeCallees,
+			inlineFull,
 		});
 
 		this.loadedXml = (await this.measureCommonStep('parse', 'retrieve AST from R code')).files.map(p => p.parsed);

src/cli/common/options.ts

@@ -77,6 +77,8 @@ export const slicerOptions = [
 	{ name: 'output',            alias: 'o', type: String,  description: 'File to write all the generated quads to (defaults to the commandline)', typeLabel: '{underline file}' },
 	{ name: 'no-magic-comments', alias: 'm', type: Boolean, description: 'Disable the effects of magic comments which force lines to be included.' },
 	{ name: 'inline',                        type: Boolean, description: 'Inline resolvable {italic source()} calls into the reconstruction so the slice is a single self-contained R text.' },
+	{ name: 'inline-full',       alias: 'I', type: Boolean, description: 'Inline {italic all} files into one, in flowR\'s loading order (respecting implicit sources), independent of whether they are sourced explicitly.' },
+	{ name: 'inline-banner',                 type: Boolean, description: 'Together with {bold --inline-full}: precede every inlined file with a banner comment naming it.' },
 	{ name: 'include-callees',               type: Boolean, description: 'If slicing backward, continue past a function-definition boundary, also including the definition\'s binding and call sites.' },
 	{ name: 'api',                           type: Boolean, description: 'Instead of human-readable output, dump a lot of json with the results of all intermediate steps.' },
 ] as const satisfies OptionDefinition[];

src/cli/repl/commands/repl-query.ts

@@ -16,21 +16,8 @@ import { jsonReplacer } from '../../../util/json';
 import type { BaseQueryResult } from '../../../queries/base-query-format';
 import { splitAtEscapeSensitive } from '../../../util/text/args';
 import { Record } from '../../../util/record';
-import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever';
-import { watchProtocol } from '../core';
-import fs from 'fs';
-
-/** Whether `s` looks like a filesystem path the user likely meant to load via {@link fileProtocol}. */
-function looksLikePath(s: string): boolean {
-	if(s.startsWith(fileProtocol) || s.startsWith(watchProtocol)) {
-		return false;
-	}
-	if(/^(~|\.{0,2}\/|[a-zA-Z]:[\\/])/.test(s)) {
-		return true;
-	}
-	// a single path-like token (a separator, no R-code punctuation) that actually exists on disk
-	return /^[^\s()]+\/[^\s()]+$/.test(s) && fs.existsSync(s);
-}
+import { requestFromInput } from '../../../r-bridge/retriever';
+import { handlePathLikeInput } from '../path-input';
 
 /**
  * Whether the analyzer already holds exactly `input` as its sole request, so a prior analysis (e.g. a
@@ -129,9 +116,7 @@ async function processQueryArgs(output: ReplOutput, analyzer: FlowrAnalysisProvi
 	}
 
 	if(input) {
-		if(looksLikePath(input)) {
-			output.stdout(ansiInfo(`'${input}' looks like a path. To analyze it, use ${bold(fileProtocol + input, output.formatter)} (or ${bold(watchProtocol + input, output.formatter)} to re-run on changes). Use ${bold(':help', output.formatter)} for more.`));
-		}
+		input = handlePathLikeInput(output, input, analyzer.flowrConfig);
 		// reuse a prior analysis (e.g. a dataflow from :df#) when the target is unchanged
 		if(!analyzerHasTarget(analyzer, input)) {
 			analyzer.reset();

src/cli/repl/core.ts

@@ -3,14 +3,15 @@
  * @module
  */
 import { prompt } from './prompt';
+import { handlePathLikeInput, watchProtocol } from './path-input';
 import * as readline from 'readline';
 import { installGhostHint } from './ghost-hint';
 import { tryExecuteRShellCommand } from './commands/repl-execute';
 import os from 'os';
 import path from 'path';
 import fs from 'fs';
 import { splitAtEscapeSensitive } from '../../util/text/args';
-import { ColorEffect, Colors, FontStyles } from '../../util/text/ansi';
+import { ansiFormatter, ColorEffect, Colors, FontStyles } from '../../util/text/ansi';
 import { getCommand, getCommandNames } from './commands/repl-commands';
 import { getValidOptionsForCompletion, scripts } from '../common/scripts-info';
 import { fileProtocol } from '../../r-bridge/retriever';
@@ -46,9 +47,16 @@ export interface CommandCompletions {
 	 * This is relevant if an argument is composed of multiple parts (e.g. comma-separated lists).
 	 */
 	readonly argumentPart?: string;
+	/** What Tab displays per completion, only used while several remain: readline inserts what it displays. */
+	readonly labels?:       ReadonlyMap<string, string>;
 }
 
-function computeCompletions(line: string, config: FlowrConfig): [string[], string] {
+/** Labels a completion with what it does, see {@link CommandCompletions#labels|labels}. */
+export function describeCompletion(insert: string, describe: string): string {
+	return `${insert}  ${ansiFormatter.format(describe, { style: FontStyles.Faint })}`;
+}
+
+function computeCompletions(line: string, config: FlowrConfig): [string[], string, ReadonlyMap<string, string>?] {
 	const splitLine = splitAtEscapeSensitive(line);
 	// did we just type a space (and are starting a new arg right now)?
 	const startingNewArg = line.endsWith(' ');
@@ -59,6 +67,7 @@ function computeCompletions(line: string, config: FlowrConfig): [string[], strin
 		if(commandNameColon) {
 			let completions: string[] = [];
 			let currentArg = startingNewArg ? '' : splitLine[splitLine.length - 1];
+			let labels: ReadonlyMap<string, string> | undefined;
 
 			const commandName = commandNameColon.slice(1);
 			const cmd = getCommand(commandName);
@@ -67,11 +76,12 @@ function computeCompletions(line: string, config: FlowrConfig): [string[], strin
 				const options = scripts[commandName as keyof typeof scripts].options;
 				completions = completions.concat(getValidOptionsForCompletion(options, splitLine).map(o => `${o} `));
 			} else if(commandName.startsWith('query')) {
-				const { completions: queryCompletions, argumentPart: splitArg } = replQueryCompleter(splitLine, startingNewArg, config);
+				const { completions: queryCompletions, argumentPart: splitArg, labels: queryLabels } = replQueryCompleter(splitLine, startingNewArg, config);
 				if(splitArg !== undefined) {
 					currentArg = splitArg;
 				}
 				completions = completions.concat(queryCompletions);
+				labels = queryLabels;
 			} else if(cmd === signatureCommand) {
 				const { completions: sigCompletions, argumentPart: splitArg } = replSignatureCompleter(splitLine, startingNewArg);
 				if(splitArg !== undefined) {
@@ -83,7 +93,7 @@ function computeCompletions(line: string, config: FlowrConfig): [string[], strin
 				completions.push(watchProtocol);
 			}
 
-			return [completions.filter(a => a.startsWith(currentArg)), currentArg];
+			return [completions.filter(a => a.startsWith(currentArg)), currentArg, labels];
 		}
 	}
 
@@ -96,7 +106,11 @@ function computeCompletions(line: string, config: FlowrConfig): [string[], strin
  * option, so Tab shows the full menu and only completes when a single option remains (standard readline behavior).
  */
 export function replCompleter(line: string, config: FlowrConfig): [string[], string] {
-	return computeCompletions(line, config);
+	const [completions, fragment, labels] = computeCompletions(line, config);
+	if(labels === undefined || completions.length <= 1) {
+		return [completions, fragment];
+	}
+	return [completions.map(c => labels.get(c) ?? c), fragment];
 }
 
 /**
@@ -161,9 +175,6 @@ export function handleString(code: string) {
 	};
 }
 
-/** Path prefix that activates watch mode; use instead of {@link fileProtocol} to re-run on every file change. */
-export const watchProtocol = 'watch://';
-
 /** Convert a statement containing {@link watchProtocol} into one using {@link fileProtocol} so it can be executed. */
 export function toFileStatement(statement: string): string {
 	return statement.replaceAll(watchProtocol, fileProtocol);
@@ -228,7 +239,7 @@ async function executeStatement(output: ReplOutput, statement: string, analyzer:
 						const args = processor.argsParser(remainingLine);
 						if(args.rCode) {
 							analyzer.reset();
-							analyzer.addRequest(args.rCode);
+							analyzer.addRequest(handlePathLikeInput(output, args.rCode, analyzer.flowrConfig));
 						}
 						await processor.fn({ output, analyzer, remainingArgs: args.remaining });
 					} else {

src/cli/repl/parser/slice-query-parser.ts

@@ -1,7 +1,10 @@
 import type { SlicingCriterion, SlicingCriteria } from '../../../slicing/criterion/parse';
 import { SliceDirection } from '../../../util/slice-direction';
-import type { CommandCompletions } from '../core';
+import { type CommandCompletions, describeCompletion } from '../core';
 import type { FlowrConfig } from '../../../config';
+import type { SliceQueryOptions } from '../../../queries/catalog/slice-query-options';
+import type { ReplOutput } from '../commands/repl-main';
+import { ColorEffect, Colors } from '../../../util/text/ansi';
 
 /**
  * Splits `(criteria)flags` into its two parts, matching the closing bracket by depth: a criterion may well
@@ -27,6 +30,28 @@ function sliceFlagSuffix(argument: string | undefined): string {
 	return splitCriteriaArgument(argument)?.flags ?? '';
 }
 
+/** A flag that may follow the closing bracket of a criteria argument. */
+export interface SliceFlag {
+	readonly flag:       string
+	/** what the flag does, kept to a few words as it is offered as a completion */
+	readonly describe:   string
+	/** the flag this one only makes sense with, if any */
+	readonly requires?:  string
+	/** the flags this one must not be combined with */
+	readonly conflicts?: readonly string[]
+}
+
+/** the flags every slicing query understands, see {@link sliceQueryOptionsParser} */
+export const SharedSliceFlags = [
+	{ flag: 'i', describe: 'inline sources', conflicts: ['I'] },
+	{ flag: 'c', describe: 'include callees' },
+	{ flag: 'I', describe: 'inline all files', conflicts: ['i'] },
+	{ flag: 'B', describe: 'banners', requires: 'I' }
+] as const satisfies readonly SliceFlag[];
+
+/** the flags of the `static-slice` query: a dice fixes both directions, so only it can be told to slice forward */
+export const StaticSliceFlags = [{ flag: 'f', describe: 'slice forward' }, ...SharedSliceFlags] as const satisfies readonly SliceFlag[];
+
 /**
  * Checks whether the given argument represents a slicing direction with an `f` suffix (in any flag order).
  */
@@ -35,19 +60,53 @@ export function sliceDirectionParser(argument: string): SliceDirection {
 }
 
 /**
- * Whether the argument requests inline slicing via an `i` suffix (e.g. `(12@x)i`, `(12@x)fi`), which inlines
- * resolvable `source()` calls into the reconstruction so the slice is a single self-contained R text.
+ * The {@link SliceQueryOptions} the flag suffix of `argument` requests, see {@link SharedSliceFlags}.
+ * An absent flag is left out entirely, so the default of the query applies.
+ */
+export function sliceQueryOptionsParser(argument: string): SliceQueryOptions {
+	const flags = sliceFlagSuffix(argument);
+	return {
+		...(flags.includes('i') ? { inlineSources: true } : {}),
+		...(flags.includes('I') ? { inlineFull: flags.includes('B') ? 'banner' as const : true } : {}),
+		...(flags.includes('c') ? { includeCallees: true } : {})
+	};
+}
+
+/** the given flags and what they do, e.g. for a help text: `f (slice forward), B (banners, needs I)` */
+export function describeSliceFlags(flags: readonly SliceFlag[]): string {
+	return flags.map(f => `${f.flag} (${f.describe}${f.requires ? `, needs ${f.requires}` : ''})`).join(', ');
+}
+
+function warn(output: ReplOutput, message: string): void {
+	output.stderr(output.formatter.format(message, { color: Colors.Yellow, effect: ColorEffect.Foreground }));
+}
+
+/**
+ * Warns about the flags of `argument` that `flags` does not know (a `f` on a dice, a typo, ...) or that
+ * {@link SliceFlag#conflicts|conflict} with each other, as they are applied silently otherwise.
  */
-export function sliceInlineParser(argument: string): boolean {
-	return sliceFlagSuffix(argument).includes('i');
+export function warnAboutSliceFlags(output: ReplOutput, argument: string, flags: readonly SliceFlag[]): void {
+	const given = [...new Set([...sliceFlagSuffix(argument)])];
+	const known = new Map(flags.map(f => [f.flag, f]));
+	const unknown = given.filter(f => !known.has(f));
+	if(unknown.length > 0) {
+		warn(output, `Ignoring unknown flag${unknown.length > 1 ? 's' : ''} ${unknown.map(f => `'${f}'`).join(', ')}. Known flags: ${describeSliceFlags(flags)}.`);
+	}
+	for(const [i, f] of given.entries()) {
+		const clash = known.get(f)?.conflicts?.filter(c => given.indexOf(c) > i) ?? [];
+		if(clash.length > 0) {
+			warn(output, `The flag '${f}' cannot be combined with ${clash.map(c => `'${c}'`).join(', ')}, the latter wins.`);
+		}
+	}
 }
 
 /**
- * Whether the argument requests to slice past function-definition boundaries via a `c` suffix
- * (e.g. `(12@x)c`, `(12@x)ic`), including the definition's binding and call sites in a backward slice.
+ * The R code of a query line, i.e. everything after the argument at `from`. The line is split at whitespace, so
+ * unquoted code arrives as several parts and only re-joining them yields all of it.
  */
-export function sliceIncludeCalleesParser(argument: string): boolean {
-	return sliceFlagSuffix(argument).includes('c');
+export function queryLineCode(line: readonly string[], from = 1): string | undefined {
+	const code = line.slice(from).join(' ').trim();
+	return code.length > 0 ? code : undefined;
 }
 
 /**
@@ -69,9 +128,34 @@ function lastCriterionFragment(arg: string): string {
 	return arg.slice(Math.max(arg.indexOf('(') + 1, arg.lastIndexOf(';') + 1));
 }
 
+/**
+ * The completions for the flag suffix of `arg`: every flag that fits the ones it carries already, plus a
+ * trailing space to move on to the code. Returns `undefined` while the criteria are still open.
+ */
+export function sliceFlagCompletions(arg: string, flags: readonly SliceFlag[]): CommandCompletions | undefined {
+	const split = splitCriteriaArgument(arg);
+	if(!split) {
+		return undefined;
+	}
+	const has = (flag: string) => split.flags.includes(flag);
+	const offered = flags.filter(f =>
+		!has(f.flag)
+		&& (f.requires === undefined || has(f.requires))
+		&& !f.conflicts?.some(has)
+	);
+	return {
+		completions: [...offered.map(f => arg + f.flag), arg + ' '],
+		labels:      new Map([
+			...offered.map(f => [arg + f.flag, describeCompletion(f.flag, f.describe)] as const),
+			[arg + ' ', describeCompletion('<space>', 'then the code')] as const
+		]),
+		argumentPart: arg
+	};
+}
+
 /**
  * Tab-completer for query arguments of the form `(line@var;line@var;...)`.
- * Guides the user step by step: `(` then digits then `@` then variable then `)`.
+ * Guides the user step by step: `(` then digits then `@` then variable then `)`, then the flags.
  */
 export function criteriaQueryCompleter(line: readonly string[], startingNewArg: boolean, _config: FlowrConfig): CommandCompletions {
 	if(line.length === 0) {
@@ -81,8 +165,9 @@ export function criteriaQueryCompleter(line: readonly string[], startingNewArg:
 		return { completions: [] };
 	}
 	const arg = line[0];
-	if(arg.endsWith(')') || arg.endsWith(')f')) {
-		return { completions: [] };
+	const flags = sliceFlagCompletions(arg, StaticSliceFlags);
+	if(flags) {
+		return flags;
 	}
 	const fragment = lastCriterionFragment(arg);
 	if(/^\d+$/.test(fragment)) {