diff --git a/src/server/context.ts b/src/server/context.ts index 2fbda5ae..08a64ca4 100644 --- a/src/server/context.ts +++ b/src/server/context.ts @@ -145,6 +145,11 @@ const contexts: Context[] = []; // Reuse `motoko` npm package instances to limit memory usage const motokoInstances = new Map(); +// Discovered project directories not yet loaded (uri → dir path) +const pendingDirectories = new Map(); +// Deduplication of in-flight lazy loads (uri → loading promise) +const loadingPromises = new Map>(); + function getMotokoInstanceKey( uri: string, version: Version, @@ -338,12 +343,48 @@ requestDefaultContext(); // Always add a default context */ export function resetContexts() { contexts.length = 0; + pendingDirectories.clear(); + loadingPromises.clear(); if (defaultContext) { defaultContext = undefined; requestDefaultContext(); // Regenerate default context } } +export function registerPendingDirectory(uri: string, dir: string) { + pendingDirectories.set(uri, dir); +} + +/** + * Find the pending directory whose URI is the longest prefix of the given file URI. + */ +export function findPendingDirectoryForUri( + fileUri: string, +): { uri: string; dir: string } | undefined { + let match: { uri: string; dir: string } | undefined; + for (const [uri, dir] of pendingDirectories) { + if (fileUri.startsWith(uri)) { + if (!match || uri.length > match.uri.length) { + match = { uri, dir }; + } + } + } + return match; +} + +export function removePendingDirectory(uri: string) { + pendingDirectories.delete(uri); + loadingPromises.delete(uri); +} + +export function getLoadingPromise(uri: string): Promise | undefined { + return loadingPromises.get(uri); +} + +export function setLoadingPromise(uri: string, promise: Promise) { + loadingPromises.set(uri, promise); +} + /** * Register a context for the given directory (specified as a URI). */ @@ -400,13 +441,16 @@ export function allContexts(): Context[] { /** * Find the most relevant context for the given URI. + * Falls back to the default context for URIs with a pending (not yet loaded) project. */ export function getContext(uri: string): Context { const context = contexts.find((context) => uri.startsWith(context.uri)); if (context) { return context; } - console.warn('Unknown context for URI:', uri); + if (!findPendingDirectoryForUri(uri)) { + console.warn('Unknown context for URI:', uri); + } return requestDefaultContext(); } diff --git a/src/server/handlers.ts b/src/server/handlers.ts index e20a30a7..3b3c74ad 100644 --- a/src/server/handlers.ts +++ b/src/server/handlers.ts @@ -56,6 +56,11 @@ import { allContexts, getContext, resetContexts, + registerPendingDirectory, + findPendingDirectoryForUri, + removePendingDirectory, + getLoadingPromise, + setLoadingPromise, } from './context'; import { addContextualDotCompletions } from './completions'; import DfxResolver from './dfx'; @@ -96,6 +101,8 @@ import { getRelativeUri, isExternalUri, rangeContainsPosition, + readSourcesCache, + writeSourcesCache, resolveFilePath, resolveVirtualPath, } from './utils'; @@ -200,43 +207,31 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { if (!sources.length) { // Prioritize MOPS over Vessel if (existsSync(join(directory, 'mops.toml'))) { - // let command = 'mops sources'; - let command = 'npx --no ic-mops sources'; - try { - const mopsVersion = execSync( - 'npx --no ic-mops -- --version', - ) - .toString() - .split(/\s/)[1]; - if (semver.gte(mopsVersion, '0.45.3')) { - command += ' --no-install'; + const diskCache = readSourcesCache(directory); + if (diskCache) { + console.log('Using cached mops sources for:', directory); + sources = diskCache; + } else { + // let command = 'mops sources'; + let command = 'npx --no ic-mops sources'; + try { + const mopsVersion = execSync( + 'npx --no ic-mops -- --version', + ) + .toString() + .split(/\s/)[1]; + if (semver.gte(mopsVersion, '0.45.3')) { + command += ' --no-install'; + } + sources = await sourcesFromCommand(command); + writeSourcesCache(directory, sources); + } catch (err: any) { + throw new Error( + `Error while finding Mops packages.\nMake sure the latest version of Mops is installed locally or globally (https://docs.mops.one/quick-start).\n${ + err?.message || err + }`, + ); } - sources = await sourcesFromCommand(command); - } catch (err: any) { - // try { - // const sources = await mopsSources(directory); - // if (!sources) { - // throw new Error('Unexpected output'); - // } - // return Object.entries(sources); - // } catch (fallbackError) { - // console.error( - // `Error in fallback Mops implementation:`, - // fallbackError, - // ); - // // Provide a verbose error message for Mops command - // throw new Error( - // `Error while running \`${command}\`: ${ - // err?.message || err - // }`, - // ); - // } - - throw new Error( - `Error while finding Mops packages.\nMake sure the latest version of Mops is installed locally or globally (https://docs.mops.one/quick-start).\n${ - err?.message || err - }`, - ); } } else if (existsSync(join(directory, 'vessel.dhall'))) { const command = 'vessel sources'; @@ -258,15 +253,18 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { } let isVirtualFileSystemReady = false; - let loadingPackages = false; let packageConfigChangeTimeout: ReturnType; + + /** + * Discover project directories and register them as pending. + * Actual context creation is deferred until a file in the project is opened. + */ function notifyPackageConfigChange(reuseCached = false) { isVirtualFileSystemReady = false; isWorkspaceReady = false; if (!reuseCached) { packageSourceCache.clear(); } - loadingPackages = true; clearTimeout(packageConfigChangeTimeout); packageConfigChangeTimeout = setTimeout(async () => { try { @@ -307,115 +305,184 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { ); } - await Promise.all( - directories.map(async (dir) => { - try { - console.log('Loading packages for directory:', dir); - - let overrideMotokoVersion: string | undefined; - if (!initializationOptions.useDefaultMocJs) { - const res = await getWorkspaceMocVersion(dir); - if (res.isOk()) { - overrideMotokoVersion = res.value.version; - console.log( - 'Detected Motoko version:', - overrideMotokoVersion, - 'from', - res.value.source, - 'in project directory:', - dir, - ); - } else { - console.warn( - 'Could not determine Motoko version in project directory', - dir, - ':', - res.error.message, - ); - } - } - - const uri = URI.file(dir).toString(); - const context = await addContext( - uri, - overrideMotokoVersion, - dir, - ); - - context.mopsArgs = getMopsMocArgs(dir); - if (context.mopsArgs.length) { - console.log( - 'Moc args from mops.toml:', - context.mopsArgs, - ); - } - - try { - context.packages = await getPackageSources(dir); - context.packages.forEach( - ([name, relativePath]) => { - const path = resolveVirtualPath( - uri, - relativePath, - ); - console.log( - 'Package:', - name, - '->', - path, - `(${uri})`, - ); - context.motoko.usePackage(name, path); - }, - ); - } catch (err) { - const detail = String(err).replace( - /^Error: /, - '', - ); - showErrorMessage( - 'Error while resolving Motoko packages:', - detail, - ); - context.error = String(err); - console.warn(err); - return; - } - } catch (err: any) { - const detail = String(err).replace(/^Error: /, ''); - showErrorMessage( - 'Error while loading Motoko packages:', - detail, - ); - console.error( - `Error while reading packages for directory (${dir}): ${err}`, - ); - return; - } - }), - ); + directories.forEach((dir) => { + const uri = URI.file(dir).toString(); + registerPendingDirectory(uri, dir); + console.log('Registered pending project directory:', dir); + }); + // Apply flags to the default context (used for files not under any project) allContexts().forEach((context) => context.applyMocFlags(settings.extraFlags), ); - loadingPackages = false; - notifyWorkspace(); // Update virtual file system - notifyDfxChange(); // Reload dfx.json - // NOTE: Useful for tests and benchmarks. - // Unknown notifications are ignored by the vscode lsp client. + notifyWorkspace(); + notifyDfxChange(); connection.sendNotification(TEST_SERVER_INITIALIZED, {}); isVirtualFileSystemReady = true; } catch (err: any) { isVirtualFileSystemReady = false; - loadingPackages = false; console.error( - `Error while loading packages: ${err?.message || err}`, + `Error while discovering projects: ${err?.message || err}`, ); } }, 1000); } + /** + * Load the compiler context for a single project directory. + * Creates the moc.js instance, resolves packages, and populates the virtual FS. + */ + async function loadProjectContext(dir: string): Promise { + console.log('Loading packages for directory:', dir); + + let overrideMotokoVersion: string | undefined; + if (!initializationOptions.useDefaultMocJs) { + const res = await getWorkspaceMocVersion(dir); + if (res.isOk()) { + overrideMotokoVersion = res.value.version; + console.log( + 'Detected Motoko version:', + overrideMotokoVersion, + 'from', + res.value.source, + 'in project directory:', + dir, + ); + } else { + console.warn( + 'Could not determine Motoko version in project directory', + dir, + ':', + res.error.message, + ); + } + } + + const uri = URI.file(dir).toString(); + const context = await addContext(uri, overrideMotokoVersion, dir); + + context.mopsArgs = getMopsMocArgs(dir); + if (context.mopsArgs.length) { + console.log('Moc args from mops.toml:', context.mopsArgs); + } + + try { + context.packages = await getPackageSources(dir); + context.packages.forEach(([name, relativePath]) => { + const path = resolveVirtualPath(uri, relativePath); + console.log('Package:', name, '->', path, `(${uri})`); + context.motoko.usePackage(name, path); + }); + } catch (err) { + const detail = String(err).replace(/^Error: /, ''); + showErrorMessage('Error while resolving Motoko packages:', detail); + context.error = String(err); + console.warn(err); + } + + context.applyMocFlags(settings.extraFlags); + populateContextWithWorkspaceFiles(context); + + return context; + } + + /** + * Write all workspace files to a single context's virtual FS and + * update its AST/import resolvers for .mo files under it. + */ + function populateContextWithWorkspaceFiles(context: Context) { + if (!workspaceFolders) return; + + const allContents: { + virtualPath: string; + uri: string; + content: string; + }[] = []; + + workspaceFolders.forEach((folder) => { + const folderPath = resolveFilePath(folder.uri); + const relativePaths = glob.sync(virtualFilePattern, { + cwd: folderPath, + dot: true, + ignore: ignoreGlobPatterns, + followSymbolicLinks: false, + }); + relativePaths.forEach((relativePath) => { + const filePath = join(folderPath, relativePath); + try { + const content = readFileSync(filePath, 'utf8'); + const virtualPath = resolveVirtualPath( + folder.uri, + relativePath, + ); + const fileUri = URI.file(filePath).toString(); + allContents.push({ virtualPath, uri: fileUri, content }); + } catch (err) { + console.error(`Error while reading file ${filePath}:`, err); + } + }); + }); + + allContents.forEach(({ virtualPath, content }) => { + context.motoko.write(virtualPath, content); + }); + + allContents.forEach(({ uri, content }) => { + if (uri.endsWith('.mo') && uri.startsWith(context.uri)) { + const { astResolver, importResolver } = context; + try { + astResolver.notify(uri, content, isVirtualFileSystemReady); + const program = astResolver.request( + uri, + isVirtualFileSystemReady, + )?.program; + importResolver.update(uri, program); + } catch (err) { + console.error(`Error while parsing (${uri}): ${err}`); + } + } + }); + } + + /** + * Ensure the compiler context for the given URI is loaded. + * If the URI belongs to a pending (not yet loaded) project, triggers lazy loading. + * Returns the loaded context, or the default context if no project matches. + */ + async function ensureContextLoaded(uri: string): Promise { + const pending = findPendingDirectoryForUri(uri); + if (!pending) { + return getContext(uri); + } + + const existing = getLoadingPromise(pending.uri); + if (existing) { + return existing; + } + + const promise = loadProjectContext(pending.dir).then( + (context) => { + removePendingDirectory(pending.uri); + return context; + }, + (err: any) => { + const detail = String(err).replace(/^Error: /, ''); + showErrorMessage( + 'Error while loading Motoko packages:', + detail, + ); + console.error( + `Error while reading packages for directory (${pending.dir}): ${err}`, + ); + removePendingDirectory(pending.uri); + return getContext(uri); + }, + ); + setLoadingPromise(pending.uri, promise); + return promise; + } + let dfxResolver: DfxResolver | undefined; let dfxChangeTimeout: ReturnType; function notifyDfxChange() { @@ -708,7 +775,8 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { } } else if ( change.uri.endsWith('.dhall') || - change.uri.endsWith('/mops.toml') + change.uri.endsWith('/mops.toml') || + change.uri.endsWith('/mops.lock') ) { notifyPackageConfigChange(); } @@ -807,18 +875,19 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { function processQueue() { clearTimeout(checkTimeout); clearTimeout(checkWorkspaceTimeout); - checkTimeout = setTimeout(() => { + checkTimeout = setTimeout(async () => { const uri = checkQueue.shift(); if (checkQueue.length) { processQueue(); } if (uri) { + await ensureContextLoaded(uri); checkImmediate(uri); } }, 0); } function scheduleCheck(uri: string | TextDocument) { - if (disableChecks || loadingPackages) { + if (disableChecks) { return false; } if (checkQueue.length === 0) { @@ -1099,10 +1168,12 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { allContexts().forEach(({ motoko }) => motoko.delete(path)); } - connection.onCodeAction((event) => { + connection.onCodeAction(async (event) => { const uri = event.textDocument.uri; const results: CodeAction[] = []; + await ensureContextLoaded(uri); + // Organize imports // TODO: Consider removing unused imports const status = getContext(uri).astResolver.request( @@ -1173,7 +1244,10 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { return results; }); - connection.onSignatureHelp(mkOnSignatureHelpHandler(documents, notify)); + connection.onSignatureHelp(async (params, token) => { + await ensureContextLoaded(params.textDocument.uri); + return mkOnSignatureHelpHandler(documents, notify)(params, token); + }); function findImportUri( context: Context, @@ -1193,7 +1267,7 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { return; } - connection.onCompletion((event) => { + connection.onCompletion(async (event) => { const { position } = event; const { uri } = event.textDocument; @@ -1205,6 +1279,7 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { try { const doc = documents.get(uri); if (!doc) return list; + await ensureContextLoaded(uri); // Flush latest document content to virtual FS before parsing, // since onDidChangeContent debounces notify() by 500ms. // This prevents getting outdated AST from the cache. @@ -1558,6 +1633,7 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { connection.onHover(async (event) => { const { position } = event; const { uri } = event.textDocument; + await ensureContextLoaded(uri); const { astResolver } = getContext(uri); const document = documents.get(uri); @@ -1613,21 +1689,24 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { }; }); - connection.onDefinition((event: TextDocumentPositionParams): Location[] => { - console.log('[Definition]'); - try { - const definitions = findDefinitions( - event.textDocument.uri, - event.position, - ); - return definitions.map(locationFromDefinition); - } catch (err) { - console.error('Error while finding definition:'); - console.error(err); - // throw err; - return []; - } - }); + connection.onDefinition( + async (event: TextDocumentPositionParams): Promise => { + console.log('[Definition]'); + try { + await ensureContextLoaded(event.textDocument.uri); + const definitions = findDefinitions( + event.textDocument.uri, + event.position, + ); + return definitions.map(locationFromDefinition); + } catch (err) { + console.error('Error while finding definition:'); + console.error(err); + // throw err; + return []; + } + }, + ); // connection.onDeclaration( // async ( @@ -1670,9 +1749,10 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { return results; }); - connection.onDocumentSymbol((event) => { + connection.onDocumentSymbol(async (event) => { const { uri } = event.textDocument; const results: DocumentSymbol[] = []; + await ensureContextLoaded(uri); const status = getContext(uri).astResolver.request( uri, isVirtualFileSystemReady, @@ -1721,13 +1801,20 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { ]; } - connection.onReferences(mkOnReferencesHandler(isVirtualFileSystemReady)); + connection.onReferences(async (event, token) => { + await ensureContextLoaded(event.textDocument.uri); + return mkOnReferencesHandler(isVirtualFileSystemReady)(event, token); + }); - connection.onPrepareRename( - mkOnPrepareRenameHandler(isVirtualFileSystemReady), - ); + connection.onPrepareRename(async (event, token) => { + await ensureContextLoaded(event.textDocument.uri); + return mkOnPrepareRenameHandler(isVirtualFileSystemReady)(event, token); + }); - connection.onRenameRequest(mkOnRenameHandler(isVirtualFileSystemReady)); + connection.onRenameRequest(async (event, token) => { + await ensureContextLoaded(event.textDocument.uri); + return mkOnRenameHandler(isVirtualFileSystemReady)(event, token); + }); // Run a file which is recognized as a unit test connection.onRequest( @@ -1892,8 +1979,9 @@ export const addHandlers = (connection: Connection, redirectConsole = true) => { }, 500); }); - documents.onDidOpen((event) => { + documents.onDidOpen(async (event) => { clearCommentStringCache(event.document.uri); + await ensureContextLoaded(event.document.uri); scheduleCheck(event.document.uri); }); documents.onDidClose(async (event) => { diff --git a/src/server/test/requestMocJs.spec.ts b/src/server/test/requestMocJs.spec.ts index 1eecd1f9..27f6afbb 100644 --- a/src/server/test/requestMocJs.spec.ts +++ b/src/server/test/requestMocJs.spec.ts @@ -33,6 +33,13 @@ describe('request moc.js', () => { beforeAll(async () => { [client, server] = await defaultBeforeAll(rootUri, true); + // Trigger lazy context loading by opening a file and waiting + // for the server to finish processing (including moc.js download) + const filePath = join(rootPath, 'Main.mo'); + const fileUri = URI.parse(filePath).toString(); + const diagsPromise = waitForDiagnostics(client, fileUri); + await openTextDocuments(client, new Map(), rootUri, [fileUri]); + await diagsPromise; }); afterAll(async () => { @@ -125,6 +132,12 @@ describe('request moc.js', () => { beforeAll(async () => { settings.mocJsPath = mocPath; [client, server] = await defaultBeforeAll(rootUri, true); + // Trigger lazy context loading + const filePath = join(rootPath, 'Main.mo'); + const fileUri = URI.parse(filePath).toString(); + const diagsPromise = waitForDiagnostics(client, fileUri); + await openTextDocuments(client, new Map(), rootUri, [fileUri]); + await diagsPromise; }); afterAll(async () => { diff --git a/src/server/utils.ts b/src/server/utils.ts index 3bcb5aca..e4bb88c1 100644 --- a/src/server/utils.ts +++ b/src/server/utils.ts @@ -1,5 +1,13 @@ -import { readFileSync, readdirSync, createWriteStream, existsSync } from 'fs'; +import { + readFileSync, + readdirSync, + createWriteStream, + existsSync, + writeFileSync, + mkdirSync, +} from 'fs'; import { unlink } from 'fs/promises'; +import { createHash } from 'crypto'; import { join, sep, basename } from 'path'; import * as motokoPlugin from 'prettier-plugin-motoko'; import * as prettier from 'prettier/standalone'; @@ -439,3 +447,62 @@ function removeOldMocVersions(path: string): ResultAsync { } return okAsync([]); } + +interface SourcesCache { + hash: string; + sources: [string, string][]; +} + +const SOURCES_CACHE_FILENAME = '.sources-cache.json'; + +function computeSourcesCacheHash(dir: string): string | undefined { + const mopsToml = join(dir, 'mops.toml'); + if (!existsSync(mopsToml)) return undefined; + + const hash = createHash('sha256'); + hash.update(readFileSync(mopsToml, 'utf8')); + const mopsLock = join(dir, 'mops.lock'); + if (existsSync(mopsLock)) { + hash.update(readFileSync(mopsLock, 'utf8')); + } + return hash.digest('hex'); +} + +export function readSourcesCache(dir: string): [string, string][] | undefined { + const hash = computeSourcesCacheHash(dir); + if (!hash) return undefined; + + const cachePath = join(dir, '.mops', SOURCES_CACHE_FILENAME); + try { + if (!existsSync(cachePath)) return undefined; + const cache: SourcesCache = JSON.parse(readFileSync(cachePath, 'utf8')); + if (cache.hash === hash && Array.isArray(cache.sources)) { + return cache.sources; + } + } catch { + // Corrupted cache, ignore + } + return undefined; +} + +export function writeSourcesCache( + dir: string, + sources: [string, string][], +): void { + const hash = computeSourcesCacheHash(dir); + if (!hash) return; + + const cacheDir = join(dir, '.mops'); + try { + if (!existsSync(cacheDir)) { + mkdirSync(cacheDir, { recursive: true }); + } + const cache: SourcesCache = { hash, sources }; + writeFileSync( + join(cacheDir, SOURCES_CACHE_FILENAME), + JSON.stringify(cache), + ); + } catch (err) { + console.warn('Failed to write sources cache:', err); + } +}