diff --git a/src/nimblepkg/config.nim b/src/nimblepkg/config.nim index b13873d4e..e2d499b3b 100644 --- a/src/nimblepkg/config.nim +++ b/src/nimblepkg/config.nim @@ -22,23 +22,21 @@ proc initConfig(): Config = result.httpProxy = initUri() result.chcp = true result.cloneUsingHttps = true - result.packageLists["official"] = PackageList(name: "Official", urls: @[ - "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json", - "https://nim-lang.org/nimble/packages.json" - ]) proc clear(pkgList: var PackageList) = pkgList.name = "" pkgList.urls = @[] pkgList.path = "" -proc addCurrentPkgList(config: var Config, currentPackageList: PackageList) = +proc addCurrentPkgList(config: var Config, currentPackageList: PackageList, hasUserPackageList: var bool) = if currentPackageList.name.len > 0: config.packageLists[currentPackageList.name.normalize] = currentPackageList + hasUserPackageList = true proc parseConfig*(): Config = result = initConfig() var confFile = getConfigDir() / "nimble" / "nimble.ini" + var hasUserPackageList = false var f = newFileStream(confFile, fmRead) if f != nil: @@ -56,10 +54,10 @@ proc parseConfig*(): Config = raise nimbleError("Package list '$1' requires either url or path" % currentPackageList.name) if currentPackageList.urls.len > 0 and currentPackageList.path != "": raise nimbleError("Attempted to specify `url` and `path` for the same package list '$1'" % currentPackageList.name) - addCurrentPkgList(result, currentPackageList) + addCurrentPkgList(result, currentPackageList, hasUserPackageList) break of cfgSectionStart: - addCurrentPkgList(result, currentPackageList) + addCurrentPkgList(result, currentPackageList, hasUserPackageList) currentSection = e.section case currentSection.normalize of "packagelist": @@ -104,3 +102,8 @@ proc parseConfig*(): Config = of cfgError: raise nimbleError("Unable to parse config file: " & e.msg) close(p) + if not hasUserPackageList: + result.packageLists["official"] = PackageList(name: "Official", urls: @[ + "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json", + "https://nim-lang.org/nimble/packages.json" + ]) diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index 08887eb68..405db06cf 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -65,6 +65,9 @@ proc doCheckout*(meth: DownloadMethod, downloadDir, branch: string, options: Opt of DownloadMethod.hg: let (_, exitCode) = doCmdEx(&"hg --cwd {downloadDir.quoteShell} checkout {branch.quoteShell}") return exitCode == 0 + of DownloadMethod.http: + # HTTP packages are extracted directly, no checkout needed + return true proc doCheckoutAsync*(meth: DownloadMethod, downloadDir, branch: string, options: Options): Future[bool] {.async.} = ## Async version of doCheckout that uses doCmdExAsync for non-blocking execution. @@ -86,6 +89,9 @@ proc doCheckoutAsync*(meth: DownloadMethod, downloadDir, branch: string, options of DownloadMethod.hg: let (_, exitCode) = await doCmdExAsync(&"hg --cwd {downloadDir.quoteShell} checkout {branch.quoteShell}") return exitCode == 0 + of DownloadMethod.http: + # HTTP packages are extracted directly, no checkout needed + return true proc doClone(meth: DownloadMethod, url, downloadDir: string, branch = "", onlyTip = true, options: Options) = @@ -105,6 +111,9 @@ proc doClone(meth: DownloadMethod, url, downloadDir: string, branch = "", tipArg = if onlyTip: "-r tip " else: "" branchArg = if branch == "": "" else: &"-b {branch.quoteShell}" discard tryDoCmdEx(&"hg clone {tipArg} {branchArg} {url} {downloadDir.quoteShell}") + of DownloadMethod.http: + # HTTP packages are downloaded as tarballs, not cloned + discard proc doCloneAsync*(meth: DownloadMethod, url, downloadDir: string, branch = "", onlyTip = true, options: Options): Future[void] {.async.} = @@ -125,6 +134,9 @@ proc doCloneAsync*(meth: DownloadMethod, url, downloadDir: string, branch = "", tipArg = if onlyTip: "-r tip " else: "" branchArg = if branch == "": "" else: &"-b {branch.quoteShell}" discard await tryDoCmdExAsync(&"hg clone {tipArg} {branchArg} {url} {downloadDir.quoteShell}") + of DownloadMethod.http: + # HTTP packages are downloaded as tarballs, not cloned + discard proc gitFetchTags*(repoDir: string, downloadMethod: DownloadMethod, options: Options) = case downloadMethod: @@ -134,6 +146,9 @@ proc gitFetchTags*(repoDir: string, downloadMethod: DownloadMethod, options: Opt of DownloadMethod.hg: # In Mercurial, pulling updates also fetches all remote tags tryDoCmdEx(&"hg --cwd {repoDir} pull") + of DownloadMethod.http: + # HTTP packages have no remote tags to fetch + discard proc gitFetchTagsAsync*(repoDir: string, downloadMethod: DownloadMethod, options: Options): Future[void] {.async.} = ## Async version of gitFetchTags that uses doCmdExAsync for non-blocking execution. @@ -144,6 +159,9 @@ proc gitFetchTagsAsync*(repoDir: string, downloadMethod: DownloadMethod, options of DownloadMethod.hg: # In Mercurial, pulling updates also fetches all remote tags discard await tryDoCmdExAsync(&"hg --cwd {repoDir} pull") + of DownloadMethod.http: + # HTTP packages have no remote tags to fetch + discard proc getTagsList*(dir: string, meth: DownloadMethod): seq[string] = var output: string @@ -153,6 +171,9 @@ proc getTagsList*(dir: string, meth: DownloadMethod): seq[string] = output = tryDoCmdEx("git tag") of DownloadMethod.hg: output = tryDoCmdEx("hg tags") + of DownloadMethod.http: + # HTTP packages have no tags + return @[] if output.len > 0: case meth of DownloadMethod.git: @@ -168,6 +189,9 @@ proc getTagsList*(dir: string, meth: DownloadMethod): seq[string] = discard parseUntil(i, tag, ' ') if tag != "tip": result.add(tag) + of DownloadMethod.http: + # HTTP packages have no tags + result = @[] else: result = @[] @@ -180,6 +204,9 @@ proc getTagsListAsync*(dir: string, meth: DownloadMethod): Future[seq[string]] { output = await tryDoCmdExAsync(&"git -C {dir.quoteShell} tag") of DownloadMethod.hg: output = await tryDoCmdExAsync(&"hg --cwd {dir.quoteShell} tags") + of DownloadMethod.http: + # HTTP packages have no tags + return @[] if output.len > 0: case meth of DownloadMethod.git: @@ -195,6 +222,9 @@ proc getTagsListAsync*(dir: string, meth: DownloadMethod): Future[seq[string]] { discard parseUntil(i, tag, ' ') if tag != "tip": result.add(tag) + of DownloadMethod.http: + # HTTP packages have no tags + result = @[] else: result = @[] @@ -217,6 +247,9 @@ proc getTagsListRemote*(url: string, meth: DownloadMethod): seq[string] = of DownloadMethod.hg: # http://stackoverflow.com/questions/2039150/show-tags-for-remote-hg-repository raise nimbleError("Hg doesn't support remote tag querying.") + of DownloadMethod.http: + # HTTP packages have no remote tags to query + return @[] proc getTagsListRemoteAsync*(url: string, meth: DownloadMethod): Future[seq[string]] {.async.} = ## Async version of getTagsListRemote that uses doCmdExAsync for non-blocking execution. @@ -238,6 +271,9 @@ proc getTagsListRemoteAsync*(url: string, meth: DownloadMethod): Future[seq[stri of DownloadMethod.hg: # http://stackoverflow.com/questions/2039150/show-tags-for-remote-hg-repository raise nimbleError("Hg doesn't support remote tag querying.") + of DownloadMethod.http: + # HTTP packages have no remote tags to query + return @[] proc getVersionList*(tags: seq[string]): OrderedTable[Version, string] = ## Return an ordered table of Version -> git tag label. Ordering is @@ -261,6 +297,7 @@ proc getHeadName*(meth: DownloadMethod): Version = case meth of DownloadMethod.git: newVersion("#head") of DownloadMethod.hg: newVersion("#tip") + of DownloadMethod.http: newVersion("#head") proc checkUrlType*(url: string, options: Options): DownloadMethod = ## Determines the download method based on the URL. @@ -303,6 +340,8 @@ proc cloneSpecificRevision(downloadMethod: DownloadMethod, downloadDir.updateSubmodules of DownloadMethod.hg: discard tryDoCmdEx(&"hg clone {url} -r {($vcsRevision).quoteShell}") + of DownloadMethod.http: + raise nimbleError("HTTP method does not support cloning specific revisions.") proc cloneSpecificRevisionAsync*(downloadMethod: DownloadMethod, url, downloadDir: string, @@ -325,6 +364,8 @@ proc cloneSpecificRevisionAsync*(downloadMethod: DownloadMethod, await downloadDir.updateSubmodulesAsync() of DownloadMethod.hg: discard await tryDoCmdExAsync(&"hg clone {url} -r {($vcsRevision).quoteShell}") + of DownloadMethod.http: + raise nimbleError("HTTP method does not support cloning specific revisions.") var tarExePathCache {.threadvar.}: string @@ -709,8 +750,55 @@ proc doDownload(url, downloadDir: string, verRange: VersionRange, else: display("Warning:", &"The package {url} has no tagged releases, downloading HEAD instead.", Warning, priority = HighPriority) + of DownloadMethod.http: + let downloadUrl = + case verRange.kind + of verSpecial: + url & "/" & substr($verRange.spe, 1) + of verEq: + url & "/" & $verRange.ver + else: + url & "/head" + display("Downloading", downloadUrl) + let data = retrieveUrl(downloadUrl) + display("Completed", "downloading " & downloadUrl) + + let filePath = downloadDir / "package.tar.gz" + display("Saving", filePath) + downloadDir.createDir + writeFile(filePath, data) + display("Completed", "saving " & filePath) + + display("Unpacking", filePath) + let cmd = getTarCmdLine(downloadDir, filePath) + let (output, exitCode) = doCmdEx(cmd) + if exitCode != QuitSuccess and not output.contains("Cannot create symlink to"): + raise nimbleError(tryDoCmdExErrorMessage(cmd, output, exitCode)) + display("Completed", "unpacking " & filePath) + + when defined(windows): + let listCmd = &"{getTarExePath()} -ztvf {filePath} --force-local" + let (cmdOutput, cmdExitCode) = doCmdEx(listCmd) + if cmdExitCode != QuitSuccess: + raise nimbleError(tryDoCmdExErrorMessage(listCmd, cmdOutput, cmdExitCode)) + for line in cmdOutput.splitLines(): + if line.contains(" -> "): + let parts = line.split + let linkPath = parts[^1] + let linkNameParts = parts[^3].split('/') + let linkName = linkNameParts[1 .. ^1].foldl(a / b) + writeFile(downloadDir / linkName, linkPath) + + filePath.removeFile + + let nimbleFile = findNimbleFile(downloadDir, true, options) + let info = extractRequiresInfo(nimbleFile, options) + if info.version != "": + result.version = newVersion(info.version) + else: + raise nimbleError("Could not determine version from downloaded package at " & downloadUrl) - if result.vcsRevision == notSetSha1Hash: + if result.vcsRevision == notSetSha1Hash and downMethod != DownloadMethod.http: # In the case the package in not downloaded as tarball we must query its # VCS revision from its download directory. result.vcsRevision = downloadDir.getVcsRevision @@ -811,8 +899,55 @@ proc doDownloadAsync(url, downloadDir: string, verRange: VersionRange, else: display("Warning:", &"The package {url} has no tagged releases, downloading HEAD instead.", Warning, priority = HighPriority) + of DownloadMethod.http: + let downloadUrl = + case verRange.kind + of verSpecial: + url & "/" & substr($verRange.spe, 1) + of verEq: + url & "/" & $verRange.ver + else: + url & "/head" + display("Downloading", downloadUrl) + let data = retrieveUrl(downloadUrl) + display("Completed", "downloading " & downloadUrl) + + let filePath = downloadDir / "package.tar.gz" + display("Saving", filePath) + downloadDir.createDir + writeFile(filePath, data) + display("Completed", "saving " & filePath) + + display("Unpacking", filePath) + let cmd = getTarCmdLine(downloadDir, filePath) + let (output, exitCode) = await doCmdExAsync(cmd) + if exitCode != QuitSuccess and not output.contains("Cannot create symlink to"): + raise nimbleError(tryDoCmdExErrorMessage(cmd, output, exitCode)) + display("Completed", "unpacking " & filePath) + + when defined(windows): + let listCmd = &"{getTarExePath()} -ztvf {filePath} --force-local" + let (cmdOutput, cmdExitCode) = await doCmdExAsync(listCmd) + if cmdExitCode != QuitSuccess: + raise nimbleError(tryDoCmdExErrorMessage(listCmd, cmdOutput, cmdExitCode)) + for line in cmdOutput.splitLines(): + if line.contains(" -> "): + let parts = line.split + let linkPath = parts[^1] + let linkNameParts = parts[^3].split('/') + let linkName = linkNameParts[1 .. ^1].foldl(a / b) + writeFile(downloadDir / linkName, linkPath) + + filePath.removeFile + + let nimbleFile = findNimbleFile(downloadDir, true, options) + let info = extractRequiresInfo(nimbleFile, options) + if info.version != "": + result.version = newVersion(info.version) + else: + raise nimbleError("Could not determine version from downloaded package at " & downloadUrl) - if result.vcsRevision == notSetSha1Hash: + if result.vcsRevision == notSetSha1Hash and downMethod != DownloadMethod.http: # In the case the package in not downloaded as tarball we must query its # VCS revision from its download directory. result.vcsRevision = downloadDir.getVcsRevision @@ -1018,6 +1153,8 @@ proc echoPackageVersions*(pkg: Package) = of DownloadMethod.hg: displayInfoLine(" versions: ", "(Remote tag retrieval not supported by " & $pkg.downloadMethod & ")") + of DownloadMethod.http: + displayInfoLine(" versions: ", "(Version info from registry only)") proc removeTrailingSlash(s: string): string = s.strip(chars = {'/'}, leading = false) diff --git a/src/nimblepkg/packageinfo.nim b/src/nimblepkg/packageinfo.nim index 0827d99c3..16933cf1e 100644 --- a/src/nimblepkg/packageinfo.nim +++ b/src/nimblepkg/packageinfo.nim @@ -79,6 +79,7 @@ proc parseDownloadMethod*(meth: string): DownloadMethod = case meth of "git": return DownloadMethod.git of "hg", "mercurial": return DownloadMethod.hg + of "http": return DownloadMethod.http else: raise nimbleError("Invalid download method: " & meth) diff --git a/src/nimblepkg/packageinfotypes.nim b/src/nimblepkg/packageinfotypes.nim index 742c3a1b6..ec334dced 100644 --- a/src/nimblepkg/packageinfotypes.nim +++ b/src/nimblepkg/packageinfotypes.nim @@ -6,7 +6,7 @@ import version, sha1hashes type DownloadMethod* {.pure.} = enum - git = "git", hg = "hg" + git = "git", hg = "hg", http = "http" Checksums* = object sha1*: Sha1Hash diff --git a/src/nimblepkg/versiondiscovery.nim b/src/nimblepkg/versiondiscovery.nim index 8f4e7f723..396cba000 100644 --- a/src/nimblepkg/versiondiscovery.nim +++ b/src/nimblepkg/versiondiscovery.nim @@ -181,6 +181,15 @@ proc getPackageMinimalVersionsFromRepo*(repoDir: string, pkg: PkgTuple, version: if taggedVersions.isSome: return taggedVersions.get + # HTTP packages have no VCS history; just return the single downloaded version + if downloadMethod == DownloadMethod.http: + try: + result.addUnique getPkgInfo(repoDir, options, nimBin, pikRequires).getMinimalInfo(options) + except CatchableError as e: + displayWarning(&"Error getting package info for {name}: {e.msg}", HighPriority) + saveTaggedVersions(name, result, options) + return result + # During version discovery, we only need to read .nimble files, not compile code # So we can safely ignore submodules to avoid issues with repos that have # submodules that fail to clone (e.g., waku's zerokit submodule) @@ -281,6 +290,15 @@ proc getPackageMinimalVersionsFromRepoAsync*(repoDir: string, pkg: PkgTuple, ver except CatchableError: discard # Continue with fetching from repo + # HTTP packages have no VCS history; just return the single downloaded version + if downloadMethod == DownloadMethod.http: + try: + result.addUnique getPkgInfo(repoDir, options, nimBin, pikRequires).getMinimalInfo(options) + except CatchableError as e: + displayWarning(&"Error getting package info for {name}: {e.msg}", HighPriority) + saveTaggedVersions(name, result, options) + return result + let tempDir = repoDir & "_versions" # During version discovery, we only need to read .nimble files, not compile code # So we can safely ignore submodules to avoid issues with repos that have @@ -369,6 +387,15 @@ proc getPackageMinimalVersionsFromRepoAsyncFast*( result = newSeq[PackageMinimalInfo]() let name = pkg[0] + # HTTP packages have no VCS history; just return the single downloaded version + if downloadMethod == DownloadMethod.http: + try: + result.addUnique getPkgInfo(repoDir, options, nimBin, pikRequires).getMinimalInfo(options) + except CatchableError as e: + displayWarning(&"Error getting package info for {name}: {e.msg}", HighPriority) + saveTaggedVersions(name, result, options) + return result + # Find the git repository root (repoDir might be a subdirectory) var gitRoot = repoDir var subdirPath = ""