Skip to content

Support reading HDF5 files from TAR file. #5132

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pnorbert wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Support reading HDF5 files from TAR file. #5132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion source/adios2/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,7 @@ if(ADIOS2_HAVE_HDF5)
core/IOHDF5.cpp
engine/hdf5/HDF5ReaderP.cpp
engine/hdf5/HDF5WriterP.cpp
toolkit/interop/hdf5/HDF5Common.cpp toolkit/interop/hdf5/HDF5Common.tcc
toolkit/interop/hdf5/HDF5Common.cpp toolkit/interop/hdf5/HDF5Common.tcc toolkit/interop/hdf5/HDF5TarOffsetVFD.cpp
)
target_link_libraries(adios2_core PRIVATE adios2_hdf5)
set_property(TARGET adios2_hdf5 PROPERTY EXPORT_NAME hdf5)
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/core/IO.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -547,7 +547,7 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm, co
else if (tarinfo.find(".h5") != std::string::npos ||
tarinfo.find(".hdf5") != std::string::npos ||
tarinfo.find(".nc") != std::string::npos)
engineType = "h5";
engineType = "hdf5";
else
helper::Throw<std::runtime_error>(
"Core", "IO", "Open",
Expand Down
196 changes: 20 additions & 176 deletions source/adios2/engine/bp5/BP5Reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,41 +506,6 @@ double BP5Reader::ReadData(PoolableFile *DataFile, const size_t WriterRank, cons

void BP5Reader::PerformGets()
{
#if defined ADIOS2_HAVE_CURL || defined ADIOS2_HAVE_XROOTD
auto lf_getXRootDHostPort = [&](int defaultPort) -> std::tuple<std::string, int> {
std::string XRootDHost = "localhost";
int XRootDPort = defaultPort;
if (m_HostConfig)
{
XRootDHost = m_HostConfig->hostname;
if (m_HostConfig->port > 0)
{
XRootDPort = m_HostConfig->port;
}
}
else if (m_RemoteHost != "localhost")
{
auto colon_pos = m_RemoteHost.find(':');
if (colon_pos == std::string::npos)
{
XRootDHost = m_RemoteHost;
}
else
{
XRootDHost = m_RemoteHost.substr(0, colon_pos);
try
{
XRootDPort = std::stoi(m_RemoteHost.substr(colon_pos + 1));
}
catch (...)
{
}
}
}
return std::make_tuple(XRootDHost, XRootDPort);
};
#endif

// if dataIsRemote is true and m_Remote is not true, this is our first time through
// PerformGets() Either we don't need a remote open (m_dataIsRemote=false), or we need to Open
// remote file (or die trying)
Expand All @@ -552,74 +517,19 @@ void BP5Reader::PerformGets()
if (m_BP5Deserializer->DefaultGetContext().PendingGetRequests.size() == 0)
return;

#if defined(ADIOS2_HAVE_CURL) || defined(ADIOS2_HAVE_XROOTD)
if (m_RemoteProtocol == HostAccessProtocol::XRootD &&
(m_XrootdTransferProtocol == XRootDTransferProtocol::HTTP ||
m_XrootdTransferProtocol == XRootDTransferProtocol::HTTPS ||
m_XrootdTransferProtocol == XRootDTransferProtocol::XrdCl))
{
// XrdCl reaches the origin/federation over HTTPS; the libcurl path
// additionally supports plain HTTP.
const bool useXrdCl = (m_XrootdTransferProtocol == XRootDTransferProtocol::XrdCl);
const bool useHttps =
useXrdCl || (m_XrootdTransferProtocol == XRootDTransferProtocol::HTTPS);
auto tup = lf_getXRootDHostPort(useHttps ? 443 : 80);
m_Remote = std::make_unique<XrootdHttpRemote>(ADIOS::GetHostOptions());
Params params;
params["UseHttps"] = useHttps ? "true" : "false";
if (useXrdCl)
params["Backend"] = "XrdCl";
// For testing, disable SSL verification (only relevant for HTTPS)
if (useHttps) // && getenv("XRootDHttpsNoVerify"))
{
params["VerifySSL"] = "false";
}
if (!m_Parameters.TarInfo.empty())
params["TarInfo"] = m_Parameters.TarInfo;
if (!m_Parameters.SelectSteps.empty())
params["SelectSteps"] = m_Parameters.SelectSteps;
if (m_Parameters.IgnoreFlattenSteps)
params["IgnoreFlattenSteps"] = "true";
// Send our file id so the server can detect stale cached metadata (0 = none).
if (m_FileUUID != 0)
params["FileUUID"] = std::to_string(m_FileUUID);
m_Remote->Open(std::get<0>(tup), std::get<1>(tup), m_RemoteName, m_OpenMode,
RowMajorOrdering, params);
}
else
#endif
#ifdef ADIOS2_HAVE_XROOTD
if (m_RemoteProtocol == HostAccessProtocol::XRootD &&
m_XrootdTransferProtocol == XRootDTransferProtocol::XRootD)
{
auto tup = lf_getXRootDHostPort(1094);
m_Remote = std::make_unique<XrootdRemote>(ADIOS::GetHostOptions());
m_Remote->Open(std::get<0>(tup), std::get<1>(tup), m_RemoteName, m_OpenMode,
RowMajorOrdering);
}
else
#endif
#ifdef ADIOS2_HAVE_SST
if (m_RemoteProtocol == HostAccessProtocol::SSH)
{
auto pair = CManagerSingleton::MakeEVPathConnection(m_RemoteHost);
m_Remote = pair.first;
int localPort = pair.second;
if (m_Remote && localPort > -1)
{
Params p;
if (!m_Parameters.TarInfo.empty())
p.emplace("TarInfo", m_Parameters.TarInfo);
if (!m_Parameters.SelectSteps.empty())
p.emplace("SelectSteps", m_Parameters.SelectSteps);
if (m_Parameters.IgnoreFlattenSteps)
p.emplace("IgnoreFlattenSteps", "true");

m_Remote->Open("localhost", localPort, m_RemoteName, m_OpenMode, RowMajorOrdering,
p);
}
}
#endif
Params params;
if (!m_Parameters.TarInfo.empty())
params["TarInfo"] = m_Parameters.TarInfo;
if (!m_Parameters.SelectSteps.empty())
params["SelectSteps"] = m_Parameters.SelectSteps;
if (m_Parameters.IgnoreFlattenSteps)
params["IgnoreFlattenSteps"] = "true";
// Send our file id so the server can detect stale cached metadata (0 = none).
if (m_FileUUID != 0)
params["FileUUID"] = std::to_string(m_FileUUID);

m_Remote = GetRemote(m_RemoteSetup, m_RemoteName, m_OpenMode, RowMajorOrdering, params);

#ifdef ADIOS2_HAVE_KVCACHE
if (getenv("useKVCache"))
{
Expand Down Expand Up @@ -1236,93 +1146,27 @@ void BP5Reader::Init()
{
m_RemoteName = m_Name;
}

m_RemoteProtocol = HostAccessProtocol::Invalid;
if (!m_Parameters.RemoteHost.empty())
{
m_RemoteHost = m_Parameters.RemoteHost;
auto it = ADIOS::GetHostOptions().find(m_Parameters.RemoteHost);
if (it != ADIOS::GetHostOptions().end())
{
for (auto &hc : it->second)
{
if (hc.protocol == HostAccessProtocol::SSH)
{
m_RemoteProtocol = hc.protocol;
m_HostConfig = const_cast<HostConfig *>(&hc);
break;
}
if (hc.protocol == HostAccessProtocol::XRootD)
{
m_RemoteProtocol = hc.protocol;
m_HostConfig = const_cast<HostConfig *>(&hc);
m_XrootdTransferProtocol = hc.transfer_protocol;
break;
}
}
}
}
else
{
if (getenv("DoXRootDXrdCl"))
{
// XrdCl client against the same HTTPS server (reuses XRootDHttpsHost).
char *env = getenv("XRootDHttpsHost");
if (env)
m_RemoteHost = std::string(env);
m_RemoteProtocol = HostAccessProtocol::XRootD;
m_XrootdTransferProtocol = XRootDTransferProtocol::XrdCl;
}
else if (getenv("DoXRootDHttps"))
{
char *env = getenv("XRootDHttpsHost");
if (env)
m_RemoteHost = std::string(env);
m_RemoteProtocol = HostAccessProtocol::XRootD;
m_XrootdTransferProtocol = XRootDTransferProtocol::HTTPS;
}
else if (getenv("DoXRootDHttp"))
{
char *env = getenv("XRootDHttpHost");
if (env)
m_RemoteHost = getenv("XRootDHttpHost");
m_RemoteProtocol = HostAccessProtocol::XRootD;
m_XrootdTransferProtocol = XRootDTransferProtocol::HTTP;
}
else if (getenv("DoXRootD"))
{
char *env = getenv("XRootDHost");
if (env)
m_RemoteHost = getenv("XRootDHost");
m_RemoteProtocol = HostAccessProtocol::XRootD;
m_XrootdTransferProtocol = XRootDTransferProtocol::XRootD;
}
if (m_RemoteHost.empty())
{
m_RemoteHost = "localhost";
}
}

if (m_RemoteHost.empty())
m_RemoteSetup = GetRemoteSetup(m_Parameters.RemoteHost);
if (m_RemoteSetup.hostName.empty())
{
helper::Throw<std::invalid_argument>(
"Engine", "BP5Reader", "OpenFiles",
"No remote hostname was found for dataset " + m_RemoteName +
". Make sure you define proper access to the server to serve this path.");
}
if (m_RemoteProtocol == HostAccessProtocol::Invalid)
if (m_RemoteSetup.protocol == HostAccessProtocol::Invalid)
{
if (m_RemoteHost == "localhost")
if (m_RemoteSetup.hostName == "localhost")
{
// special case for debugging on localhost
m_RemoteProtocol = HostAccessProtocol::SSH;
m_RemoteSetup.protocol = HostAccessProtocol::SSH;
}
else
{
helper::Throw<std::invalid_argument>(
"Engine", "BP5Reader", "OpenFiles",
"No acceptable protocol (xrootd or ssh) was found for " + m_RemoteHost +
" to read " + m_RemoteName +
"No acceptable protocol (xrootd or ssh) was found for " +
m_RemoteSetup.hostName + " to read " + m_RemoteName +
". Make sure you define proper access to the server to serve this path.");
}
}
Expand Down
7 changes: 2 additions & 5 deletions source/adios2/engine/bp5/BP5Reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,8 @@ class BP5Reader : public BP5Engine, public Engine
/* Remote dataset information */
bool m_dataIsRemote = false;
std::shared_ptr<Remote> m_Remote;
std::string m_RemoteHost; // host name of remote (as index in config map)
std::string m_RemoteName; // path on remote location
adios2::HostConfig *m_HostConfig = nullptr; // when found the HostConfig to be used
HostAccessProtocol m_RemoteProtocol = HostAccessProtocol::Invalid; // ssh or xrootd
XRootDTransferProtocol m_XrootdTransferProtocol = XRootDTransferProtocol::XRootD;
RemoteSetup m_RemoteSetup;
std::string m_RemoteName; // path on remote location

bool m_WriterIsActive = true;
adios2::profiling::JSONProfiler m_JSONProfiler;
Expand Down
8 changes: 6 additions & 2 deletions source/adios2/engine/campaign/CampaignData.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -480,7 +480,8 @@ static int sqlcb_tarinfo(void *p, int argc, char **argv, char **azColName)
return 0;
};

std::string CampaignData::GetTarIdx(const size_t dsIdx, const size_t repIdx)
std::string CampaignData::GetTarIdx(const size_t dsIdx, const size_t repIdx,
const FileFormat format)
{
std::stringstream ss;
CampaignReplica &rep = datasets[dsIdx].replicas[repIdx];
Expand All @@ -503,7 +504,10 @@ std::string CampaignData::GetTarIdx(const size_t dsIdx, const size_t repIdx)
sqlite3_free(zErrMsg);
}

return ss.str();
if (format == FileFormat::HDF5)
return ".h5" + ss.str();
else
return ss.str();
}

void CampaignData::Close()
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/engine/campaign/CampaignData.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ class CampaignData
*/
std::vector<size_t> FindRemoteReplicas(const size_t datasetIdx, const HostOptions &hostOptions);

std::string GetTarIdx(const size_t dsIdx, const size_t repIdx);
std::string GetTarIdx(const size_t dsIdx, const size_t repIdx, const FileFormat format);

private:
void DumpToFileOrMemory(const size_t fileIdx, std::string &keyHex, const std::string &path,
Expand Down
15 changes: 9 additions & 6 deletions source/adios2/engine/campaign/CampaignReader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@ std::string CampaignReader::SaveRemoteMD(size_t dsIdx, size_t repIdx, adios2::co
{
std::string tarpath = itTarName->second;
remotePath = m_CampaignData.directory[rep.dirIdx].path + "/" + tarpath;
taropt = m_CampaignData.GetTarIdx(dsIdx, repIdx);
taropt = m_CampaignData.GetTarIdx(dsIdx, repIdx, ds.format);
if (taropt.empty())
{
std::cout << "ERROR: Remote file " << remotePath
Expand Down Expand Up @@ -420,7 +420,8 @@ std::string CampaignReader::SaveRemoteMD(size_t dsIdx, size_t repIdx, adios2::co
// Retrieve key
if (!m_ConnectionManager)
{
m_ConnectionManager = std::make_unique<Remote>(core::ADIOS::StaticGetHostOptions());
const RemoteSetup rs; // defaults OK, just want connection to local service
m_ConnectionManager = std::make_unique<Remote>(rs);
}
m_CampaignData.keys[rep.keyIdx].keyHex =
m_ConnectionManager->GetKeyFromConnectionManager(
Expand Down Expand Up @@ -965,7 +966,7 @@ void CampaignReader::InitTransports()
{
std::string tarpath = itTarName->second;
localPath = m_CampaignData.directory[rep.dirIdx].path + PathSeparator + tarpath;
taropt = m_CampaignData.GetTarIdx(dsIdx, repIdx);
taropt = m_CampaignData.GetTarIdx(dsIdx, repIdx, ds.format);
if (taropt.empty())
{
std::cout << "ERROR: Local file " << localPath
Expand Down Expand Up @@ -1093,7 +1094,7 @@ void CampaignReader::InitTransports()
std::string tarpath = itTarName->second;
localPath =
m_CampaignData.directory[rep.dirIdx].path + PathSeparator + tarpath;
taropt = m_CampaignData.GetTarIdx(dsIdx, repIdx);
taropt = m_CampaignData.GetTarIdx(dsIdx, repIdx, FileFormat::IMAGE);
if (taropt.empty())
{
std::cout << "ERROR: Local image " << localPath
Expand Down Expand Up @@ -1529,14 +1530,16 @@ void CampaignReader::ReadRemoteFile(const std::string &remoteHost, const std::st
#ifdef ADIOS2_HAVE_XROOTD
if (getenv("DoXRootD"))
{
remote = std::make_unique<XrootdRemote>(ADIOS::GetHostOptions());
const RemoteSetup rs = GetRemoteSetup("localhost");
remote = std::make_unique<XrootdRemote>(rs);
remote->Open("localhost", 1094, m_Name, m_OpenMode, true);
}
else
#endif
#ifdef ADIOS2_HAVE_SST
{
remote = std::make_unique<EVPathRemote>(ADIOS::GetHostOptions());
const RemoteSetup rs = GetRemoteSetup(remoteHost);
remote = std::make_unique<EVPathRemote>(rs);
int localPort = remote->LaunchRemoteServerViaConnectionManager(remoteHost);
remote->OpenSimpleFile("localhost", localPort, remotePath);
}
Expand Down
Loading
Loading