From 9298b75ad61f7a47719519b5630851c524ad7b76 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 30 Jan 2026 12:47:03 -0800 Subject: [PATCH 01/80] Remove the OPENVDB_USE_DELAYED_LOADING flag and all the code guarded by it Signed-off-by: Dan Bailey --- .github/workflows/weekly.yml | 4 +- CMakeLists.txt | 2 - cmake/FindOpenVDB.cmake | 18 - openvdb/openvdb/CMakeLists.txt | 39 - openvdb/openvdb/io/Archive.cc | 164 +--- openvdb/openvdb/io/File.cc | 106 +-- openvdb/openvdb/io/File.h | 29 +- openvdb/openvdb/io/Stream.cc | 52 -- openvdb/openvdb/io/TempFile.cc | 144 --- openvdb/openvdb/io/TempFile.h | 54 -- openvdb/openvdb/io/io.h | 60 -- openvdb/openvdb/points/AttributeArray.h | 27 +- openvdb/openvdb/points/StreamCompression.cc | 83 +- openvdb/openvdb/points/StreamCompression.h | 3 - openvdb/openvdb/tree/LeafBuffer.h | 186 +--- openvdb/openvdb/tree/LeafNode.h | 41 +- .../openvdb/unittest/TestAttributeArray.cc | 821 ------------------ openvdb/openvdb/unittest/TestCount.cc | 44 - openvdb/openvdb/unittest/TestFile.cc | 4 - .../openvdb/unittest/TestPointConversion.cc | 1 - openvdb/openvdb/unittest/TestPointCount.cc | 59 -- openvdb/openvdb/unittest/TestPointDataLeaf.cc | 105 --- .../unittest/TestPointRasterizeFrustum.cc | 113 +-- .../openvdb/unittest/TestStreamCompression.cc | 121 +-- openvdb/openvdb/unittest/util.h | 12 - openvdb/openvdb/version.h.in | 5 - pyproject.toml | 1 - 27 files changed, 51 insertions(+), 2247 deletions(-) delete mode 100644 openvdb/openvdb/io/TempFile.cc delete mode 100644 openvdb/openvdb/io/TempFile.h diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index 353303e54c..614cb37de9 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -146,7 +146,7 @@ jobs: matrix: config: - { name: 'all', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=ON -DUSE_ZLIB=ON -DUSE_EXR=ON -DUSE_PNG=ON' } - - { name: 'lite', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF -DOPENVDB_USE_DELAYED_LOADING=OFF' } + - { name: 'lite', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF' } - { name: 'half', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_IMATH_HALF=ON' } - { name: 'sse', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_SIMD=SSE42' } - { name: 'avx', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_SIMD=AVX' } @@ -216,7 +216,6 @@ jobs: --components=\"core,axcore,python,bin,render,test,axbin\" --cargs=\' -DCMAKE_CXX_STANDARD=20 - -DOPENVDB_USE_DELAYED_LOADING=OFF -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install ${{ matrix.config.cmake }} \' @@ -421,7 +420,6 @@ jobs: --components="${{ matrix.config.components }}" --cargs=\' -A x64 -G \"Visual Studio 17 2022\" - -DOPENVDB_USE_DELAYED_LOADING=OFF -DUSE_BLOSC=OFF \ -DUSE_ZLIB=OFF \ -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET} diff --git a/CMakeLists.txt b/CMakeLists.txt index 089381aa41..c66ad21fa7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,7 +80,6 @@ option(OPENVDB_BUILD_NANOVDB "Build the NanoVDB library. Turns ON if USE_NANOVDB # Global options option(OPENVDB_ENABLE_RPATH "Build with RPATH information" ON) option(OPENVDB_ENABLE_ASSERTS "Build with asserts in OpenVDB code enabled" OFF) -option(OPENVDB_USE_DELAYED_LOADING "Build the core OpenVDB library with delayed-loading." ON) option(OPENVDB_CXX_STRICT "Enable or disable pre-defined compiler warnings" OFF) cmake_dependent_option(OPENVDB_INSTALL_CMAKE_MODULES "Install the provided OpenVDB CMake modules when building the core library" @@ -249,7 +248,6 @@ mark_as_advanced( OPENVDB_BUILD_HOUDINI_ABITESTS OPENVDB_CXX_STRICT OPENVDB_ENABLE_RPATH - OPENVDB_USE_DELAYED_LOADING OPENVDB_FUTURE_DEPRECATION OPENVDB_SIMD SYSTEM_LIBRARY_PATHS diff --git a/cmake/FindOpenVDB.cmake b/cmake/FindOpenVDB.cmake index 400afd2544..17a4c2da53 100644 --- a/cmake/FindOpenVDB.cmake +++ b/cmake/FindOpenVDB.cmake @@ -64,8 +64,6 @@ This will define the following variables: True if the OpenVDB Library has been built with log4cplus support ``OpenVDB_USES_IMATH_HALF`` True if the OpenVDB Library has been built with Imath half support -``OpenVDB_USES_DELAYED_LOADING`` - True if the OpenVDB Library has been built with delayed-loading ``OpenVDB_ABI`` Set if this module was able to determine the ABI number the located OpenVDB Library was built against. Unset otherwise. @@ -522,7 +520,6 @@ set(OpenVDB_USES_BLOSC ${USE_BLOSC}) set(OpenVDB_USES_ZLIB ${USE_ZLIB}) set(OpenVDB_USES_LOG4CPLUS ${USE_LOG4CPLUS}) set(OpenVDB_USES_IMATH_HALF ${USE_IMATH_HALF}) -set(OpenVDB_USES_DELAYED_LOADING ${OPENVDB_USE_DELAYED_LOADING}) set(OpenVDB_DEFINITIONS) if(WIN32) @@ -555,7 +552,6 @@ if(_OPENVDB_HAS_NEW_VERSION_HEADER) OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_IMATH_HALF" OpenVDB_USES_IMATH_HALF) OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_BLOSC" OpenVDB_USES_BLOSC) OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_ZLIB" OpenVDB_USES_ZLIB) - OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_DELAYED_LOADING" OpenVDB_USES_DELAYED_LOADING) elseif(NOT OPENVDB_USE_STATIC_LIBS) # Use GetPrerequisites to see which libraries this OpenVDB lib has linked to # which we can query for optional deps. This basically runs ldd/otoll/objdump @@ -602,11 +598,6 @@ elseif(NOT OPENVDB_USE_STATIC_LIBS) if(NOT ${_HAS_DEP} EQUAL -1) set(OpenVDB_USES_IMATH_HALF ON) endif() - - string(FIND ${PREREQUISITE} "boost_iostreams" _HAS_DEP) - if(NOT ${_HAS_DEP} EQUAL -1) - set(OpenVDB_USES_DELAYED_LOADING ON) - endif() endforeach() unset(_OPENVDB_PREREQUISITE_LIST) @@ -628,10 +619,6 @@ if(OpenVDB_USES_IMATH_HALF) find_package(Imath REQUIRED CONFIG) endif() -if(OpenVDB_USES_DELAYED_LOADING) - find_package(Boost REQUIRED COMPONENTS iostreams) -endif() - if(UNIX) find_package(Threads REQUIRED) endif() @@ -643,11 +630,6 @@ endif() set(_OPENVDB_VISIBLE_DEPENDENCIES "") -if(OpenVDB_USES_DELAYED_LOADING) - list(APPEND _OPENVDB_VISIBLE_DEPENDENCIES Boost::iostreams) - list(APPEND OpenVDB_DEFINITIONS OPENVDB_USE_DELAYED_LOADING) -endif() - if(OpenVDB_USES_IMATH_HALF) list(APPEND _OPENVDB_VISIBLE_DEPENDENCIES Imath::Imath) endif() diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 8348d0d527..c496509a72 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -116,19 +116,6 @@ endif() # Collect and configure lib dependencies -if(OPENVDB_USE_DELAYED_LOADING) - find_package(Boost ${MINIMUM_BOOST_VERSION} REQUIRED COMPONENTS iostreams) - - if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_BOOST_VERSION) - # The X.Y.Z boost version value isn't available until CMake 3.14 - set(FULL_BOOST_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") - if(${FULL_BOOST_VERSION} VERSION_LESS FUTURE_MINIMUM_BOOST_VERSION) - message(DEPRECATION "Support for Boost versions < ${FUTURE_MINIMUM_BOOST_VERSION} " - "is deprecated and will be removed.") - endif() - endif() -endif() - find_package(TBB ${MINIMUM_TBB_VERSION} REQUIRED COMPONENTS tbb) if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_TBB_VERSION) if(${TBB_VERSION} VERSION_LESS FUTURE_MINIMUM_TBB_VERSION) @@ -251,27 +238,6 @@ if(UNIX) list(APPEND OPENVDB_CORE_DEPENDENT_LIBS Threads::Threads) endif() -# Pull in Boost last as houdini's boost (hboost) is fully namespaced and libs -# are renamed too. Boost can be pulled in at any time, but do it last so that, -# if it's in a shared place (like /usr/local) it doesn't accidently pull in -# other headers. - -if(OPENVDB_USE_DELAYED_LOADING) - list(APPEND OPENVDB_CORE_DEPENDENT_LIBS Boost::iostreams) - if(WIN32) - # Boost headers contain #pragma commands on Windows which causes Boost - # libraries to be linked in automatically. Custom boost installations - # may find that these naming conventions don't always match and can - # cause linker errors. This option disables this feature of Boost. Note - # -DBOOST_ALL_NO_LIB can also be provided manually. - if(OPENVDB_DISABLE_BOOST_IMPLICIT_LINKING) - list(APPEND OPENVDB_CORE_DEPENDENT_LIBS - Boost::disable_autolinking # add -DBOOST_ALL_NO_LIB - ) - endif() - endif() -endif() - ########################################################################## ##### Core library configuration @@ -369,7 +335,6 @@ set(OPENVDB_LIBRARY_SOURCE_FILES io/GridDescriptor.cc io/Queue.cc io/Stream.cc - io/TempFile.cc math/Maps.cc math/Proximity.cc math/QuantizedUnitVec.cc @@ -409,7 +374,6 @@ set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/io.h io/Queue.h io/Stream.h - io/TempFile.h ) set(OPENVDB_LIBRARY_MATH_INCLUDE_FILES @@ -670,9 +634,6 @@ endif() if(USE_LOG4CPLUS) list(APPEND OPENVDB_CORE_PUBLIC_DEFINES -DOPENVDB_USE_LOG4CPLUS) endif() -if(OPENVDB_USE_DELAYED_LOADING) - list(APPEND OPENVDB_CORE_PUBLIC_DEFINES -DOPENVDB_USE_DELAYED_LOADING) -endif() ########################################################################## diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 4431d69299..03576b2566 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -13,35 +13,6 @@ #include #include -#ifdef OPENVDB_USE_DELAYED_LOADING -// Boost.Interprocess uses a header-only portion of Boost.DateTime -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-macros" -#endif -#define BOOST_DATE_TIME_NO_LIB -#ifdef __clang__ -#pragma clang diagnostic pop -#endif -#include -#include -#include -#include - -#ifdef _WIN32 -#include // open_existing_file(), close_file() -extern "C" __declspec(dllimport) bool __stdcall GetFileTime( - void* fh, void* ctime, void* atime, void* mtime); -// boost::interprocess::detail was renamed to boost::interprocess::ipcdetail in Boost 1.48. -// Ensure that both namespaces exist. -namespace boost { namespace interprocess { namespace detail {} namespace ipcdetail {} } } -#else -#include // for struct stat -#include // for stat() -#include // for unlink() -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include #include // for std::find_if() @@ -443,117 +414,6 @@ operator<<(std::ostream& os, const StreamMetadata::AuxDataMap& auxData) //////////////////////////////////////// -#ifdef OPENVDB_USE_DELAYED_LOADING - - -// Memory-mapping a VDB file permits threaded input (and output, potentially, -// though that might not be practical for compressed files or files containing -// multiple grids). In particular, a memory-mapped file can be loaded lazily, -// meaning that the voxel buffers of the leaf nodes of a grid's tree are not allocated -// until they are actually accessed. When access to its buffer is requested, -// a leaf node allocates memory for the buffer and then streams in (and decompresses) -// its contents from the memory map, starting from a stream offset that was recorded -// at the time the node was constructed. The memory map must persist as long as -// there are unloaded leaf nodes; this is ensured by storing a shared pointer -// to the map in each unloaded node. - -class MappedFile::Impl -{ -public: - Impl(const std::string& filename, bool autoDelete) - : mMap(filename.c_str(), boost::interprocess::read_only) - , mRegion(mMap, boost::interprocess::read_only) - , mAutoDelete(autoDelete) - { - if (mAutoDelete) { -#ifndef _WIN32 - // On Unix systems, unlink the file so that it gets deleted once it is closed. - ::unlink(mMap.get_name()); -#endif - } - } - - ~Impl() - { - std::string filename; - if (const char* s = mMap.get_name()) filename = s; - OPENVDB_LOG_DEBUG_RUNTIME("closing memory-mapped file " << filename); - if (mNotifier) mNotifier(filename); - if (mAutoDelete) { - if (!boost::interprocess::file_mapping::remove(filename.c_str())) { - if (errno != ENOENT) { - // Warn if the file exists but couldn't be removed. - std::string mesg = getErrorString(); - if (!mesg.empty()) mesg = " (" + mesg + ")"; - OPENVDB_LOG_WARN("failed to remove temporary file " << filename << mesg); - } - } - } - } - - boost::interprocess::file_mapping mMap; - boost::interprocess::mapped_region mRegion; - bool mAutoDelete; - Notifier mNotifier; -#if OPENVDB_ABI_VERSION_NUMBER <= 12 - mutable std::atomic mLastWriteTime; -#endif - -private: - Impl(const Impl&); // not copyable - Impl& operator=(const Impl&); // not copyable -}; - - -MappedFile::MappedFile(const std::string& filename, bool autoDelete): - mImpl(new Impl(filename, autoDelete)) -{ -} - - -MappedFile::~MappedFile() -{ -} - - -std::string -MappedFile::filename() const -{ - std::string result; - if (const char* s = mImpl->mMap.get_name()) result = s; - return result; -} - - -SharedPtr -MappedFile::createBuffer() const -{ - return SharedPtr{ - new boost::iostreams::stream_buffer{ - static_cast(mImpl->mRegion.get_address()), mImpl->mRegion.get_size()}}; -} - - -void -MappedFile::setNotifier(const Notifier& notifier) -{ - mImpl->mNotifier = notifier; -} - - -void -MappedFile::clearNotifier() -{ - mImpl->mNotifier = nullptr; -} - - -#endif // OPENVDB_USE_DELAYED_LOADING - - -//////////////////////////////////////// - - std::string getErrorString(int errorNum) { @@ -886,25 +746,6 @@ setGridBackgroundValuePtr(std::ios_base& strm, const void* background) } -#ifdef OPENVDB_USE_DELAYED_LOADING -MappedFile::Ptr -getMappedFilePtr(std::ios_base& strm) -{ - if (const void* ptr = strm.pword(GetSteamState().mappedFile)) { - return *static_cast(ptr); - } - return MappedFile::Ptr(); -} - - -void -setMappedFilePtr(std::ios_base& strm, io::MappedFile::Ptr& mappedFile) -{ - strm.pword(GetSteamState().mappedFile) = &mappedFile; -} -#endif // OPENVDB_USE_DELAYED_LOADING - - StreamMetadata::Ptr getStreamMetadataPtr(std::ios_base& strm) { @@ -1120,14 +961,11 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co bool Archive::isDelayedLoadingEnabled() { -#ifdef OPENVDB_USE_DELAYED_LOADING - return (nullptr == std::getenv("OPENVDB_DISABLE_DELAYED_LOAD")); -#else return false; -#endif } + namespace { struct NoBBox {}; diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index aee16c8879..dfb0c797e1 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -5,20 +5,11 @@ #include "File.h" -#include "TempFile.h" #include #include #include #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#include -#ifndef _WIN32 -#include -#include -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include // stat() #include // for getenv(), strtoul() @@ -99,14 +90,6 @@ struct File::Impl Archive::NamedGridMap mNamedGrids; // All grids stored in the file (used only when mHasGridOffsets is false) GridPtrVecPtr mGrids; -#ifdef OPENVDB_USE_DELAYED_LOADING - // The memory-mapped file - MappedFile::Ptr mFileMapping; - // The buffer for the input stream, if it is a memory-mapped file - SharedPtr mStreamBuf; - // File size limit for copying during delayed loading - Index64 mCopyMaxBytes; -#endif }; // class File::Impl @@ -117,9 +100,6 @@ File::File(const std::string& filename): mImpl(new Impl) { mImpl->mFilename = filename; mImpl->mIsOpen = false; -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mCopyMaxBytes = Impl::getDefaultCopyMaxBytes(); -#endif setInputHasGridOffsets(true); } @@ -146,9 +126,6 @@ File::operator=(const File& other) mImpl->mFilename = otherImpl.mFilename; mImpl->mMeta = otherImpl.mMeta; mImpl->mIsOpen = false; // don't want two file objects reading from the same stream -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mCopyMaxBytes = otherImpl.mCopyMaxBytes; -#endif mImpl->mGridDescriptors = otherImpl.mGridDescriptors; mImpl->mNamedGrids = otherImpl.mNamedGrids; mImpl->mGrids = otherImpl.mGrids; @@ -252,22 +229,6 @@ File::getSize() const } -#ifdef OPENVDB_USE_DELAYED_LOADING -Index64 -File::copyMaxBytes() const -{ - return mImpl->mCopyMaxBytes; -} - - -void -File::setCopyMaxBytes(Index64 bytes) -{ - mImpl->mCopyMaxBytes = bytes; -} -#endif - - //////////////////////////////////////// @@ -279,11 +240,7 @@ File::isOpen() const bool -#ifdef OPENVDB_USE_DELAYED_LOADING -File::open(bool delayLoad, const MappedFile::Notifier& notifier) -#else -File::open(bool /*delayLoad = true*/) -#endif // OPENVDB_USE_DELAYED_LOADING +File::open() { if (isOpen()) { OPENVDB_THROW(IoError, filename() << " is already open"); @@ -292,53 +249,8 @@ File::open(bool /*delayLoad = true*/) // Open the file. std::unique_ptr newStream; - SharedPtr newStreamBuf; -#ifdef OPENVDB_USE_DELAYED_LOADING - MappedFile::Ptr newFileMapping; - if (!delayLoad || !Archive::isDelayedLoadingEnabled()) { -#endif - newStream.reset(new std::ifstream( - filename().c_str(), std::ios_base::in | std::ios_base::binary)); -#ifdef OPENVDB_USE_DELAYED_LOADING - } else { - bool isTempFile = false; - std::string fname = filename(); - if (getSize() < copyMaxBytes()) { - // If the file is not too large, make a temporary private copy of it - // and open the copy instead. The original file can then be modified - // or removed without affecting delayed load. - try { - TempFile tempFile; - std::ifstream fstrm(filename().c_str(), - std::ios_base::in | std::ios_base::binary); - boost::iostreams::copy(fstrm, tempFile); - fname = tempFile.filename(); - isTempFile = true; - } catch (std::exception& e) { - std::string mesg; - if (e.what()) mesg = std::string(" (") + e.what() + ")"; - OPENVDB_LOG_WARN("failed to create a temporary copy of " << filename() - << " for delayed loading" << mesg - << "; will read directly from " << filename() << " instead"); - } - } - - // While the file is open, its mapping, stream buffer and stream - // must all be maintained. Once the file is closed, the buffer and - // the stream can be discarded, but the mapping needs to persist - // if any grids were lazily loaded. - try { - newFileMapping.reset(new MappedFile(fname, /*autoDelete=*/isTempFile)); - newStreamBuf = newFileMapping->createBuffer(); - newStream.reset(new std::istream(newStreamBuf.get())); - } catch (std::exception& e) { - std::ostringstream ostr; - ostr << "could not open file " << filename(); - if (e.what() != nullptr) ostr << " (" << e.what() << ")"; - OPENVDB_THROW(IoError, ostr.str()); - } - } -#endif // OPENVDB_USE_DELAYED_LOADING + newStream.reset(new std::ifstream( + filename().c_str(), std::ios_base::in | std::ios_base::binary)); if (newStream->fail()) { OPENVDB_THROW(IoError, "could not open file " << filename()); @@ -356,11 +268,6 @@ File::open(bool /*delayLoad = true*/) throw; } -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mFileMapping = newFileMapping; - if (mImpl->mFileMapping) mImpl->mFileMapping->setNotifier(notifier); - mImpl->mStreamBuf = newStreamBuf; -#endif mImpl->mInStream.swap(newStream); // Tag the input stream with the file format and library version numbers @@ -371,9 +278,6 @@ File::open(bool /*delayLoad = true*/) Archive::setFormatVersion(inputStream()); Archive::setLibraryVersion(inputStream()); Archive::setDataCompression(inputStream()); -#ifdef OPENVDB_USE_DELAYED_LOADING - io::setMappedFilePtr(inputStream(), mImpl->mFileMapping); -#endif // Read in the VDB metadata. mImpl->mMeta = MetaMap::Ptr(new MetaMap); @@ -422,10 +326,6 @@ File::close() mImpl->mNamedGrids.clear(); mImpl->mInStream.reset(); mImpl->mStreamMetadata.reset(); -#ifdef OPENVDB_USE_DELAYED_LOADING - mImpl->mStreamBuf.reset(); - mImpl->mFileMapping.reset(); -#endif mImpl->mIsOpen = false; setInputHasGridOffsets(true); diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 41dd6f9694..dbde5b6d51 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -54,20 +54,11 @@ class OPENVDB_API File: public Archive /// @details The file does not necessarily exist on disk yet. const std::string& filename() const; -#ifdef OPENVDB_USE_DELAYED_LOADING /// @brief Open the file, read the file header and the file-level metadata, /// and populate the grid descriptors, but do not load any grids into memory. - /// @details If @a delayLoad is true, map the file into memory and enable delayed loading - /// of grids, and if a notifier is provided, call it when the file gets unmapped. - /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD to disable - /// delayed loading unconditionally. /// @throw IoError if the file is not a valid VDB file. /// @return @c true if the file's UUID has changed since it was last read. - /// @see setCopyMaxBytes - bool open(bool delayLoad = true, const MappedFile::Notifier& = MappedFile::Notifier()); -#else - bool open(bool /*delayLoad*/ = false); -#endif + bool open(); /// Return @c true if the file has been opened for reading. bool isOpen() const; @@ -79,24 +70,6 @@ class OPENVDB_API File: public Archive /// @throw IoError if the file size cannot be determined. Index64 getSize() const; -#ifdef OPENVDB_USE_DELAYED_LOADING - /// @brief Return the size in bytes above which this file will not be - /// automatically copied during delayed loading. - Index64 copyMaxBytes() const; - /// @brief If this file is opened with delayed loading enabled, make a private copy - /// of the file if its size in bytes is less than the specified value. - /// @details Making a private copy ensures that the file can't change on disk - /// before it has been fully read. - /// @warning If the file is larger than this size, it is the user's responsibility - /// to ensure that it does not change on disk before it has been fully read. - /// Undefined behavior and/or a crash might result otherwise. - /// @note Copying is enabled by default, but it can be disabled for individual files - /// by setting the maximum size to zero bytes. A default size limit can be specified - /// by setting the environment variable @c OPENVDB_DELAYED_LOAD_COPY_MAX_BYTES - /// to the desired number of bytes. - void setCopyMaxBytes(Index64 bytes); -#endif - /// Return @c true if a grid of the given name exists in this file. bool hasGrid(const Name&) const; diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 662a2f4cd9..4ba3fce816 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -5,14 +5,9 @@ #include "File.h" ///< @todo refactor #include "GridDescriptor.h" -#include "TempFile.h" #include #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#include -#endif - #include // for remove() #include // for std::bind() #include @@ -49,59 +44,12 @@ struct Stream::Impl //////////////////////////////////////// -#ifdef OPENVDB_USE_DELAYED_LOADING - -namespace { - -/// @todo Use MappedFile auto-deletion instead. -void -removeTempFile(const std::string expectedFilename, const std::string& filename) -{ - if (filename == expectedFilename) { - if (0 != std::remove(filename.c_str())) { - std::string mesg = getErrorString(); - if (!mesg.empty()) mesg = " (" + mesg + ")"; - OPENVDB_LOG_WARN("failed to remove temporary file " << filename << mesg); - } - } -} - -} - -#endif // OPENVDB_USE_DELAYED_LOADING - - Stream::Stream(std::istream& is, bool delayLoad): mImpl(new Impl) { if (!is) return; (void) delayLoad; -#ifdef OPENVDB_USE_DELAYED_LOADING - if (delayLoad && Archive::isDelayedLoadingEnabled()) { - // Copy the contents of the stream to a temporary private file - // and open the file instead. - std::unique_ptr tempFile; - try { - tempFile.reset(new TempFile); - } catch (std::exception& e) { - std::string mesg; - if (e.what()) mesg = std::string(" (") + e.what() + ")"; - OPENVDB_LOG_WARN("failed to create a temporary file for delayed loading" << mesg - << "; will read directly from the input stream instead"); - } - if (tempFile) { - boost::iostreams::copy(is, *tempFile); - const std::string& filename = tempFile->filename(); - mImpl->mFile.reset(new File(filename)); - mImpl->mFile->setCopyMaxBytes(0); // don't make a copy of the temporary file - /// @todo Need to pass auto-deletion flag to MappedFile. - mImpl->mFile->open(delayLoad, - std::bind(&removeTempFile, filename, std::placeholders::_1)); - } - } -#endif // OPENVDB_USE_DELAYED_LOADING - if (!mImpl->mFile) { readHeader(is); diff --git a/openvdb/openvdb/io/TempFile.cc b/openvdb/openvdb/io/TempFile.cc deleted file mode 100644 index 290f2e6b9c..0000000000 --- a/openvdb/openvdb/io/TempFile.cc +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -/// @file TempFile.cc - -#ifdef OPENVDB_USE_DELAYED_LOADING - -#include "TempFile.h" - -#include -#ifndef _WIN32 -#include -#include -#include // for std::getenv(), mkstemp() -#include // for mode_t -#include // for mkdir(), umask() -#include // for access() -#else -#include // for std::filebuf -#endif -#include // for std::tmpnam(), L_tmpnam, P_tmpdir -#include -#include -#include -#include - - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -struct TempFile::TempFileImpl -{ - const std::string& filename() const { return mPath; } - - bool is_open() const { return mBuffer.is_open(); } - - /// @internal boost::filesystem::unique_path(), etc. might be useful here, - /// but as of 9/2014, Houdini ships without the Boost.Filesystem library, - /// which makes it much less convenient to use that library. -#ifndef _WIN32 - TempFileImpl(std::ostream& os): mFileDescr(-1) { this->init(os); } - - void init(std::ostream& os) - { - std::string fn = this->getTempDir() + "/openvdb_temp_XXXXXX"; - std::vector fnbuf(fn.begin(), fn.end()); - fnbuf.push_back(char(0)); - - //const mode_t savedMode = ::umask(~(S_IRUSR | S_IWUSR)); - mFileDescr = ::mkstemp(&fnbuf[0]); - //::umask(savedMode); - if (mFileDescr < 0) { - OPENVDB_THROW(IoError, "failed to generate temporary file"); - } - - mPath.assign(&fnbuf[0]); - - mDevice = DeviceType(mFileDescr, boost::iostreams::never_close_handle); - mBuffer.open(mDevice); - os.rdbuf(&mBuffer); - - if (!os.good()) { - OPENVDB_THROW(IoError, "failed to open temporary file " + mPath); - } - } - - void close() { mBuffer.close(); if (mFileDescr >= 0) ::close(mFileDescr); } - - static std::string getTempDir() - { - if (const char* dir = std::getenv("OPENVDB_TEMP_DIR")) { - if (0 != ::access(dir, F_OK)) { -#ifdef _WIN32 - ::mkdir(dir); -#else - ::mkdir(dir, S_IRUSR | S_IWUSR | S_IXUSR); -#endif - if (0 != ::access(dir, F_OK)) { - OPENVDB_THROW(IoError, - "failed to create OPENVDB_TEMP_DIR (" + std::string(dir) + ")"); - } - } - return dir; - } - if (const char* dir = std::getenv("TMPDIR")) return dir; - return P_tmpdir; - } - - using DeviceType = boost::iostreams::file_descriptor_sink; - using BufferType = boost::iostreams::stream_buffer; - - std::string mPath; - DeviceType mDevice; - BufferType mBuffer; - int mFileDescr; -#else // _WIN32 - // Use only standard library routines; no POSIX. - - TempFileImpl(std::ostream& os) { this->init(os); } - - void init(std::ostream& os) - { - char fnbuf[L_tmpnam]; - const char* filename = std::tmpnam(fnbuf); - if (!filename) { - OPENVDB_THROW(IoError, "failed to generate name for temporary file"); - } - /// @todo This is not safe, since another process could open a file - /// with this name before we do. Unfortunately, there is no safe, - /// portable way to create a temporary file. - mPath = filename; - - const std::ios_base::openmode mode = (std::ios_base::out | std::ios_base::binary); - os.rdbuf(mBuffer.open(mPath.c_str(), mode)); - if (!os.good()) { - OPENVDB_THROW(IoError, "failed to open temporary file " + mPath); - } - } - - void close() { mBuffer.close(); } - - std::string mPath; - std::filebuf mBuffer; -#endif // _WIN32 - -private: - TempFileImpl(const TempFileImpl&); // disable copying - TempFileImpl& operator=(const TempFileImpl&); // disable assignment -}; - - -TempFile::TempFile(): std::ostream(nullptr), mImpl(new TempFileImpl(*this)) {} -TempFile::~TempFile() { this->close(); } -const std::string& TempFile::filename() const { return mImpl->filename(); } -bool TempFile::is_open() const { return mImpl->is_open(); } -void TempFile::close() { mImpl->close(); } - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb - -#endif // OPENVDB_USE_DELAYED_LOADING diff --git a/openvdb/openvdb/io/TempFile.h b/openvdb/openvdb/io/TempFile.h deleted file mode 100644 index cbbdec2250..0000000000 --- a/openvdb/openvdb/io/TempFile.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -/// @file TempFile.h - -#ifdef OPENVDB_USE_DELAYED_LOADING - -#ifndef OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED -#define OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED - -#include -#include -#include - - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -/// Output stream to a unique temporary file -class OPENVDB_API TempFile: public std::ostream -{ -public: - /// @brief Create and open a unique file. - /// @details On UNIX systems, the file is created in the directory specified by - /// the environment variable @c OPENVDB_TEMP_DIR, if that variable is defined, - /// or else in the directory specified by @c TMPDIR, if that variable is defined. - /// Otherwise (and on non-UNIX systems), the file is created in the system default - /// temporary directory. - TempFile(); - ~TempFile(); - - /// Return the path to the temporary file. - const std::string& filename() const; - - /// Return @c true if the file is open for writing. - bool is_open() const; - - /// Close the file. - void close(); - -private: - struct TempFileImpl; - std::unique_ptr mImpl; -}; - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb - -#endif // OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED - -#endif // OPENVDB_USE_DELAYED_LOADING diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index ff3adb6d26..40bce111d5 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -127,56 +127,6 @@ struct MultiPass {}; //////////////////////////////////////// -class File; - -#ifdef OPENVDB_USE_DELAYED_LOADING - -/// @brief Handle to control the lifetime of a memory-mapped .vdb file -class OPENVDB_API MappedFile -{ -public: - using Ptr = SharedPtr; - - ~MappedFile(); - MappedFile(const MappedFile&) = delete; // not copyable - MappedFile& operator=(const MappedFile&) = delete; - - /// Return the filename of the mapped file. - std::string filename() const; - - /// @brief Return a new stream buffer for the mapped file. - /// @details Typical usage is - /// @code - /// openvdb::io::MappedFile::Ptr mappedFile = ...; - /// auto buf = mappedFile->createBuffer(); - /// std::istream istrm{buf.get()}; - /// // Read from istrm... - /// @endcode - /// The buffer must persist as long as the stream is open. - SharedPtr createBuffer() const; - - using Notifier = std::function; - /// @brief Register a function that will be called with this file's name - /// when the file is unmapped. - void setNotifier(const Notifier&); - /// Deregister the notifier. - void clearNotifier(); - -private: - friend class File; - friend class ::TestMappedFile; - - explicit MappedFile(const std::string& filename, bool autoDelete = false); - - class Impl; - std::unique_ptr mImpl; -}; // class MappedFile - -#endif // OPENVDB_USE_DELAYED_LOADING - -//////////////////////////////////////// - - /// Return a string (possibly empty) describing the given system error code. std::string getErrorString(int errorNum); @@ -249,16 +199,6 @@ OPENVDB_API bool getWriteGridStatsMetadata(std::ios_base&); /// and store them as grid metadata when writing to the given stream. OPENVDB_API void setWriteGridStatsMetadata(std::ios_base&, bool writeGridStats); -#ifdef OPENVDB_USE_DELAYED_LOADING -/// @brief Return a shared pointer to the memory-mapped file with which the given stream -/// is associated, or a null pointer if the stream is not associated with a memory-mapped file. -OPENVDB_API SharedPtr getMappedFilePtr(std::ios_base&); -/// @brief Associate the given stream with (a shared pointer to) a memory-mapped file. -/// @note The shared pointer object (not just the io::MappedFile object to which it points) -/// must remain valid until the file is closed. -OPENVDB_API void setMappedFilePtr(std::ios_base&, SharedPtr&); -#endif // OPENVDB_USE_DELAYED_LOADING - /// @brief Return a shared pointer to an object that stores metadata (file format, /// compression scheme, etc.) for use when reading from or writing to the given stream. OPENVDB_API SharedPtr getStreamMetadataPtr(std::ios_base&); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index 9fba9c4c53..46d0b02318 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -1663,13 +1663,6 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu return; } -#ifdef OPENVDB_USE_DELAYED_LOADING - // If this array is being read from a memory-mapped file, delay loading of its data - // until the data is actually accessed. - io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is.getInputStream()); - const bool delayLoad = (mappedFile.get() != nullptr); -#endif - if (is.sizeOnly()) { size_t compressedBytes(mCompressedBytes); @@ -1686,22 +1679,12 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu this->deallocate(); -#ifdef OPENVDB_USE_DELAYED_LOADING - this->setOutOfCore(delayLoad); - is.read(mPageHandle, std::streamsize(mPageHandle->size()), delayLoad); -#else + this->setOutOfCore(false); is.read(mPageHandle, std::streamsize(mPageHandle->size()), false); -#endif // OPENVDB_USE_DELAYED_LOADING - -#ifdef OPENVDB_USE_DELAYED_LOADING - if (!delayLoad) { -#endif - std::unique_ptr buffer = mPageHandle->read(); - mData.reset(reinterpret_cast(buffer.release())); - mPageHandle.reset(); -#ifdef OPENVDB_USE_DELAYED_LOADING - } -#endif + + std::unique_ptr buffer = mPageHandle->read(); + mData.reset(reinterpret_cast(buffer.release())); + mPageHandle.reset(); // clear page state diff --git a/openvdb/openvdb/points/StreamCompression.cc b/openvdb/openvdb/points/StreamCompression.cc index 3f24b1b0e6..fde60d9f93 100644 --- a/openvdb/openvdb/points/StreamCompression.cc +++ b/openvdb/openvdb/points/StreamCompression.cc @@ -297,10 +297,6 @@ Page::uncompressedBytes() const const char* Page::buffer(const int index) const { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) this->load(); -#endif - return mData.get() + index; } @@ -336,51 +332,23 @@ Page::readBuffers(std::istream&is, bool delayed) bool isCompressed = mInfo->compressedBytes > 0; -#ifdef OPENVDB_USE_DELAYED_LOADING - io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is); - - if (delayed && mappedFile) { - SharedPtr meta = io::getStreamMetadataPtr(is); - OPENVDB_ASSERT(meta); - - std::streamoff filepos = is.tellg(); + std::unique_ptr buffer(new char[ + (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)]); + is.read(buffer.get(), (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)); - // seek over the page - is.seekg((isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes), - std::ios_base::cur); - - mInfo->mappedFile = mappedFile; - mInfo->meta = meta; - mInfo->filepos = filepos; - - OPENVDB_ASSERT(mInfo->mappedFile); + if (mInfo->compressedBytes > 0) { + this->decompress(buffer); + } else { + this->copy(buffer, -static_cast(mInfo->compressedBytes)); } - else { -#endif - std::unique_ptr buffer(new char[ - (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)]); - is.read(buffer.get(), (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)); - - if (mInfo->compressedBytes > 0) { - this->decompress(buffer); - } else { - this->copy(buffer, -static_cast(mInfo->compressedBytes)); - } - mInfo.reset(); -#ifdef OPENVDB_USE_DELAYED_LOADING - } -#endif + mInfo.reset(); } bool Page::isOutOfCore() const { -#ifdef OPENVDB_USE_DELAYED_LOADING - return bool(mInfo); -#else return false; -#endif } @@ -409,41 +377,6 @@ Page::decompress(const std::unique_ptr& temp) void Page::doLoad() const { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (!this->isOutOfCore()) return; - - Page* self = const_cast(this); - - // This lock will be contended at most once, after which this buffer - // will no longer be out-of-core. - tbb::spin_mutex::scoped_lock lock(self->mMutex); - if (!this->isOutOfCore()) return; - - OPENVDB_ASSERT(self->mInfo); - - int compressedBytes = static_cast(self->mInfo->compressedBytes); - bool compressed = compressedBytes > 0; - if (!compressed) compressedBytes = -compressedBytes; - - OPENVDB_ASSERT(compressedBytes); - - std::unique_ptr temp(new char[compressedBytes]); - - OPENVDB_ASSERT(self->mInfo->mappedFile); - SharedPtr buf = self->mInfo->mappedFile->createBuffer(); - OPENVDB_ASSERT(buf); - - std::istream is(buf.get()); - io::setStreamMetadataPtr(is, self->mInfo->meta, /*transfer=*/true); - is.seekg(self->mInfo->filepos); - - is.read(temp.get(), compressedBytes); - - if (compressed) self->decompress(temp); - else self->copy(temp, compressedBytes); - - self->mInfo.reset(); -#endif } diff --git a/openvdb/openvdb/points/StreamCompression.h b/openvdb/openvdb/points/StreamCompression.h index c7dcd4a8e9..d63e9c1116 100644 --- a/openvdb/openvdb/points/StreamCompression.h +++ b/openvdb/openvdb/points/StreamCompression.h @@ -114,9 +114,6 @@ class OPENVDB_API Page private: struct Info { -#ifdef OPENVDB_USE_DELAYED_LOADING - io::MappedFile::Ptr mappedFile; -#endif SharedPtr meta; std::streamoff filepos; long compressedBytes; diff --git a/openvdb/openvdb/tree/LeafBuffer.h b/openvdb/openvdb/tree/LeafBuffer.h index a684db8da6..671781a129 100644 --- a/openvdb/openvdb/tree/LeafBuffer.h +++ b/openvdb/openvdb/tree/LeafBuffer.h @@ -36,23 +36,9 @@ class LeafBuffer using NodeMaskType = util::NodeMask; static const Index SIZE = 1 << 3 * Log2Dim; -#ifdef OPENVDB_USE_DELAYED_LOADING - struct FileInfo - { - FileInfo(): bufpos(0) , maskpos(0) {} - std::streamoff bufpos; - std::streamoff maskpos; - io::MappedFile::Ptr mapping; - SharedPtr meta; - }; -#endif - /// Default constructor inline LeafBuffer(): mData(new ValueType[SIZE]) { -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = 0; -#endif } /// Construct a buffer populated with the specified value. explicit inline LeafBuffer(const ValueType&); @@ -61,9 +47,6 @@ class LeafBuffer /// Construct a buffer but don't allocate memory for the full array of values. LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) { -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = 0; -#endif } /// Destructor inline ~LeafBuffer(); @@ -71,23 +54,13 @@ class LeafBuffer /// Return @c true if this buffer's values have not yet been read from disk. bool isOutOfCore() const { -#ifdef OPENVDB_USE_DELAYED_LOADING - return bool(mOutOfCore); -#else return false; -#endif } /// Return @c true if memory for this buffer has not yet been allocated. bool empty() const { return !mData || this->isOutOfCore(); } /// Allocate memory for this buffer if it has not already been allocated. bool allocate() { if (mData == nullptr) mData = new ValueType[SIZE]; return true; } -#ifdef OPENVDB_USE_DELAYED_LOADING - /// Enable out-of-core in the LeafBuffer. - void enableOutOfCore(SharedPtr& meta, const std::streamoff& bufpos, - io::MappedFile::Ptr& mappedFile, const std::streamoff& maskpos); -#endif - /// Populate this buffer with a constant value. inline void fill(const ValueType&); @@ -142,31 +115,18 @@ class LeafBuffer inline void setOutOfCore(bool b) { (void) b; -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = b; -#endif } // To facilitate inlining in the common case in which the buffer is in-core, // the loading logic is split into a separate function, doLoad(). inline void loadValues() const { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) this->doLoad(); -#endif } inline void doLoad() const; inline bool detachFromFile(); using FlagsType = std::atomic; -#ifdef OPENVDB_USE_DELAYED_LOADING - union { - ValueType* mData; - FileInfo* mFileInfo; - }; -#else ValueType* mData; -#endif FlagsType mOutOfCore; // interpreted as bool; extra bits reserved for future use tbb::spin_mutex mMutex; // 1 byte //int8_t mReserved[3]; // padding for alignment @@ -185,9 +145,6 @@ inline LeafBuffer::LeafBuffer(const ValueType& val) : mData(new ValueType[SIZE]) { -#ifdef OPENVDB_USE_DELAYED_LOADING - mOutOfCore = 0; -#endif this->fill(val); } @@ -196,15 +153,7 @@ template inline LeafBuffer::~LeafBuffer() { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) { - this->detachFromFile(); - } else { - this->deallocate(); - } -#else this->deallocate(); -#endif } @@ -212,25 +161,14 @@ template inline LeafBuffer::LeafBuffer(const LeafBuffer& other) : mData(nullptr) -#ifdef OPENVDB_USE_DELAYED_LOADING - , mOutOfCore(other.mOutOfCore.load()) -#endif { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (other.isOutOfCore()) { - mFileInfo = new FileInfo(*other.mFileInfo); - } else { -#endif - if (other.mData != nullptr) { - this->allocate(); - ValueType* target = mData; - const ValueType* source = other.mData; - Index n = SIZE; - while (n--) *target++ = *source++; - } -#ifdef OPENVDB_USE_DELAYED_LOADING + if (other.mData != nullptr) { + this->allocate(); + ValueType* target = mData; + const ValueType* source = other.mData; + Index n = SIZE; + while (n--) *target++ = *source++; } -#endif } @@ -249,47 +187,17 @@ inline LeafBuffer& LeafBuffer::operator=(const LeafBuffer& other) { if (&other != this) { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) { - this->detachFromFile(); - } else { - if (other.isOutOfCore()) this->deallocate(); - } - if (other.isOutOfCore()) { - mOutOfCore.store(other.mOutOfCore.load(std::memory_order_acquire), - std::memory_order_release); - mFileInfo = new FileInfo(*other.mFileInfo); - } else { -#endif - if (other.mData != nullptr) { - this->allocate(); - ValueType* target = mData; - const ValueType* source = other.mData; - Index n = SIZE; - while (n--) *target++ = *source++; - } -#ifdef OPENVDB_USE_DELAYED_LOADING + if (other.mData != nullptr) { + this->allocate(); + ValueType* target = mData; + const ValueType* source = other.mData; + Index n = SIZE; + while (n--) *target++ = *source++; } -#endif } return *this; } -#ifdef OPENVDB_USE_DELAYED_LOADING -template -void LeafBuffer::enableOutOfCore( - SharedPtr& meta, const std::streamoff& bufpos, - io::MappedFile::Ptr& mappedFile, const std::streamoff& maskpos) -{ - this->setOutOfCore(true); - mFileInfo = new FileInfo; - mFileInfo->meta = meta; - mFileInfo->bufpos = bufpos; - mFileInfo->mapping = mappedFile; - mFileInfo->maskpos = maskpos; -} -#endif - template inline void LeafBuffer::fill(const ValueType& val) @@ -323,15 +231,6 @@ inline void LeafBuffer::swap(LeafBuffer& other) { std::swap(mData, other.mData); -#ifdef OPENVDB_USE_DELAYED_LOADING - // Two atomics can't be swapped because it would require hardware support: - // https://en.wikipedia.org/wiki/Double_compare-and-swap - // Note that there's a window in which other.mOutOfCore could be written - // between our load from it and our store to it. - auto tmp = other.mOutOfCore.load(std::memory_order_acquire); - tmp = mOutOfCore.exchange(std::move(tmp)); - other.mOutOfCore.store(std::move(tmp), std::memory_order_release); -#endif } @@ -340,14 +239,7 @@ inline Index LeafBuffer::memUsage() const { size_t n = sizeof(*this); -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) n += sizeof(FileInfo); - else { -#endif - if (mData) n += SIZE * sizeof(ValueType); -#ifdef OPENVDB_USE_DELAYED_LOADING - } -#endif + if (mData) n += SIZE * sizeof(ValueType); return static_cast(n); } @@ -369,10 +261,6 @@ LeafBuffer::data() const this->loadValues(); if (mData == nullptr) { LeafBuffer* self = const_cast(this); -#ifdef OPENVDB_USE_DELAYED_LOADING - // This lock will be contended at most once. - tbb::spin_mutex::scoped_lock lock(self->mMutex); -#endif if (mData == nullptr) self->mData = new ValueType[SIZE]; } return mData; @@ -384,10 +272,6 @@ LeafBuffer::data() { this->loadValues(); if (mData == nullptr) { -#ifdef OPENVDB_USE_DELAYED_LOADING - // This lock will be contended at most once. - tbb::spin_mutex::scoped_lock lock(mMutex); -#endif if (mData == nullptr) mData = new ValueType[SIZE]; } return mData; @@ -413,9 +297,6 @@ LeafBuffer::deallocate() { if (mData != nullptr) { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) return false; -#endif delete[] mData; mData = nullptr; return true; @@ -428,39 +309,6 @@ template inline void LeafBuffer::doLoad() const { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (!this->isOutOfCore()) return; - - LeafBuffer* self = const_cast*>(this); - - // This lock will be contended at most once, after which this buffer - // will no longer be out-of-core. - tbb::spin_mutex::scoped_lock lock(self->mMutex); - if (!this->isOutOfCore()) return; - - std::unique_ptr info(self->mFileInfo); - OPENVDB_ASSERT(info.get() != nullptr); - OPENVDB_ASSERT(info->mapping.get() != nullptr); - OPENVDB_ASSERT(info->meta.get() != nullptr); - - /// @todo For now, we have to clear the mData pointer in order for allocate() to take effect. - self->mData = nullptr; - self->allocate(); - - SharedPtr buf = info->mapping->createBuffer(); - std::istream is(buf.get()); - - io::setStreamMetadataPtr(is, info->meta, /*transfer=*/true); - - NodeMaskType mask; - is.seekg(info->maskpos); - mask.load(is); - - is.seekg(info->bufpos); - io::readCompressedValues(is, self->mData, SIZE, mask, io::getHalfFloat(is)); - - self->setOutOfCore(false); -#endif } @@ -468,14 +316,6 @@ template inline bool LeafBuffer::detachFromFile() { -#ifdef OPENVDB_USE_DELAYED_LOADING - if (this->isOutOfCore()) { - delete mFileInfo; - mFileInfo = nullptr; - this->setOutOfCore(false); - return true; - } -#endif return false; } diff --git a/openvdb/openvdb/tree/LeafNode.h b/openvdb/openvdb/tree/LeafNode.h index 9e7fbbf7c4..6894d17391 100644 --- a/openvdb/openvdb/tree/LeafNode.h +++ b/openvdb/openvdb/tree/LeafNode.h @@ -1375,10 +1375,6 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo SharedPtr meta = io::getStreamMetadataPtr(is); const bool seekable = meta && meta->seekable(); -#ifdef OPENVDB_USE_DELAYED_LOADING - std::streamoff maskpos = is.tellg(); -#endif - if (seekable) { // Seek over the value mask. mValueMask.seek(is); @@ -1396,35 +1392,16 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo mValueMask.setOff(); mBuffer.setOutOfCore(false); } else { -#ifdef OPENVDB_USE_DELAYED_LOADING - // If this node lies completely inside the clipping region and it is being read - // from a memory-mapped file, delay loading of its buffer until the buffer - // is actually accessed. (If this node requires clipping, its buffer - // must be accessed and therefore must be loaded.) - io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is); - const bool delayLoad = ((mappedFile.get() != nullptr) && clipBBox.isInside(nodeBBox)); - - if (delayLoad) { - // Save the offset to the value mask (maskpos), because the in-memory copy - // might change before the value buffer gets read. - mBuffer.enableOutOfCore(meta, is.tellg(), mappedFile, maskpos); - // Skip over voxel values. - skipCompressedValues(seekable, is, fromHalf); - } else { -#endif - mBuffer.allocate(); - io::readCompressedValues(is, mBuffer.mData, SIZE, mValueMask, fromHalf); - mBuffer.setOutOfCore(false); - - // Get this tree's background value. - T background = zeroVal(); - if (const void* bgPtr = io::getGridBackgroundValuePtr(is)) { - background = *static_cast(bgPtr); - } - this->clip(clipBBox, background); -#ifdef OPENVDB_USE_DELAYED_LOADING + mBuffer.allocate(); + io::readCompressedValues(is, mBuffer.mData, SIZE, mValueMask, fromHalf); + mBuffer.setOutOfCore(false); + + // Get this tree's background value. + T background = zeroVal(); + if (const void* bgPtr = io::getGridBackgroundValuePtr(is)) { + background = *static_cast(bgPtr); } -#endif + this->clip(clipBBox, background); } if (numBuffers > 1) { diff --git a/openvdb/openvdb/unittest/TestAttributeArray.cc b/openvdb/openvdb/unittest/TestAttributeArray.cc index 0b949c3ece..dc360e6fb1 100644 --- a/openvdb/openvdb/unittest/TestAttributeArray.cc +++ b/openvdb/openvdb/unittest/TestAttributeArray.cc @@ -12,31 +12,6 @@ #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#ifdef __clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-macros" -#endif -// Boost.Interprocess uses a header-only portion of Boost.DateTime -#define BOOST_DATE_TIME_NO_LIB -#ifdef __clang__ -#pragma GCC diagnostic pop -#endif -#include -#include - -#ifdef _WIN32 -#include // open_existing_file(), close_file() -// boost::interprocess::detail was renamed to boost::interprocess::ipcdetail in Boost 1.48. -// Ensure that both namespaces exist. -namespace boost { namespace interprocess { namespace detail {} namespace ipcdetail {} } } -#include -#else -#include // for struct stat -#include // for stat() -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include #include @@ -113,7 +88,6 @@ class TestAttributeArray: public ::testing::Test void testRegistry(); void testAccessorEval(); - void testDelayedLoad(); }; // class TestAttributeArray @@ -1400,801 +1374,6 @@ TEST_F(TestAttributeArray, testStrided) } } -#ifdef OPENVDB_USE_DELAYED_LOADING -void -TestAttributeArray::testDelayedLoad() -{ - using AttributeArrayI = TypedAttributeArray; - using AttributeArrayF = TypedAttributeArray; - - AttributeArrayI::registerType(); - AttributeArrayF::registerType(); - - SharedPtr mappedFile; - - io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); - - std::string tempDir; - if (const char* dir = std::getenv("TMPDIR")) tempDir = dir; -#ifdef _WIN32 - if (tempDir.empty()) { - char tempDirBuffer[MAX_PATH+1]; - int tempDirLen = GetTempPath(MAX_PATH+1, tempDirBuffer); - EXPECT_TRUE(tempDirLen > 0 && tempDirLen <= MAX_PATH); - tempDir = tempDirBuffer; - } -#else - if (tempDir.empty()) tempDir = P_tmpdir; -#endif - - { // IO - const Index count = 50; - AttributeArrayI attrA(count); - - for (unsigned i = 0; i < unsigned(count); ++i) { - attrA.set(i, int(i)); - } - - AttributeArrayF attrA2(count); - - std::string filename; - - // write out attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed1"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrA.writeMetadata(fileout, false, /*paged=*/true); - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrA.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrA.writePagedBuffers(outputStream, false); - outputStream.flush(); - - attrA2.writeMetadata(fileout, false, /*paged=*/true); - compression::PagedOutputStream outputStreamSize2(fileout); - outputStreamSize2.setSizeOnly(true); - attrA2.writePagedBuffers(outputStreamSize2, false); - outputStreamSize2.flush(); - compression::PagedOutputStream outputStream2(fileout); - outputStream2.setSizeOnly(false); - attrA2.writePagedBuffers(outputStream2, false); - outputStream2.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check manual loading of data - { - AttributeArrayI attrB; - AttributeArrayF attrB2; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(matchingNamePairs(attrA.type(), attrB.type())); - EXPECT_EQ(attrA.size(), attrB.size()); - EXPECT_EQ(attrA.isUniform(), attrB.isUniform()); - EXPECT_EQ(attrA.isTransient(), attrB.isTransient()); - EXPECT_EQ(attrA.isHidden(), attrB.isHidden()); - - AttributeArrayI attrBcopy(attrB); - AttributeArrayI attrBequal = attrB; - - EXPECT_TRUE(attrB.isOutOfCore()); - EXPECT_TRUE(attrBcopy.isOutOfCore()); - EXPECT_TRUE(attrBequal.isOutOfCore()); - - EXPECT_TRUE(!static_cast(attrB).isDataLoaded()); - EXPECT_TRUE(!static_cast(attrBcopy).isDataLoaded()); - EXPECT_TRUE(!static_cast(attrBequal).isDataLoaded()); - - attrB.loadData(); - attrBcopy.loadData(); - attrBequal.loadData(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrBcopy.isOutOfCore()); - EXPECT_TRUE(!attrBequal.isOutOfCore()); - - EXPECT_TRUE(static_cast(attrB).isDataLoaded()); - EXPECT_TRUE(static_cast(attrBcopy).isDataLoaded()); - EXPECT_TRUE(static_cast(attrBequal).isDataLoaded()); - - EXPECT_EQ(attrA.memUsage(), attrB.memUsage()); - EXPECT_EQ(attrA.memUsage(), attrBcopy.memUsage()); - EXPECT_EQ(attrA.memUsage(), attrBequal.memUsage()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - EXPECT_EQ(attrA.get(i), attrBcopy.get(i)); - EXPECT_EQ(attrA.get(i), attrBequal.get(i)); - } - - attrB2.readMetadata(filein); - compression::PagedInputStream inputStream2(filein); - inputStream2.setSizeOnly(true); - attrB2.readPagedBuffers(inputStream2); - inputStream2.setSizeOnly(false); - attrB2.readPagedBuffers(inputStream2); - - EXPECT_TRUE(matchingNamePairs(attrA2.type(), attrB2.type())); - EXPECT_EQ(attrA2.size(), attrB2.size()); - EXPECT_EQ(attrA2.isUniform(), attrB2.isUniform()); - EXPECT_EQ(attrA2.isTransient(), attrB2.isTransient()); - EXPECT_EQ(attrA2.isHidden(), attrB2.isHidden()); - - AttributeArrayF attrB2copy(attrB2); - AttributeArrayF attrB2equal = attrB2; - - EXPECT_TRUE(attrB2.isOutOfCore()); - EXPECT_TRUE(attrB2copy.isOutOfCore()); - EXPECT_TRUE(attrB2equal.isOutOfCore()); - attrB2.loadData(); - attrB2copy.loadData(); - attrB2equal.loadData(); - - EXPECT_TRUE(!attrB2.isOutOfCore()); - EXPECT_TRUE(!attrB2copy.isOutOfCore()); - EXPECT_TRUE(!attrB2equal.isOutOfCore()); - - EXPECT_EQ(attrA2.memUsage(), attrB2.memUsage()); - EXPECT_EQ(attrA2.memUsage(), attrB2copy.memUsage()); - EXPECT_EQ(attrA2.memUsage(), attrB2equal.memUsage()); - - EXPECT_EQ(attrA2.get(0), attrB2.get(0)); - EXPECT_EQ(attrA2.get(0), attrB2copy.get(0)); - EXPECT_EQ(attrA2.get(0), attrB2equal.get(0)); - } - - // read in using delayed load and check fill() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.fill(5); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(5, attrB.get(i)); - } - } - - // read in using delayed load and check streaming (write handle) - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.setStreaming(true); - - { - AttributeWriteHandle handle(attrB); - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrB.isUniform()); - } - - EXPECT_TRUE(!attrB.isUniform()); - } - - // read in using delayed load and check streaming (read handle) - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.setStreaming(true); - - { - AttributeHandle handle(attrB); - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrB.isUniform()); - } - - EXPECT_TRUE(attrB.isUniform()); - } - - // read in using delayed load and check implicit load through get() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - attrB.get(0); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - } - } - - // read in using delayed load and check implicit load through compress() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - } - - // read in using delayed load and check copy and assignment constructors - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeArrayI attrC(attrB); - AttributeArrayI attrD = attrB; - - EXPECT_TRUE(attrB.isOutOfCore()); - EXPECT_TRUE(attrC.isOutOfCore()); - EXPECT_TRUE(attrD.isOutOfCore()); - - attrB.loadData(); - attrC.loadData(); - attrD.loadData(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrC.isOutOfCore()); - EXPECT_TRUE(!attrD.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - EXPECT_EQ(attrA.get(i), attrC.get(i)); - EXPECT_EQ(attrA.get(i), attrD.get(i)); - } - } - - // read in using delayed load and check implicit load through AttributeHandle - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeHandle handle(attrB); - - EXPECT_TRUE(!attrB.isOutOfCore()); - } - - // read in using delayed load and check detaching of file (using collapse()) - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - EXPECT_TRUE(!attrB.isUniform()); - - attrB.collapse(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(attrB.isUniform()); - - EXPECT_EQ(0, attrB.get(0)); - } - - // read in and write out using delayed load to check writing out-of-core attributes - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - std::string filename2 = tempDir + "/openvdb_delayed5"; - std::ofstream fileout2(filename2.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout2, streamMetadata); - io::setDataCompression(fileout2, io::COMPRESS_BLOSC); - - attrB.writeMetadata(fileout2, false, /*paged=*/true); - compression::PagedOutputStream outputStreamSize(fileout2); - outputStreamSize.setSizeOnly(true); - attrB.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout2); - outputStream.setSizeOnly(false); - attrB.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout2.close(); - - AttributeArrayI attrB2; - - std::ifstream filein2(filename2.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein2, streamMetadata); - io::setMappedFilePtr(filein2, mappedFile); - - attrB2.readMetadata(filein2); - compression::PagedInputStream inputStream2(filein2); - inputStream2.setSizeOnly(true); - attrB2.readPagedBuffers(inputStream2); - inputStream2.setSizeOnly(false); - attrB2.readPagedBuffers(inputStream2); - - EXPECT_TRUE(attrB2.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrB.get(i), attrB2.get(i)); - } - - filein2.close(); - } - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - AttributeArrayI attrUniform(count); - - // write out uniform attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed2"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrUniform.writeMetadata(fileout, false, /*paged=*/true); - - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrUniform.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrUniform.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check fill() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isUniform()); - - attrB.fill(5); - - EXPECT_TRUE(attrB.isUniform()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(5, attrB.get(i)); - } - } - - AttributeArrayI attrStrided(count, /*stride=*/3); - - EXPECT_EQ(Index(3), attrStrided.stride()); - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - // write out strided attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed3"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrStrided.writeMetadata(fileout, false, /*paged=*/true); - - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrStrided.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrStrided.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check fill() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_EQ(Index(3), attrB.stride()); - } - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - // write out compressed attribute array to a temp file - { - filename = tempDir + "/openvdb_delayed4"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - attrA.writeMetadata(fileout, false, /*paged=*/true); - - compression::PagedOutputStream outputStreamSize(fileout); - outputStreamSize.setSizeOnly(true); - attrA.writePagedBuffers(outputStreamSize, false); - outputStreamSize.flush(); - compression::PagedOutputStream outputStream(fileout); - outputStream.setSizeOnly(false); - attrA.writePagedBuffers(outputStream, false); - outputStream.flush(); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check manual loading of data - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - attrB.loadData(); - EXPECT_TRUE(!attrB.isOutOfCore()); - - EXPECT_EQ(attrA.memUsage(), attrB.memUsage()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - } - } - - // read in using delayed load and check partial read state - { - std::unique_ptr attrB(new AttributeArrayI); - - EXPECT_TRUE(!(attrB->flags() & AttributeArray::PARTIALREAD)); - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB->readMetadata(filein); - - // PARTIALREAD flag should now be set - EXPECT_TRUE(attrB->flags() & AttributeArray::PARTIALREAD); - - // copy-construct and assign AttributeArray - AttributeArrayI attrC(*attrB); - EXPECT_TRUE(attrC.flags() & AttributeArray::PARTIALREAD); - AttributeArrayI attrD = *attrB; - EXPECT_TRUE(attrD.flags() & AttributeArray::PARTIALREAD); - - // verify deleting attrB is safe - attrB.reset(); - - // verify data is not valid - EXPECT_TRUE(!attrC.validData()); - - { // attempting to write a partially-read AttributeArray throws - std::string filename = tempDir + "/openvdb_partial1"; - ScopedFile f(filename); - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - EXPECT_THROW(attrC.writeMetadata(fileout, false, /*paged=*/true), IoError); - } - - // continue loading with copy-constructed AttributeArray - - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrC.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrC.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrC.isOutOfCore()); - attrC.loadData(); - EXPECT_TRUE(!attrC.isOutOfCore()); - - // verify data is now valid - EXPECT_TRUE(attrC.validData()); - - EXPECT_EQ(attrA.memUsage(), attrC.memUsage()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrC.get(i)); - } - } - - // read in using delayed load and check implicit load through get() - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - attrB.get(0); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - } - } - -#ifdef OPENVDB_USE_BLOSC - // read in using delayed load and check copy and assignment constructors - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeArrayI attrC(attrB); - AttributeArrayI attrD = attrB; - - EXPECT_TRUE(attrB.isOutOfCore()); - EXPECT_TRUE(attrC.isOutOfCore()); - EXPECT_TRUE(attrD.isOutOfCore()); - - attrB.loadData(); - attrC.loadData(); - attrD.loadData(); - - EXPECT_TRUE(!attrB.isOutOfCore()); - EXPECT_TRUE(!attrC.isOutOfCore()); - EXPECT_TRUE(!attrD.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), attrB.get(i)); - EXPECT_EQ(attrA.get(i), attrC.get(i)); - EXPECT_EQ(attrA.get(i), attrD.get(i)); - } - } - - // read in using delayed load and check implicit load through AttributeHandle - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - attrB.readMetadata(filein); - compression::PagedInputStream inputStream(filein); - inputStream.setSizeOnly(true); - attrB.readPagedBuffers(inputStream); - inputStream.setSizeOnly(false); - attrB.readPagedBuffers(inputStream); - - EXPECT_TRUE(attrB.isOutOfCore()); - - AttributeHandle handle(attrB); - - EXPECT_TRUE(!attrB.isOutOfCore()); - - for (unsigned i = 0; i < unsigned(count); ++i) { - EXPECT_EQ(attrA.get(i), handle.get(i)); - } - } -#endif - - // Clean up temp files. - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - - // write out invalid serialization flags as metadata to a temp file - { - filename = tempDir + "/openvdb_delayed5"; - std::ofstream fileout(filename.c_str(), std::ios_base::binary); - io::setStreamMetadataPtr(fileout, streamMetadata); - io::setDataCompression(fileout, io::COMPRESS_BLOSC); - - // write out unknown serialization flags to check forwards-compatibility - - Index64 bytes(0); - uint8_t flags(0); - uint8_t serializationFlags(Int16(0x10)); - Index size(0); - - fileout.write(reinterpret_cast(&bytes), sizeof(Index64)); - fileout.write(reinterpret_cast(&flags), sizeof(uint8_t)); - fileout.write(reinterpret_cast(&serializationFlags), sizeof(uint8_t)); - fileout.write(reinterpret_cast(&size), sizeof(Index)); - - fileout.close(); - } - - mappedFile = TestMappedFile::create(filename); - - // read in using delayed load and check metadata fail due to serialization flags - { - AttributeArrayI attrB; - - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - EXPECT_THROW(attrB.readMetadata(filein), openvdb::IoError); - } - - // cleanup temp files - - std::remove(mappedFile->filename().c_str()); - std::remove(filename.c_str()); - } -} -TEST_F(TestAttributeArray, testDelayedLoad) { testDelayedLoad(); } -#endif - TEST_F(TestAttributeArray, testDefaultValue) { diff --git a/openvdb/openvdb/unittest/TestCount.cc b/openvdb/openvdb/unittest/TestCount.cc index 3ce3f5227b..0509b71359 100644 --- a/openvdb/openvdb/unittest/TestCount.cc +++ b/openvdb/openvdb/unittest/TestCount.cc @@ -8,7 +8,6 @@ #include // tools::createLevelSetSphere #include // tools::sdfToFogVolume #include -#include class TestCount: public ::testing::Test @@ -238,49 +237,6 @@ TEST_F(TestCount, testMemUsage) EXPECT_EQ(expectedMaxMem, inCoreMemUsage); EXPECT_EQ(expectedMaxMem, memUsageIfLoaded); - -#ifdef OPENVDB_USE_DELAYED_LOADING - // Write out the grid and read it in with delay-loading. Check the - // expected memory usage values.] - - openvdb::initialize(); - - std::string filename; - - // write out grid to a temp file - { - io::TempFile file; - filename = file.filename(); - io::File fileOut(filename); - fileOut.write({grid}); - } - - io::File fileIn(filename); - fileIn.open(true); // delay-load - auto grids = fileIn.getGrids(); - fileIn.close(); - - grid = GridBase::grid((*grids)[0]); - EXPECT_TRUE(grid); - - inCoreMemUsage = tools::memUsage(grid->tree()); - memUsageIfLoaded = tools::memUsageIfLoaded(grid->tree()); - - EXPECT_EQ(expectedMaxMem, memUsageIfLoaded); - EXPECT_TRUE(inCoreMemUsage < expectedMaxMem); - - // in core memory should be the max memory without the leaf buffers but - // with the FileInfo - - const Index64 leafBuffers = sizeof(FloatGrid::ValueType) * FloatTree::LeafNodeType::SIZE; - const Index64 fileInfo = sizeof(FloatTree::LeafNodeType::Buffer::FileInfo); - const Index64 expectedInCoreMemUsage = expectedMaxMem + (leafCount * (-leafBuffers + fileInfo)); - EXPECT_EQ(expectedInCoreMemUsage, inCoreMemUsage); - - std::remove(filename.c_str()); - - openvdb::uninitialize(); -#endif } diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index dd344cb515..0d9c9a3e8a 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -622,10 +622,6 @@ TEST_F(TestFile, testWriteInstancedGrids) EXPECT_TRUE(grid.get() != nullptr); density = gridPtrCast(grid)->treePtr(); EXPECT_TRUE(density.get() != nullptr); -#ifdef OPENVDB_USE_DELAYED_LOADING - EXPECT_TRUE(density->unallocatedLeafCount() > 0); - EXPECT_EQ(density->leafCount(), density->unallocatedLeafCount()); -#endif // OPENVDB_USE_DELAYED_LOADING grid = findGridByName(*grids, "density_copy"); EXPECT_TRUE(grid.get() != nullptr); EXPECT_TRUE(gridPtrCast(grid)->treePtr().get() != nullptr); diff --git a/openvdb/openvdb/unittest/TestPointConversion.cc b/openvdb/openvdb/unittest/TestPointConversion.cc index d6420019cf..8d4132b92d 100644 --- a/openvdb/openvdb/unittest/TestPointConversion.cc +++ b/openvdb/openvdb/unittest/TestPointConversion.cc @@ -1,7 +1,6 @@ // Copyright Contributors to the OpenVDB Project // SPDX-License-Identifier: Apache-2.0 -#include #include #include #include diff --git a/openvdb/openvdb/unittest/TestPointCount.cc b/openvdb/openvdb/unittest/TestPointCount.cc index 58ea887907..f02661456c 100644 --- a/openvdb/openvdb/unittest/TestPointCount.cc +++ b/openvdb/openvdb/unittest/TestPointCount.cc @@ -3,7 +3,6 @@ #include #include -#include #include #include @@ -315,19 +314,6 @@ TEST_F(TestPointCount, testGroup) GroupFilter groupFilter("test", attributeSet); bool inCoreOnly; -#ifdef OPENVDB_USE_DELAYED_LOADING - inCoreOnly = true; - - EXPECT_EQ(pointCount(inputTree, NullFilter(), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, ActiveFilter(), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, InactiveFilter(), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, groupFilter, inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, ActiveFilter()), inCoreOnly), Index64(0)); - EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, InactiveFilter()), inCoreOnly), Index64(0)); -#endif - inCoreOnly = false; EXPECT_EQ(pointCount(inputTree, NullFilter(), inCoreOnly), Index64(4)); @@ -529,51 +515,6 @@ TEST_F(TestPointCount, testOffsets) fileOut.write(grids); } -#ifdef OPENVDB_USE_DELAYED_LOADING - // test point offsets for a delay-loaded grid - { - io::File fileIn(filename); - fileIn.open(); - - GridPtrVecPtr grids = fileIn.getGrids(); - - fileIn.close(); - - EXPECT_EQ(grids->size(), size_t(1)); - - PointDataGrid::Ptr inputGrid = GridBase::grid((*grids)[0]); - - EXPECT_TRUE(inputGrid); - - PointDataTree& inputTree = inputGrid->tree(); - - std::vector offsets; - std::vector includeGroups; - std::vector excludeGroups; - - MultiGroupFilter filter(includeGroups, excludeGroups, inputTree.cbeginLeaf()->attributeSet()); - Index64 total = pointOffsets(offsets, inputTree, filter, /*inCoreOnly=*/true); - - EXPECT_EQ(offsets.size(), size_t(4)); - EXPECT_EQ(offsets[0], Index64(0)); - EXPECT_EQ(offsets[1], Index64(0)); - EXPECT_EQ(offsets[2], Index64(0)); - EXPECT_EQ(offsets[3], Index64(0)); - EXPECT_EQ(total, Index64(0)); - - offsets.clear(); - - total = pointOffsets(offsets, inputTree, filter, /*inCoreOnly=*/false); - - EXPECT_EQ(offsets.size(), size_t(4)); - EXPECT_EQ(offsets[0], Index64(1)); - EXPECT_EQ(offsets[1], Index64(3)); - EXPECT_EQ(offsets[2], Index64(4)); - EXPECT_EQ(offsets[3], Index64(5)); - EXPECT_EQ(total, Index64(5)); - } -#endif - std::remove(filename.c_str()); } diff --git a/openvdb/openvdb/unittest/TestPointDataLeaf.cc b/openvdb/openvdb/unittest/TestPointDataLeaf.cc index bd96959c45..651b3174d9 100644 --- a/openvdb/openvdb/unittest/TestPointDataLeaf.cc +++ b/openvdb/openvdb/unittest/TestPointDataLeaf.cc @@ -1165,111 +1165,6 @@ TEST_F(TestPointDataLeaf, testIO) EXPECT_TRUE(leaf == *leafFromDisk); } - -#ifdef OPENVDB_USE_DELAYED_LOADING - { // read grids from file and pre-fetch - PointDataGrid::Ptr gridFromDisk; - - { - io::File file("leaf.vdb"); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - gridFromDisk = openvdb::gridPtrCast(baseGrid); - } - - LeafType* leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); - EXPECT_TRUE(leafFromDisk); - - const AttributeVec3s& position( - AttributeVec3s::cast(leafFromDisk->constAttributeArray("P"))); - const AttributeF& density( - AttributeF::cast(leafFromDisk->constAttributeArray("density"))); - - EXPECT_TRUE(leafFromDisk->buffer().isOutOfCore()); -#ifdef OPENVDB_USE_BLOSC - EXPECT_TRUE(position.isOutOfCore()); - EXPECT_TRUE(density.isOutOfCore()); -#else - // delayed-loading is only available on attribute arrays when using Blosc - EXPECT_TRUE(!position.isOutOfCore()); - EXPECT_TRUE(!density.isOutOfCore()); -#endif - - // prefetch voxel data only - prefetch(gridFromDisk->tree(), /*position=*/false, /*attributes=*/false); - - // ensure out-of-core data is now in-core after pre-fetching - - EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore()); -#ifdef OPENVDB_USE_BLOSC - EXPECT_TRUE(position.isOutOfCore()); - EXPECT_TRUE(density.isOutOfCore()); -#else - EXPECT_TRUE(!position.isOutOfCore()); - EXPECT_TRUE(!density.isOutOfCore()); -#endif - - { // re-open - io::File file("leaf.vdb"); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - gridFromDisk = openvdb::gridPtrCast(baseGrid); - } - - leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); - EXPECT_TRUE(leafFromDisk); - - const AttributeVec3s& position2( - AttributeVec3s::cast(leafFromDisk->constAttributeArray("P"))); - const AttributeF& density2( - AttributeF::cast(leafFromDisk->constAttributeArray("density"))); - - // prefetch voxel and position attribute data - prefetch(gridFromDisk->tree(), /*position=*/true, /*attribute=*/false); - - // ensure out-of-core voxel and position data is now in-core after pre-fetching - - EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore()); - EXPECT_TRUE(!position2.isOutOfCore()); -#ifdef OPENVDB_USE_BLOSC - EXPECT_TRUE(density2.isOutOfCore()); -#else - EXPECT_TRUE(!density2.isOutOfCore()); -#endif - - { // re-open - io::File file("leaf.vdb"); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - gridFromDisk = openvdb::gridPtrCast(baseGrid); - } - - leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); - EXPECT_TRUE(leafFromDisk); - - const AttributeVec3s& position3( - AttributeVec3s::cast(leafFromDisk->constAttributeArray("P"))); - const AttributeF& density3( - AttributeF::cast(leafFromDisk->constAttributeArray("density"))); - - // prefetch all data - prefetch(gridFromDisk->tree()); - - // ensure out-of-core voxel and position data is now in-core after pre-fetching - - EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore()); - EXPECT_TRUE(!position3.isOutOfCore()); - EXPECT_TRUE(!density3.isOutOfCore()); - } - - remove("leaf.vdb"); -#endif // OPENVDB_USE_DELAYED_LOADING } { // test multi-buffer IO with varying attribute storage per-leaf diff --git a/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc b/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc index a7f85cd33e..572b66dd2b 100644 --- a/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc +++ b/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc @@ -2154,9 +2154,6 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) auto leaf = points->tree().cbeginLeaf(); EXPECT_TRUE(leaf); -#ifdef OPENVDB_USE_DELAYED_LOADING - EXPECT_TRUE(leaf->buffer().isOutOfCore()); -#endif using Rasterizer = FrustumRasterizer; using Settings = FrustumRasterizerSettings; @@ -2327,17 +2324,10 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) auto points2 = points->deepCopy(); points2->setTransform(transform); -#ifdef OPENVDB_USE_DELAYED_LOADING - // verify both grids are out-of-core - - EXPECT_TRUE(points->tree().cbeginLeaf()->buffer().isOutOfCore()); - EXPECT_TRUE(points2->tree().cbeginLeaf()->buffer().isOutOfCore()); -#endif - #ifndef ONLY_RASTER_FLOAT // memory tests - if (io::Archive::isDelayedLoadingEnabled() && io::Archive::hasBloscCompression()) { + if (io::Archive::hasBloscCompression()) { FloatGrid::Ptr density1, density2, density3; Vec3SGrid::Ptr velocity1, velocity2, velocity3; @@ -2345,122 +2335,31 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) const size_t mb = 1024*1024; const size_t tinyMemory = static_cast(0.1*mb); - size_t initialMemory; - { // memory test 1 - retain caches and streaming disabled Rasterizer rasterizer(settings); rasterizer.addPoints(points, /*stream=*/false); rasterizer.addPoints(points2, /*stream=*/false); - initialMemory = rasterizer.memUsage(); + // memory usage should be around 80-100MB throughout rasterization - EXPECT_TRUE(initialMemory > size_t(4*mb) && initialMemory < size_t(16*mb)); + EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) && + rasterizer.memUsage() < size_t(100*mb)); EXPECT_EQ(size_t(2), rasterizer.size()); velocity1 = rasterizer.rasterizeAttribute("v"); EXPECT_EQ(Index64(219780), velocity1->activeVoxelCount()); - EXPECT_TRUE(rasterizer.memUsage() > size_t(71*mb) && - rasterizer.memUsage() < size_t(91*mb)); + EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) && + rasterizer.memUsage() < size_t(100*mb)); density1 = rasterizer.rasterizeDensity("density"); EXPECT_EQ(Index64(219780), density1->activeVoxelCount()); - // no data is discarded so expect a fairly high memory footprint - EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) && rasterizer.memUsage() < size_t(100*mb)); } - - { // memory test 2 - retain caches and streaming enabled - - { // reopen file and deep copy while setting transform - io::File file(filename); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - points = openvdb::gridPtrCast(baseGrid); - points2 = points->deepCopy(); - points2->setTransform(transform); - } - - Rasterizer rasterizer(settings); - - rasterizer.addPoints(points, /*stream=*/true); - rasterizer.addPoints(points2, /*stream=*/true); - - EXPECT_EQ(initialMemory, rasterizer.memUsage()); - - EXPECT_EQ(size_t(2), rasterizer.size()); - - velocity2 = rasterizer.rasterizeAttribute("v"); - EXPECT_EQ(Index64(219780), velocity2->activeVoxelCount()); - - size_t postRasterMemory = rasterizer.memUsage(); - - EXPECT_TRUE(postRasterMemory > size_t(70*mb) && postRasterMemory < size_t(85*mb)); - - density2 = rasterizer.rasterizeDensity("density"); - EXPECT_EQ(Index64(219780), density2->activeVoxelCount()); - - // as data is being streamed, second attribute shouldn't change memory usage very much - - EXPECT_TRUE(rasterizer.memUsage() < (postRasterMemory + tinyMemory)); - } - - { // memory test 3 - release caches and streaming enabled - - { // reopen file and deep copy while setting transform - io::File file(filename); - file.open(); - openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); - file.close(); - - points = openvdb::gridPtrCast(baseGrid); - points2 = points->deepCopy(); - points2->setTransform(transform); - } - - auto points3 = points->deepCopy(); - auto points4 = points2->deepCopy(); - - Settings settings2(*frustum); - settings2.threshold = 0.0f; - - Mask mask2(*frustum, nullptr, BBoxd(), /*clipToFrustum=*/false); - - Rasterizer rasterizer(settings2, mask2); - - rasterizer.addPoints(points, /*stream=*/true); - rasterizer.addPoints(points2, /*stream=*/true); - - EXPECT_EQ(initialMemory, rasterizer.memUsage()); - EXPECT_EQ(size_t(2), rasterizer.size()); - - density3 = rasterizer.rasterizeDensity("density", RasterMode::ACCUMULATE, true); - EXPECT_EQ(Index64(219780), density3->activeVoxelCount()); - - // all voxel data, attribute data and caches are being discarded, - // so memory after rasterizing shouldn't change very much - - EXPECT_TRUE(rasterizer.memUsage() < (initialMemory + tinyMemory)); - - // deep-copies of delay-loaded point grids need to be used for repeat rasterization - - rasterizer.clear(); - rasterizer.addPoints(points3, /*stream=*/true); - rasterizer.addPoints(points4, /*stream=*/true); - - EXPECT_EQ(size_t(2), rasterizer.size()); - - EXPECT_TRUE(rasterizer.memUsage() < (initialMemory + tinyMemory)); - - velocity3 = rasterizer.rasterizeAttribute("v", RasterMode::ACCUMULATE, true); - EXPECT_EQ(Index64(219780), velocity3->activeVoxelCount()); - } } #endif diff --git a/openvdb/openvdb/unittest/TestStreamCompression.cc b/openvdb/openvdb/unittest/TestStreamCompression.cc index ceb8bcf96b..27af0bf40d 100644 --- a/openvdb/openvdb/unittest/TestStreamCompression.cc +++ b/openvdb/openvdb/unittest/TestStreamCompression.cc @@ -8,31 +8,6 @@ #include -#ifdef OPENVDB_USE_DELAYED_LOADING -#ifdef __clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-macros" -#endif -// Boost.Interprocess uses a header-only portion of Boost.DateTime -#define BOOST_DATE_TIME_NO_LIB -#ifdef __clang__ -#pragma GCC diagnostic pop -#endif -#include -#include -#include -#include - -#ifdef _WIN32 -#include // open_existing_file(), close_file() -#include -#else -#include // for struct stat -#include // for stat() -#include // for unlink() -#endif -#endif // OPENVDB_USE_DELAYED_LOADING - #include #include #include // for std::iota() @@ -41,6 +16,10 @@ #include #endif +#ifdef _WIN32 +#include +#endif + using namespace openvdb; using namespace openvdb::compression; @@ -467,98 +446,6 @@ TestStreamCompression::testPagedStreams() EXPECT_EQ(fileout.tellp(), std::streampos(values.size()+sizeof(int)*pages)); #endif - -#ifdef OPENVDB_USE_DELAYED_LOADING - auto mappedFile = TestMappedFile::create(filename); - - // read - std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary); - io::setStreamMetadataPtr(filein, streamMetadata); - io::setMappedFilePtr(filein, mappedFile); - - EXPECT_EQ(filein.tellg(), std::streampos(0)); - - PagedInputStream istreamSizeOnly(filein); - istreamSizeOnly.setSizeOnly(true); - - std::vector handles; - - for (size_t i = 0; i < values.size(); i += increment) { - if (size_t(i+increment) > values.size()) { - handles.push_back(istreamSizeOnly.createHandle(values.size() - i)); - } - else { - handles.push_back(istreamSizeOnly.createHandle(increment)); - } - } - -#ifdef OPENVDB_USE_BLOSC - // two integers - compressed size and uncompressed size - EXPECT_EQ(filein.tellg(), std::streampos(pages*sizeof(int)*2)); -#else - // one integer - uncompressed size - EXPECT_EQ(filein.tellg(), std::streampos(pages*sizeof(int))); -#endif - - PagedInputStream istream(filein); - - int pageHandle = 0; - - for (size_t i = 0; i < values.size(); i += increment) { - if (size_t(i+increment) > values.size()) { - istream.read(handles[pageHandle++], values.size() - i); - } - else { - istream.read(handles[pageHandle++], increment); - } - } - - // first three handles live in the same page - - Page& page0 = handles[0]->page(); - Page& page1 = handles[1]->page(); - Page& page2 = handles[2]->page(); - Page& page3 = handles[3]->page(); - - EXPECT_TRUE(page0.isOutOfCore()); - EXPECT_TRUE(page1.isOutOfCore()); - EXPECT_TRUE(page2.isOutOfCore()); - EXPECT_TRUE(page3.isOutOfCore()); - - handles[0]->read(); - - // store the Page shared_ptr - - Page::Ptr page = handles[0]->mPage; - - // verify use count is four (one plus three handles) - - EXPECT_EQ(page.use_count(), long(4)); - - // on reading from the first handle, all pages referenced - // in the first three handles are in-core - - EXPECT_TRUE(!page0.isOutOfCore()); - EXPECT_TRUE(!page1.isOutOfCore()); - EXPECT_TRUE(!page2.isOutOfCore()); - EXPECT_TRUE(page3.isOutOfCore()); - - handles[1]->read(); - - EXPECT_TRUE(handles[0]->mPage); - - handles[2]->read(); - - handles.erase(handles.begin()); - handles.erase(handles.begin()); - handles.erase(handles.begin()); - - // after all three handles have been read, - // page should have just one use count (itself) - - EXPECT_EQ(page.use_count(), long(1)); - -#endif // OPENVDB_USE_DELAYED_LOADING } std::remove(filename.c_str()); } diff --git a/openvdb/openvdb/unittest/util.h b/openvdb/openvdb/unittest/util.h index d2aeadea69..2527c6c8b9 100644 --- a/openvdb/openvdb/unittest/util.h +++ b/openvdb/openvdb/unittest/util.h @@ -10,18 +10,6 @@ #include // for pruneLevelSet #include -#ifdef OPENVDB_USE_DELAYED_LOADING -/// @brief io::MappedFile has a private constructor, so declare a class that acts as the friend -class TestMappedFile -{ -public: - static openvdb::io::MappedFile::Ptr create(const std::string& filename) - { - return openvdb::SharedPtr(new openvdb::io::MappedFile(filename)); - } -}; -#endif - namespace unittest_util { diff --git a/openvdb/openvdb/version.h.in b/openvdb/openvdb/version.h.in index b5838936a2..3cd8043661 100644 --- a/openvdb/openvdb/version.h.in +++ b/openvdb/openvdb/version.h.in @@ -138,11 +138,6 @@ #cmakedefine OPENVDB_USE_ZLIB #endif -/* Denotes whether VDB was built with Delayed Loading support */ -#ifndef OPENVDB_USE_DELAYED_LOADING -#cmakedefine OPENVDB_USE_DELAYED_LOADING -#endif - /* Denotes whether VDB was built asserts enabled in VDB code */ #ifndef OPENVDB_ENABLE_ASSERTS #cmakedefine OPENVDB_ENABLE_ASSERTS diff --git a/pyproject.toml b/pyproject.toml index b42bcd5c80..33a942f382 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ wheel.packages = [] OPENVDB_CORE_STATIC="OFF" USE_EXPLICIT_INSTANTIATION="OFF" DISABLE_DEPENDENCY_VERSION_CHECKS="ON" -OPENVDB_USE_DELAYED_LOADING="OFF" OPENVDB_BUILD_PYTHON_MODULE="ON" USE_NUMPY="ON" From 27f2841e2bf9959266d9527c46acf86d7134d3e3 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 30 Jan 2026 12:32:05 -0800 Subject: [PATCH 02/80] Remove Delayed Load Metadata seek optimization Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 2 - openvdb/openvdb/io/Archive.cc | 111 +------ openvdb/openvdb/io/Compression.h | 46 +-- openvdb/openvdb/io/DelayedLoadMetadata.cc | 300 ------------------ openvdb/openvdb/io/DelayedLoadMetadata.h | 102 ------ openvdb/openvdb/io/File.cc | 2 +- openvdb/openvdb/openvdb.cc | 1 - openvdb/openvdb/openvdb.h | 6 +- openvdb/openvdb/unittest/CMakeLists.txt | 1 - .../unittest/TestDelayedLoadMetadata.cc | 206 ------------ openvdb/openvdb/unittest/TestFile.cc | 157 +-------- openvdb/openvdb/unittest/TestStream.cc | 1 - 12 files changed, 25 insertions(+), 910 deletions(-) delete mode 100644 openvdb/openvdb/io/DelayedLoadMetadata.cc delete mode 100644 openvdb/openvdb/io/DelayedLoadMetadata.h delete mode 100644 openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index c496509a72..91fc5fcfd7 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -330,7 +330,6 @@ set(OPENVDB_LIBRARY_SOURCE_FILES Grid.cc io/Archive.cc io/Compression.cc - io/DelayedLoadMetadata.cc io/File.cc io/GridDescriptor.cc io/Queue.cc @@ -368,7 +367,6 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/Archive.h io/Compression.h - io/DelayedLoadMetadata.h io/File.h io/GridDescriptor.h io/io.h diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 03576b2566..0e48fdae04 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -4,7 +4,6 @@ #include "Archive.h" #include "GridDescriptor.h" -#include "DelayedLoadMetadata.h" #include "io.h" #include @@ -180,7 +179,7 @@ struct StreamMetadata::Impl uint32_t mPass = 0; MetaMap mGridMetadata; AuxDataMap mAuxData; - bool mDelayedLoadMeta = DelayedLoadMetadata::isRegisteredType(); + bool mDelayedLoadMeta = false; uint64_t mLeaf = 0; uint32_t mTest = 0; // for testing only }; // struct StreamMetadata @@ -277,7 +276,6 @@ StreamMetadata::str() const ostr << "compression: " << compressionToString(compression()) << "\n"; ostr << "half_float: " << halfFloat() << "\n"; ostr << "seekable: " << seekable() << "\n"; - ostr << "delayed_load_meta: " << delayedLoadMeta() << "\n"; ostr << "pass: " << pass() << "\n"; ostr << "counting_passes: " << countingPasses() << "\n"; ostr << "write_grid_stats_metadata: " << writeGridStats() << "\n"; @@ -310,73 +308,6 @@ writeAsType(std::ostream& os, const std::any& val) return false; } -struct PopulateDelayedLoadMetadataOp -{ - DelayedLoadMetadata& metadata; - uint32_t compression; - - PopulateDelayedLoadMetadataOp(DelayedLoadMetadata& _metadata, uint32_t _compression) - : metadata(_metadata) - , compression(_compression) { } - - template - void operator()(const GridT& grid) const - { - using TreeT = typename GridT::TreeType; - using ValueT = typename TreeT::ValueType; - using LeafT = typename TreeT::LeafNodeType; - using MaskT = typename LeafT::NodeMaskType; - - const TreeT& tree = grid.constTree(); - const Index64 leafCount = tree.leafCount(); - - // early exit if not leaf nodes - if (leafCount == Index64(0)) return; - - metadata.resizeMask(leafCount); - - if (compression & (COMPRESS_BLOSC | COMPRESS_ZIP)) { - metadata.resizeCompressedSize(leafCount); - } - - const auto background = tree.background(); - const bool saveFloatAsHalf = grid.saveFloatAsHalf(); - - tree::LeafManager leafManager(tree); - - leafManager.foreach( - [&](const LeafT& leaf, size_t idx) { - // set mask value - MaskCompress maskCompressData( - leaf.valueMask(), /*childMask=*/MaskT(), leaf.buffer().data(), background); - metadata.setMask(idx, maskCompressData.metadata); - - if (compression & (COMPRESS_BLOSC | COMPRESS_ZIP)) { - // set compressed size value - size_t sizeBytes(8); - size_t compressedSize = io::writeCompressedValuesSize( - leaf.buffer().data(), LeafT::SIZE, - leaf.valueMask(), maskCompressData.metadata, saveFloatAsHalf, compression); - metadata.setCompressedSize(idx, compressedSize+sizeBytes); - } - } - ); - } -}; - -bool populateDelayedLoadMetadata(DelayedLoadMetadata& metadata, - const GridBase& gridBase, uint32_t compression) -{ - PopulateDelayedLoadMetadataOp op(metadata, compression); - - using AllowedTypes = TypeList< - Int32Grid, Int64Grid, - FloatGrid, DoubleGrid, - Vec3IGrid, Vec3SGrid, Vec3DGrid>; - - return gridBase.apply(op); -} - } // unnamed namespace std::ostream& @@ -1016,17 +947,9 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const //grid->insertMeta(GridBase::META_FILE_COMPRESSION, // StringMetadata(compressionToString(c))); - const VersionId version = getLibraryVersion(is); - if (version.first < 6 || (version.first == 6 && version.second <= 1)) { - // If delay load metadata exists, but the file format version does not support - // delay load metadata, this likely means the original grid was read and then - // written using a prior version of OpenVDB and ABI>=5 where unknown metadata - // can be blindly copied. This means that it is possible for the metadata to - // no longer be in sync with the grid, so we remove it to ensure correctness. - - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } + // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists + if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { + grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); } streamMetadata->gridMetadata() = static_cast(*grid); @@ -1036,15 +959,6 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const // reset leaf value to zero streamMetadata->setLeaf(0); - // drop DelayedLoadMetadata from the grid as it is only useful for IO - // a stream metadata non-zero value disables this behaviour for testing - - if (streamMetadata->__test() == uint32_t(0)) { - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } - } - grid->readTransform(is); if (!gd.isInstance()) { grid->readTopology(is); @@ -1231,25 +1145,18 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, // Save the compression settings for this grid. setGridCompression(os, *grid); - // copy grid and add delay load metadata - const auto copyOfGrid = grid->copyGrid(); // shallow copy - const auto nonConstCopyOfGrid = ConstPtrCast(copyOfGrid); - nonConstCopyOfGrid->insertMeta(GridBase::META_FILE_DELAYED_LOAD, - DelayedLoadMetadata()); - DelayedLoadMetadata::Ptr delayLoadMeta = - nonConstCopyOfGrid->getMetadata(GridBase::META_FILE_DELAYED_LOAD); - if (!populateDelayedLoadMetadata(*delayLoadMeta, *grid, compression())) { - nonConstCopyOfGrid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } - // Save the grid's metadata and transform. if (getWriteGridStatsMetadata(os)) { // Compute and add grid statistics metadata. + const auto copyOfGrid = grid->copyGrid(); // shallow copy + const auto nonConstCopyOfGrid = ConstPtrCast(copyOfGrid); nonConstCopyOfGrid->addStatsMetadata(); nonConstCopyOfGrid->insertMeta(GridBase::META_FILE_COMPRESSION, StringMetadata(compressionToString(getDataCompression(os)))); + copyOfGrid->writeMeta(os); + } else { + grid->writeMeta(os); } - copyOfGrid->writeMeta(os); grid->writeTransform(os); // Save the grid's structure. diff --git a/openvdb/openvdb/io/Compression.h b/openvdb/openvdb/io/Compression.h index 44da4d8bcc..85bbf6b09f 100644 --- a/openvdb/openvdb/io/Compression.h +++ b/openvdb/openvdb/io/Compression.h @@ -9,7 +9,6 @@ #include // for negative() #include #include "io.h" // for getDataCompression(), etc. -#include "DelayedLoadMetadata.h" #include #include #include @@ -236,28 +235,20 @@ OPENVDB_API void bloscFromStream(std::istream&, char* data, size_t numBytes); /// @param count the number of elements to read in /// @param compression whether and how the data is compressed (either COMPRESS_NONE, /// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC) -/// @param metadata optional pointer to a DelayedLoadMetadata object that stores -/// the size of the compressed buffer -/// @param metadataOffset offset into DelayedLoadMetadata, ignored if pointer is null /// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled /// without Blosc support. /// @details This default implementation is instantiated only for types /// whose size can be determined by the sizeof() operator. template inline void -readData(std::istream& is, T* data, Index count, uint32_t compression, - DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) +readData(std::istream& is, T* data, Index count, uint32_t compression) { const bool seek = data == nullptr; if (seek) { OPENVDB_ASSERT(!getStreamMetadataPtr(is) || getStreamMetadataPtr(is)->seekable()); } - const bool hasCompression = compression & (COMPRESS_BLOSC | COMPRESS_ZIP); - if (metadata && seek && hasCompression) { - size_t compressedSize = metadata->getCompressedSize(metadataOffset); - is.seekg(compressedSize, std::ios_base::cur); - } else if (compression & COMPRESS_BLOSC) { + if (compression & COMPRESS_BLOSC) { bloscFromStream(is, reinterpret_cast(data), sizeof(T) * count); } else if (compression & COMPRESS_ZIP) { unzipFromStream(is, reinterpret_cast(data), sizeof(T) * count); @@ -271,8 +262,7 @@ readData(std::istream& is, T* data, Index count, uint32_t compression, /// Specialization for std::string input template<> inline void -readData(std::istream& is, std::string* data, Index count, uint32_t /*compression*/, - DelayedLoadMetadata* /*metadata*/, size_t /*metadataOffset*/) +readData(std::istream& is, std::string* data, Index count, uint32_t /*compression*/) { for (Index i = 0; i < count; ++i) { size_t len = 0; @@ -294,25 +284,22 @@ template struct HalfReader; /// Partial specialization for non-floating-point types (no half to float promotion) template struct HalfReader { - static inline void read(std::istream& is, T* data, Index count, uint32_t compression, - DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) { - readData(is, data, count, compression, metadata, metadataOffset); + static inline void read(std::istream& is, T* data, Index count, uint32_t compression) { + readData(is, data, count, compression); } }; /// Partial specialization for floating-point types template struct HalfReader { using HalfT = typename RealToHalf::HalfT; - static inline void read(std::istream& is, T* data, Index count, uint32_t compression, - DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) { + static inline void read(std::istream& is, T* data, Index count, uint32_t compression) { if (count < 1) return; if (data == nullptr) { // seek mode - pass through null pointer - readData(is, nullptr, count, compression, metadata, metadataOffset); + readData(is, nullptr, count, compression); } else { std::vector halfData(count); // temp buffer into which to read half float values - readData(is, reinterpret_cast(&halfData[0]), count, compression, - metadata, metadataOffset); + readData(is, reinterpret_cast(&halfData[0]), count, compression); // Copy half float values from the temporary buffer to the full float output array. std::copy(halfData.begin(), halfData.end(), data); } @@ -476,25 +463,12 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, const bool seek = (destBuf == nullptr); OPENVDB_ASSERT(!seek || (!meta || meta->seekable())); - // Get delayed load metadata if it exists - - DelayedLoadMetadata::Ptr delayLoadMeta; - uint64_t leafIndex(0); - if (seek && meta && meta->delayedLoadMeta()) { - delayLoadMeta = - meta->gridMetadata().getMetadata("file_delayed_load"); - leafIndex = meta->leaf(); - } - int8_t metadata = NO_MASK_AND_ALL_VALS; // Read the flag that specifies what, if any, additional metadata // (selection mask and/or inactive value(s)) is saved. if (seek && !maskCompressed) { is.seekg(/*bytes=*/1, std::ios_base::cur); - } else if (seek && delayLoadMeta) { - metadata = delayLoadMeta->getMask(leafIndex); - is.seekg(/*bytes=*/1, std::ios_base::cur); } else { is.read(reinterpret_cast(&metadata), /*bytes=*/1); } @@ -560,10 +534,10 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, // Read in the buffer. if (fromHalf) { HalfReader::isReal, ValueT>::read( - is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex); + is, (seek ? nullptr : tempBuf), tempCount, compression); } else { readData( - is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex); + is, (seek ? nullptr : tempBuf), tempCount, compression); } // If mask compression is enabled and the number of active values read into diff --git a/openvdb/openvdb/io/DelayedLoadMetadata.cc b/openvdb/openvdb/io/DelayedLoadMetadata.cc deleted file mode 100644 index 1f958b0a4c..0000000000 --- a/openvdb/openvdb/io/DelayedLoadMetadata.cc +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -#include "DelayedLoadMetadata.h" - -#include -#include - -#ifdef OPENVDB_USE_BLOSC -#include - -namespace { - -inline size_t padMask(size_t bytes) -{ - return size_t(std::ceil(static_cast(bytes+1) / - sizeof(openvdb::io::DelayedLoadMetadata::MaskType))); -} - -inline size_t padCompressedSize(size_t bytes) -{ - return size_t(std::ceil(static_cast(bytes+1) / - sizeof(openvdb::io::DelayedLoadMetadata::CompressedSizeType))); -} - -} // namespace - -#endif - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -DelayedLoadMetadata::DelayedLoadMetadata(const DelayedLoadMetadata& other) - : Metadata() - , mMask(other.mMask) - , mCompressedSize(other.mCompressedSize) -{ -} - -Name DelayedLoadMetadata::typeName() const -{ - return DelayedLoadMetadata::staticTypeName(); -} - -Metadata::Ptr DelayedLoadMetadata::copy() const -{ - Metadata::Ptr metadata(new DelayedLoadMetadata()); - metadata->copy(*this); - return metadata; -} - -void DelayedLoadMetadata::copy(const Metadata& other) -{ - const DelayedLoadMetadata* t = dynamic_cast(&other); - if (t == nullptr) OPENVDB_THROW(TypeError, "Incompatible type during copy"); - mMask = t->mMask; - mCompressedSize = t->mCompressedSize; -} - -std::string DelayedLoadMetadata::str() const -{ - return ""; -} - -bool DelayedLoadMetadata::asBool() const -{ - return false; -} - -Index32 DelayedLoadMetadata::size() const -{ - if (mMask.empty() && mCompressedSize.empty()) return Index32(0); - - // count - size_t size = sizeof(Index32); - - { // mask - size += sizeof(Index32); - size_t compressedSize = compression::bloscCompressedSize( - reinterpret_cast(mMask.data()), mMask.size()*sizeof(MaskType)); - - if (compressedSize > 0) size += compressedSize; - else size += mMask.size()*sizeof(MaskType); - } - { // compressed size - size += sizeof(Index32); - if (!mCompressedSize.empty()) { - size_t compressedSize = compression::bloscCompressedSize( - reinterpret_cast(mCompressedSize.data()), mCompressedSize.size()*sizeof(CompressedSizeType)); - - if (compressedSize > 0) size += compressedSize; - else size += mCompressedSize.size()*sizeof(CompressedSizeType); - } - } - - return static_cast(size); -} - -void DelayedLoadMetadata::clear() -{ - mMask.clear(); - mCompressedSize.clear(); -} - -bool DelayedLoadMetadata::empty() const -{ - return mMask.empty() && mCompressedSize.empty(); -} - -void DelayedLoadMetadata::resizeMask(size_t size) -{ - mMask.resize(size); -} - -void DelayedLoadMetadata::resizeCompressedSize(size_t size) -{ - mCompressedSize.resize(size); -} - -DelayedLoadMetadata::MaskType DelayedLoadMetadata::getMask(size_t index) const -{ - OPENVDB_ASSERT(DelayedLoadMetadata::isRegisteredType()); - OPENVDB_ASSERT(index < mMask.size()); - return mMask[index]; -} - -void DelayedLoadMetadata::setMask(size_t index, const MaskType& value) -{ - OPENVDB_ASSERT(index < mMask.size()); - mMask[index] = value; -} - -DelayedLoadMetadata::CompressedSizeType DelayedLoadMetadata::getCompressedSize(size_t index) const -{ - OPENVDB_ASSERT(DelayedLoadMetadata::isRegisteredType()); - OPENVDB_ASSERT(index < mCompressedSize.size()); - return mCompressedSize[index]; -} - -void DelayedLoadMetadata::setCompressedSize(size_t index, const CompressedSizeType& value) -{ - OPENVDB_ASSERT(index < mCompressedSize.size()); - mCompressedSize[index] = value; -} - -void DelayedLoadMetadata::readValue(std::istream& is, Index32 numBytes) -{ - if (numBytes == 0) return; - - // initial header size - size_t total = sizeof(Index32); - - Index32 count = 0; - is.read(reinterpret_cast(&count), sizeof(Index32)); - total += sizeof(Index32); - - Index32 bytes = 0; - is.read(reinterpret_cast(&bytes), sizeof(Index32)); - total += sizeof(Index32); - - if (bytes > Index32(0)) { - std::unique_ptr compressedBuffer(new char[bytes]); - is.read(reinterpret_cast(compressedBuffer.get()), bytes); - - total += bytes; - -#ifdef OPENVDB_USE_BLOSC - // pad to include BLOSC_MAX_OVERHEAD - size_t uncompressedBytes = openvdb::compression::bloscUncompressedSize(compressedBuffer.get()); - const size_t paddedCount = padMask(uncompressedBytes + BLOSC_MAX_OVERHEAD); - - mMask.reserve(paddedCount); - mMask.resize(count); - - // resize should never modify capacity for smaller vector sizes - OPENVDB_ASSERT(mMask.capacity() >= paddedCount); - - compression::bloscDecompress(reinterpret_cast(mMask.data()), count*sizeof(MaskType), mMask.capacity()*sizeof(MaskType), compressedBuffer.get()); -#endif - } else { - mMask.resize(count); - is.read(reinterpret_cast(mMask.data()), count*sizeof(MaskType)); - total += count*sizeof(MaskType); - } - - is.read(reinterpret_cast(&bytes), sizeof(Index32)); - - if (bytes != std::numeric_limits::max()) { - if (bytes > Index32(0)) { - std::unique_ptr compressedBuffer(new char[bytes]); - is.read(reinterpret_cast(compressedBuffer.get()), bytes); - - total += size_t(bytes); - -#ifdef OPENVDB_USE_BLOSC - // pad to include BLOSC_MAX_OVERHEAD - size_t uncompressedBytes = openvdb::compression::bloscUncompressedSize(compressedBuffer.get()); - const size_t paddedCount = padCompressedSize(uncompressedBytes + BLOSC_MAX_OVERHEAD); - - mCompressedSize.reserve(paddedCount); - mCompressedSize.resize(count); - - // resize should never modify capacity for smaller vector sizes - OPENVDB_ASSERT(mCompressedSize.capacity() >= paddedCount); - - compression::bloscDecompress(reinterpret_cast(mCompressedSize.data()), count*sizeof(CompressedSizeType), mCompressedSize.capacity()*sizeof(CompressedSizeType), compressedBuffer.get()); -#endif - } else { - mCompressedSize.resize(count); - is.read(reinterpret_cast(mCompressedSize.data()), count*sizeof(CompressedSizeType)); - total += count*sizeof(CompressedSizeType); - } - } - - Index32 totalBytes = static_cast(total); - - if (totalBytes < numBytes) { - // Read and discard any unknown bytes at the end of the metadata for forwards-compatibility - // (without seeking, because the stream might not be seekable). - const size_t BUFFER_SIZE = 1024; - std::vector buffer(BUFFER_SIZE); - for (Index32 bytesRemaining = numBytes - totalBytes; bytesRemaining > 0; ) { - const Index32 bytesToSkip = std::min(bytesRemaining, BUFFER_SIZE); - is.read(&buffer[0], bytesToSkip); - bytesRemaining -= bytesToSkip; - } - } -} - -void DelayedLoadMetadata::writeValue(std::ostream& os) const -{ - // metadata has a limit of 2^32 bytes - OPENVDB_ASSERT(mMask.size() < std::numeric_limits::max()); - OPENVDB_ASSERT(mCompressedSize.size() < std::numeric_limits::max()); - - if (mMask.empty() && mCompressedSize.empty()) return; - - OPENVDB_ASSERT(mCompressedSize.empty() || (mMask.size() == mCompressedSize.size())); - - Index32 count = static_cast(mMask.size()); - os.write(reinterpret_cast(&count), sizeof(Index32)); - - const Index32 zeroSize(0); - const Index32 maxSize(std::numeric_limits::max()); - - { // mask buffer - size_t compressedBytes(0); - std::unique_ptr compressedBuffer; - if (compression::bloscCanCompress()) { - compressedBuffer = compression::bloscCompress( - reinterpret_cast(mMask.data()), - mMask.size()*sizeof(MaskType), compressedBytes, /*resize=*/false); - } - - if (compressedBuffer) { - OPENVDB_ASSERT(compressedBytes < std::numeric_limits::max()); - Index32 bytes(static_cast(compressedBytes)); - os.write(reinterpret_cast(&bytes), sizeof(Index32)); - os.write(reinterpret_cast(compressedBuffer.get()), compressedBytes); - } - else { - os.write(reinterpret_cast(&zeroSize), sizeof(Index32)); - os.write(reinterpret_cast(mMask.data()), - mMask.size()*sizeof(MaskType)); - } - } - - // compressed size buffer - - if (mCompressedSize.empty()) { - // write out maximum Index32 value to denote no compressed sizes stored - os.write(reinterpret_cast(&maxSize), sizeof(Index32)); - } else { - size_t compressedBytes(0); - std::unique_ptr compressedBuffer; - if (compression::bloscCanCompress()) { - compressedBuffer = compression::bloscCompress( - reinterpret_cast(mCompressedSize.data()), - mCompressedSize.size()*sizeof(CompressedSizeType), compressedBytes, /*resize=*/false); - } - - if (compressedBuffer) { - OPENVDB_ASSERT(compressedBytes < std::numeric_limits::max()); - Index32 bytes(static_cast(compressedBytes)); - os.write(reinterpret_cast(&bytes), sizeof(Index32)); - os.write(reinterpret_cast(compressedBuffer.get()), compressedBytes); - } - else { - os.write(reinterpret_cast(&zeroSize), sizeof(Index32)); - os.write(reinterpret_cast(mCompressedSize.data()), - mCompressedSize.size()*sizeof(CompressedSizeType)); - } - } -} - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb diff --git a/openvdb/openvdb/io/DelayedLoadMetadata.h b/openvdb/openvdb/io/DelayedLoadMetadata.h deleted file mode 100644 index 8ba9d6f519..0000000000 --- a/openvdb/openvdb/io/DelayedLoadMetadata.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED -#define OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED - -#include -#include -#include -#include -#include - - -namespace openvdb { -OPENVDB_USE_VERSION_NAMESPACE -namespace OPENVDB_VERSION_NAME { -namespace io { - -/// @brief Store a buffer of data that can be optionally used -/// during reading for faster delayed-load I/O performance -class OPENVDB_API DelayedLoadMetadata: public Metadata -{ -public: - using Ptr = SharedPtr; - using ConstPtr = SharedPtr; - using MaskType = int8_t; - using CompressedSizeType = int64_t; - - DelayedLoadMetadata() = default; - DelayedLoadMetadata(const DelayedLoadMetadata& other); - ~DelayedLoadMetadata() override = default; - - Name typeName() const override; - Metadata::Ptr copy() const override; - void copy(const Metadata&) override; - std::string str() const override; - bool asBool() const override; - Index32 size() const override; - - static Name staticTypeName() { return "__delayedload"; } - - static Metadata::Ptr createMetadata() - { - Metadata::Ptr ret(new DelayedLoadMetadata); - return ret; - } - - static void registerType() - { - Metadata::registerType(DelayedLoadMetadata::staticTypeName(), - DelayedLoadMetadata::createMetadata); - } - - static void unregisterType() - { - Metadata::unregisterType(DelayedLoadMetadata::staticTypeName()); - } - - static bool isRegisteredType() - { - return Metadata::isRegisteredType(DelayedLoadMetadata::staticTypeName()); - } - - /// @brief Delete the contents of the mask and compressed size arrays - void clear(); - /// @brief Return @c true if both arrays are empty - bool empty() const; - - /// @brief Resize the mask array - void resizeMask(size_t size); - /// @brief Resize the compressed size array - void resizeCompressedSize(size_t size); - - /// @brief Return the mask value for a specific index - /// @note throws if index is out-of-range or DelayedLoadMask not registered - MaskType getMask(size_t index) const; - /// @brief Set the mask value for a specific index - /// @note throws if index is out-of-range - void setMask(size_t index, const MaskType& value); - - /// @brief Return the compressed size value for a specific index - /// @note throws if index is out-of-range or DelayedLoadMask not registered - CompressedSizeType getCompressedSize(size_t index) const; - /// @brief Set the compressed size value for a specific index - /// @note throws if index is out-of-range - void setCompressedSize(size_t index, const CompressedSizeType& value); - -protected: - void readValue(std::istream&, Index32 numBytes) override; - void writeValue(std::ostream&) const override; - -private: - std::vector mMask; - std::vector mCompressedSize; -}; // class DelayedLoadMetadata - - -} // namespace io -} // namespace OPENVDB_VERSION_NAME -} // namespace openvdb - -#endif // OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index dfb0c797e1..218dcbb7e9 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -727,7 +727,7 @@ File::readGridPartial(GridBase::Ptr grid, std::istream& is, readGridCompression(is); grid->readMeta(is); - // drop DelayedLoadMetadata from the grid as it is only useful for IO + // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); } diff --git a/openvdb/openvdb/openvdb.cc b/openvdb/openvdb/openvdb.cc index d4691c36a7..f830054849 100644 --- a/openvdb/openvdb/openvdb.cc +++ b/openvdb/openvdb/openvdb.cc @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 #include "openvdb.h" -#include "io/DelayedLoadMetadata.h" #include "points/PointDataGrid.h" #include "tools/PointIndexGrid.h" #include "util/logging.h" diff --git a/openvdb/openvdb/openvdb.h b/openvdb/openvdb/openvdb.h index a4db4b7337..8ce06d4167 100644 --- a/openvdb/openvdb/openvdb.h +++ b/openvdb/openvdb/openvdb.h @@ -46,9 +46,6 @@ OPENVDB_API void initialize(); OPENVDB_API void uninitialize(); -// foward declare some default types -namespace io { class DelayedLoadMetadata; } - /// Common tree types using BoolTree = tree::Tree4::Type; using DoubleTree = tree::Tree4::Type; @@ -222,8 +219,7 @@ using MetaTypes = TypeList< Vec4SMetadata, Vec4DMetadata, Mat4SMetadata, - Mat4DMetadata, - io::DelayedLoadMetadata>; + Mat4DMetadata>; } // namespace OPENVDB_VERSION_NAME diff --git a/openvdb/openvdb/unittest/CMakeLists.txt b/openvdb/openvdb/unittest/CMakeLists.txt index 3878f95b65..990849d809 100644 --- a/openvdb/openvdb/unittest/CMakeLists.txt +++ b/openvdb/openvdb/unittest/CMakeLists.txt @@ -87,7 +87,6 @@ else() TestCount.cc TestCpt.cc TestCurl.cc - TestDelayedLoadMetadata.cc TestDense.cc TestDenseSparseTools.cc TestDiagnostics.cc diff --git a/openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc b/openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc deleted file mode 100644 index d1617863e8..0000000000 --- a/openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright Contributors to the OpenVDB Project -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include - -#include - -class TestDelayedLoadMetadata : public ::testing::Test -{ -}; - - -TEST_F(TestDelayedLoadMetadata, test) -{ - using namespace openvdb::io; - - // registration - - EXPECT_TRUE(!DelayedLoadMetadata::isRegisteredType()); - - DelayedLoadMetadata::registerType(); - - EXPECT_TRUE(DelayedLoadMetadata::isRegisteredType()); - - DelayedLoadMetadata::unregisterType(); - - EXPECT_TRUE(!DelayedLoadMetadata::isRegisteredType()); - - openvdb::initialize(); - - EXPECT_TRUE(DelayedLoadMetadata::isRegisteredType()); - - // construction - - DelayedLoadMetadata metadata; - - EXPECT_TRUE(metadata.empty()); - - metadata.resizeMask(size_t(2)); - - EXPECT_TRUE(!metadata.empty()); - - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - metadata.setMask(1, DelayedLoadMetadata::MaskType(-3)); - - EXPECT_EQ(metadata.getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(metadata.getMask(1), DelayedLoadMetadata::MaskType(-3)); - - metadata.resizeCompressedSize(size_t(3)); - - metadata.setCompressedSize(0, DelayedLoadMetadata::CompressedSizeType(6)); - metadata.setCompressedSize(1, DelayedLoadMetadata::CompressedSizeType(101)); - metadata.setCompressedSize(2, DelayedLoadMetadata::CompressedSizeType(-13522)); - - EXPECT_EQ(metadata.getCompressedSize(0), DelayedLoadMetadata::CompressedSizeType(6)); - EXPECT_EQ(metadata.getCompressedSize(1), DelayedLoadMetadata::CompressedSizeType(101)); - EXPECT_EQ(metadata.getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522)); - - // copy construction - - DelayedLoadMetadata metadataCopy1(metadata); - - EXPECT_TRUE(!metadataCopy1.empty()); - - EXPECT_EQ(metadataCopy1.getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(metadataCopy1.getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522)); - - openvdb::Metadata::Ptr baseMetadataCopy2 = metadata.copy(); - DelayedLoadMetadata::Ptr metadataCopy2 = - openvdb::StaticPtrCast(baseMetadataCopy2); - - EXPECT_EQ(metadataCopy2->getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(metadataCopy2->getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522)); - - // I/O - - metadata.clear(); - EXPECT_TRUE(metadata.empty()); - - const size_t headerInitialSize(sizeof(openvdb::Index32)); - const size_t headerCountSize(sizeof(openvdb::Index32)); - const size_t headerMaskSize(sizeof(openvdb::Index32)); - const size_t headerCompressedSize(sizeof(openvdb::Index32)); - const size_t headerTotalSize(headerInitialSize + headerCountSize + headerMaskSize + headerCompressedSize); - - { // empty buffer - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - EXPECT_EQ(ss.tellp(), std::streampos(headerInitialSize)); - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_TRUE(newMetadata.empty()); - } - - { // single value, no compressed sizes - metadata.clear(); - metadata.resizeMask(size_t(1)); - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - std::streampos expectedPos(headerTotalSize + sizeof(int8_t)); - EXPECT_EQ(ss.tellp(), expectedPos); - EXPECT_EQ(static_cast(expectedPos)-headerInitialSize, size_t(metadata.size())); - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_TRUE(!newMetadata.empty()); - EXPECT_EQ(newMetadata.getMask(0), DelayedLoadMetadata::MaskType(5)); - } - - { // single value, with compressed sizes - metadata.clear(); - metadata.resizeMask(size_t(1)); - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - - metadata.resizeCompressedSize(size_t(1)); - metadata.setCompressedSize(0, DelayedLoadMetadata::CompressedSizeType(-10322)); - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - std::streampos expectedPos(headerTotalSize + sizeof(int8_t) + sizeof(int64_t)); - - EXPECT_EQ(expectedPos, ss.tellp()); - EXPECT_EQ(static_cast(ss.tellp())-headerInitialSize, size_t(metadata.size())); - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_TRUE(!newMetadata.empty()); - EXPECT_EQ(newMetadata.getMask(0), DelayedLoadMetadata::MaskType(5)); - EXPECT_EQ(newMetadata.getCompressedSize(0), DelayedLoadMetadata::CompressedSizeType(-10322)); - } - - { // larger, but compressible buffer - metadata.clear(); - - const size_t size = 1000; - - const size_t uncompressedBufferSize = (sizeof(int8_t)+sizeof(int64_t))*size; - - metadata.resizeMask(size); - metadata.resizeCompressedSize(size); - for (size_t i = 0; i < size; i++) { - metadata.setMask(i, - DelayedLoadMetadata::MaskType(static_cast((i%32)*2))); - metadata.setCompressedSize(i, - DelayedLoadMetadata::CompressedSizeType(static_cast((i%64)*200))); - } - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - metadata.write(ss); - - EXPECT_EQ(static_cast(ss.tellp())-headerInitialSize, size_t(metadata.size())); - - std::streampos uncompressedSize(uncompressedBufferSize + headerTotalSize); -#ifdef OPENVDB_USE_BLOSC - // expect a compression ratio of more than 10x - EXPECT_TRUE(ss.tellp() * 10 < uncompressedSize); -#else - EXPECT_TRUE(ss.tellp() == uncompressedSize); -#endif - - DelayedLoadMetadata newMetadata; - newMetadata.read(ss); - EXPECT_EQ(metadata.size(), newMetadata.size()); - for (size_t i = 0; i < size; i++) { - EXPECT_EQ(metadata.getMask(i), newMetadata.getMask(i)); - } - } - - // when read as unknown metadata should be treated as temporary metadata - - { - metadata.clear(); - metadata.resizeMask(size_t(1)); - metadata.setMask(0, DelayedLoadMetadata::MaskType(5)); - - std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary); - - openvdb::MetaMap metamap; - metamap.insertMeta("delayload", metadata); - - EXPECT_EQ(size_t(1), metamap.metaCount()); - - metamap.writeMeta(ss); - - { - openvdb::MetaMap newMetamap; - newMetamap.readMeta(ss); - - EXPECT_EQ(size_t(1), newMetamap.metaCount()); - } - - { - DelayedLoadMetadata::unregisterType(); - - openvdb::MetaMap newMetamap; - newMetamap.readMeta(ss); - - EXPECT_EQ(size_t(0), newMetamap.metaCount()); - } - } -} diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 0d9c9a3e8a..40206d1beb 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -53,7 +53,6 @@ class TestFile: public ::testing::Test void testReadGridDescriptors(); void testEmptyGridIO(); void testOpen(); - void testDelayedLoadMetadata(); void testNonVdbOpen(); }; @@ -190,10 +189,8 @@ TestFile::testWriteGrid() gd2_grid->readTransform(istr); gd2_grid->readTopology(istr); - // Remove delay load metadata if it exists. - if ((*gd2_grid)["file_delayed_load"]) { - gd2_grid->removeMeta("file_delayed_load"); - } + // Delay load metadata should not exist. + ASSERT_FALSE(bool((*gd2_grid)["file_delayed_load"])); // Ensure that we have the same metadata. EXPECT_EQ(grid->metaCount(), gd2_grid->metaCount()); @@ -1450,10 +1447,8 @@ TEST_F(TestFile, testReadGridMetadata) if ((*statsMetadata)[it->first]) { otherMetadata->removeMeta(it->first); } - // Remove delay load metadata if it exists. - if ((*otherMetadata)["file_delayed_load"]) { - otherMetadata->removeMeta("file_delayed_load"); - } + // Delay load metadata should not exist. + ASSERT_FALSE(bool((*otherMetadata)["file_delayed_load"])); } EXPECT_EQ(srcGrid->str(), otherMetadata->str()); @@ -2530,147 +2525,3 @@ TEST_F(TestFile, testBlosc) } } #endif - - -void -TestFile::testDelayedLoadMetadata() -{ - openvdb::initialize(); - - using namespace openvdb; - - io::File file("something.vdb2"); - - // Create a level set grid. - auto lsGrid = createLevelSet(); - lsGrid->setName("sphere"); - unittest_util::makeSphere(/*dim=*/Coord(100), /*ctr=*/Vec3f(50, 50, 50), /*r=*/20.0, - *lsGrid, unittest_util::SPHERE_SPARSE_NARROW_BAND); - - // Write the VDB to a string stream. - std::ostringstream ostr(std::ios_base::binary); - - // Create the grid descriptor out of this grid. - io::GridDescriptor gd(Name("sphere"), lsGrid->type()); - - // Write out the grid. - file.writeGrid(gd, lsGrid, ostr, /*seekable=*/true); - - // Duplicate VDB string stream. - std::ostringstream ostr2(std::ios_base::binary); - - { // Read back in, clip and write out again to verify metadata is rebuilt. - std::istringstream istr(ostr.str(), std::ios_base::binary); - io::setVersion(istr, file.libraryVersion(), file.fileVersion()); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - - const BBoxd clipBbox(Vec3d(-10.0,-10.0,-10.0), Vec3d(10.0,10.0,10.0)); - io::Archive::readGrid(grid, gd2, istr, clipBbox); - - // Verify clipping is working as expected. - EXPECT_TRUE(grid->baseTreePtr()->leafCount() < lsGrid->tree().leafCount()); - - file.writeGrid(gd, grid, ostr2, /*seekable=*/true); - } - - // Since the input is only a fragment of a VDB file (in particular, - // it doesn't have a header), set the file format version number explicitly. - // On read, the delayed load metadata for OpenVDB library versions less than 6.1 - // should be removed to ensure correctness as it possible for the metadata to - // have been treated as unknown and blindly copied over when read and re-written - // using this library version resulting in out-of-sync metadata. - - // By default, DelayedLoadMetadata is dropped from the grid during read so - // as not to be exposed to the user. - - { // read using current library version - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, file.libraryVersion(), file.fileVersion()); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - // To test the version mechanism, a stream metadata object is created with - // a non-zero test value and set on the input stream. This disables the - // behaviour where the DelayedLoadMetadata is dropped from the grid. - - io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); - streamMetadata->__setTest(uint32_t(1)); - - { // read using current library version - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, file.libraryVersion(), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 5.0 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(5,0), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 4.9 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(4,9), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 6.1 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(6,1), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(!((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - { // read using library version of 6.2 - std::istringstream istr(ostr2.str(), std::ios_base::binary); - io::setVersion(istr, VersionId(6,2), file.fileVersion()); - io::setStreamMetadataPtr(istr, streamMetadata, /*transfer=*/false); - - io::GridDescriptor gd2; - GridBase::Ptr grid = gd2.read(istr); - gd2.seekToGrid(istr); - io::Archive::readGrid(grid, gd2, istr); - - EXPECT_TRUE(((*grid)[GridBase::META_FILE_DELAYED_LOAD])); - } - - remove("something.vdb2"); -} -TEST_F(TestFile, testDelayedLoadMetadata) { testDelayedLoadMetadata(); } - diff --git a/openvdb/openvdb/unittest/TestStream.cc b/openvdb/openvdb/unittest/TestStream.cc index cc9bde5b75..8183317ea9 100644 --- a/openvdb/openvdb/unittest/TestStream.cc +++ b/openvdb/openvdb/unittest/TestStream.cc @@ -45,7 +45,6 @@ TestStream::SetUp() openvdb::Int32Metadata::registerType(); openvdb::Int64Metadata::registerType(); openvdb::Vec3IMetadata::registerType(); - openvdb::io::DelayedLoadMetadata::registerType(); // Register maps openvdb::math::MapRegistry::clear(); From d6074750e52c3689e7c42d0732d331585497273a Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 30 Jan 2026 14:15:03 -0800 Subject: [PATCH 03/80] File::open() and Stream() deprecations Signed-off-by: Dan Bailey --- .../nanovdb/cmd/convert/nanovdb_convert.cc | 2 +- .../coarsen_nanovdb_cuda.cpp | 2 +- .../dilate_nanovdb_cuda.cpp | 2 +- .../merge_nanovdb_cuda.cpp | 4 +- .../examples/ex_raytrace_level_set/openvdb.cc | 2 +- .../refine_nanovdb_cuda.cpp | 2 +- nanovdb/nanovdb/unittest/TestOpenVDB.cc | 8 +-- openvdb/openvdb/io/File.h | 10 +++ openvdb/openvdb/io/Stream.cc | 71 +++++++++---------- openvdb/openvdb/io/Stream.h | 10 +-- openvdb/openvdb/io/io.h | 42 +++++++++++ openvdb/openvdb/unittest/TestFastSweeping.cc | 6 +- openvdb/openvdb/unittest/TestMultiResGrid.cc | 2 +- openvdb/openvdb/unittest/TestTools.cc | 4 +- openvdb_cmd/vdb_render/main.cc | 2 +- openvdb_cmd/vdb_tool/include/Geometry.h | 2 +- openvdb_cmd/vdb_tool/include/Tool.h | 5 +- .../openvdb_houdini/GEO_VDBTranslator.cc | 4 +- .../openvdb_houdini/reference/GEO_PrimVDB.cc | 4 +- 19 files changed, 116 insertions(+), 68 deletions(-) diff --git a/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc b/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc index 2298508a93..bab45dccd6 100644 --- a/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc +++ b/nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc @@ -257,7 +257,7 @@ int main(int argc, char* argv[]) } if (verbose) std::cout << "Opening OpenVDB file named \"" << inputFile << "\"" << std::endl; openvdb::io::File file(inputFile); - file.open(false); //disable delayed loading + file.open(); if (gridName.empty()) {// convert all grid in the file auto grids = file.getGrids(); std::vector > handles; diff --git a/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp index 8826a3000c..7154e02740 100644 --- a/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp @@ -50,7 +50,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read input VDB file"); openvdb::initialize(); openvdb::io::File inFile(argv[1]); - inFile.open(false); // disable delayed loading + inFile.open(); auto baseGrids = inFile.getGrids(); inFile.close(); auto grid = openvdb::gridPtrCast(baseGrids->at(0)); diff --git a/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp index 6f6000f03c..6e65006934 100644 --- a/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp @@ -44,7 +44,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read input VDB file"); openvdb::initialize(); openvdb::io::File inFile(argv[1]); - inFile.open(false); // disable delayed loading + inFile.open(); auto baseGrids = inFile.getGrids(); inFile.close(); auto grid = openvdb::gridPtrCast(baseGrids->at(0)); diff --git a/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp index 8b6626558b..e8cba39e02 100644 --- a/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp @@ -41,7 +41,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read first input VDB file"); openvdb::initialize(); openvdb::io::File inFile1(argv[1]); - inFile1.open(false); // disable delayed loading + inFile1.open(); auto baseGrids1 = inFile1.getGrids(); inFile1.close(); auto grid1 = openvdb::gridPtrCast(baseGrids1->at(0)); @@ -51,7 +51,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read second input VDB file"); openvdb::initialize(); openvdb::io::File inFile2(argv[2]); - inFile2.open(false); // disable delayed loading + inFile2.open(); auto baseGrids2 = inFile2.getGrids(); inFile2.close(); auto grid2 = openvdb::gridPtrCast(baseGrids2->at(0)); diff --git a/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc b/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc index b8ca62c838..7b8f12256e 100644 --- a/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc +++ b/nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc @@ -37,7 +37,7 @@ void runOpenVDB(nanovdb::GridHandle& handle, int nu openvdb::initialize(); std::string filename = "C:/Users/william/Downloads/dragon.vdb"; openvdb::io::File file(filename); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = file.readGrid("ls_dragon"); std::cout << "Loading OpenVDB grid[" << srcGrid->getName() << "]...\n"; #endif diff --git a/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp b/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp index 19795c5dfd..f86980a72b 100644 --- a/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp +++ b/nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp @@ -81,7 +81,7 @@ int main(int argc, char *argv[]) cpuTimer.start("Read input VDB file"); openvdb::initialize(); openvdb::io::File inFile(argv[1]); - inFile.open(false); // disable delayed loading + inFile.open(); auto baseGrids = inFile.getGrids(); inFile.close(); auto grid = openvdb::gridPtrCast(baseGrids->at(0)); diff --git a/nanovdb/nanovdb/unittest/TestOpenVDB.cc b/nanovdb/nanovdb/unittest/TestOpenVDB.cc index fd47a7e017..8ada169df9 100644 --- a/nanovdb/nanovdb/unittest/TestOpenVDB.cc +++ b/nanovdb/nanovdb/unittest/TestOpenVDB.cc @@ -140,7 +140,7 @@ class TestOpenVDB : public ::testing::Test mTimer.start("Reading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); grid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); } catch(const std::exception& e) { std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl; @@ -2860,7 +2860,7 @@ TEST_F(TestOpenVDB, LevelSetFiles) //mTimer.start("\nReading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); const size_t pos = fileName.find_last_of("/\\") + 1; @@ -2938,7 +2938,7 @@ TEST_F(TestOpenVDB, FogFiles) //mTimer.start("Reading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); const size_t pos = fileName.find_last_of("/\\") + 1; @@ -3008,7 +3008,7 @@ TEST_F(TestOpenVDB, PointFiles) //mTimer.start("Reading grid from the file \"" + fileName + "\""); try { openvdb::io::File file(fileName); - file.open(false); //disable delayed loading + file.open(); auto srcGrid = openvdb::gridPtrCast(file.readGrid(file.beginName().gridName())); //std::cerr << "Read PointDataGrid named \"" << srcGrid->getName() << "\"" << std::endl; diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index dbde5b6d51..be8ab89968 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -60,6 +60,11 @@ class OPENVDB_API File: public Archive /// @return @c true if the file's UUID has changed since it was last read. bool open(); + OPENVDB_DEPRECATED_MESSAGE("Use File::open() instead.This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool open(bool /*delayLoad*/) { return open(); } + OPENVDB_DEPRECATED_MESSAGE("Use File::open() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool open(bool /*delayLoad*/, const MappedFile::Notifier& /*notifier*/) { return open(); } + /// Return @c true if the file has been opened for reading. bool isOpen() const; @@ -70,6 +75,11 @@ class OPENVDB_API File: public Archive /// @throw IoError if the file size cannot be determined. Index64 getSize() const; + OPENVDB_DEPRECATED_MESSAGE("Always returns 0. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 copyMaxBytes() const { return 0; } + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void setCopyMaxBytes(Index64 /*bytes*/) { } + /// Return @c true if a grid of the given name exists in this file. bool hasGrid(const Name&) const; diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 4ba3fce816..e3ff05c2ed 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -44,47 +44,44 @@ struct Stream::Impl //////////////////////////////////////// -Stream::Stream(std::istream& is, bool delayLoad): mImpl(new Impl) +Stream::Stream(std::istream& is): mImpl(new Impl) { if (!is) return; - (void) delayLoad; - - if (!mImpl->mFile) { - readHeader(is); - - // Tag the input stream with the library and file format version numbers - // and the compression options specified in the header. - StreamMetadata::Ptr streamMetadata(new StreamMetadata); - io::setStreamMetadataPtr(is, streamMetadata, /*transfer=*/false); - io::setVersion(is, libraryVersion(), fileVersion()); - io::setDataCompression(is, compression()); - - // Read in the VDB metadata. - mImpl->mMeta.reset(new MetaMap); - mImpl->mMeta->readMeta(is); - - // Read in the number of grids. - const int32_t gridCount = readGridCount(is); - - // Read in all grids and insert them into mGrids. - mImpl->mGrids.reset(new GridPtrVec); - std::vector descriptors; - descriptors.reserve(gridCount); - Archive::NamedGridMap namedGrids; - for (int32_t i = 0; i < gridCount; ++i) { - GridDescriptor gd; - gd.read(is); - descriptors.push_back(gd); - GridBase::Ptr grid = readGrid(gd, is); - mImpl->mGrids->push_back(grid); - namedGrids[gd.uniqueName()] = grid; - } + // Delayed loading has been removed - always read directly from the stream + readHeader(is); + + // Tag the input stream with the library and file format version numbers + // and the compression options specified in the header. + StreamMetadata::Ptr streamMetadata(new StreamMetadata); + io::setStreamMetadataPtr(is, streamMetadata, /*transfer=*/false); + io::setVersion(is, libraryVersion(), fileVersion()); + io::setDataCompression(is, compression()); + + // Read in the VDB metadata. + mImpl->mMeta.reset(new MetaMap); + mImpl->mMeta->readMeta(is); + + // Read in the number of grids. + const int32_t gridCount = readGridCount(is); + + // Read in all grids and insert them into mGrids. + mImpl->mGrids.reset(new GridPtrVec); + std::vector descriptors; + descriptors.reserve(gridCount); + Archive::NamedGridMap namedGrids; + for (int32_t i = 0; i < gridCount; ++i) { + GridDescriptor gd; + gd.read(is); + descriptors.push_back(gd); + GridBase::Ptr grid = readGrid(gd, is); + mImpl->mGrids->push_back(grid); + namedGrids[gd.uniqueName()] = grid; + } - // Connect instances (grids that share trees with other grids). - for (size_t i = 0, N = descriptors.size(); i < N; ++i) { - Archive::connectInstance(descriptors[i], namedGrids); - } + // Connect instances (grids that share trees with other grids). + for (size_t i = 0, N = descriptors.size(); i < N; ++i) { + Archive::connectInstance(descriptors[i], namedGrids); } } diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index e2978dc20d..480f5bf074 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -22,11 +22,11 @@ class OPENVDB_API Stream: public Archive { public: /// @brief Read grids from an input stream. - /// @details If @a delayLoad is true, map the contents of the input stream - /// into memory and enable delayed loading of grids. - /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD - /// to disable delayed loading unconditionally. - explicit Stream(std::istream&, bool delayLoad = true); + /// @param is The input stream to read from + explicit Stream(std::istream&); + + OPENVDB_DEPRECATED_MESSAGE("Use Stream(std::istream&) instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Stream(std::istream& is, bool /*delayLoad*/) : Stream(is) { } /// Construct an archive for stream output. Stream(); diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index 40bce111d5..cd1c460aaf 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -127,6 +127,43 @@ struct MultiPass {}; //////////////////////////////////////// +class File; + + +// This class is deprecated and will be removed. Delayed loading is no longer supported. +// It is retained for API backwards compatibility only, but all methods are no-ops. +class OPENVDB_API MappedFile +{ +public: + using Ptr = SharedPtr; + using Notifier = std::function; + + OPENVDB_DEPRECATED_MESSAGE("This class is deprecated and will be removed. Delayed loading is no longer supported.") + explicit MappedFile(const std::string& /*filename*/, bool /*autoDelete*/ = false) {} + + /// @brief Destructor + ~MappedFile() = default; + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + + OPENVDB_DEPRECATED_MESSAGE("Always returns an empty string. This method is deprecated and will be removed. Delayed loading is no longer supported.") + std::string filename() const { return std::string(); } + + OPENVDB_DEPRECATED_MESSAGE("Always returns a null stream buffer pointer. This method is deprecated and will be removed. Delayed loading is no longer supported.") + SharedPtr createBuffer() const { return SharedPtr(); } + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void setNotifier(const Notifier&) {} + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void clearNotifier() {} +}; // class MappedFile + + +//////////////////////////////////////// + + /// Return a string (possibly empty) describing the given system error code. std::string getErrorString(int errorNum); @@ -199,6 +236,11 @@ OPENVDB_API bool getWriteGridStatsMetadata(std::ios_base&); /// and store them as grid metadata when writing to the given stream. OPENVDB_API void setWriteGridStatsMetadata(std::ios_base&, bool writeGridStats); +OPENVDB_DEPRECATED_MESSAGE("Always returns a null pointer. This function is deprecated and will be removed. Delayed loading is no longer supported.") +inline SharedPtr getMappedFilePtr(std::ios_base&) { return SharedPtr(); } +OPENVDB_DEPRECATED_MESSAGE("This function is deprecated and will be removed. Delayed loading is no longer supported.") +inline void setMappedFilePtr(std::ios_base&, SharedPtr&) { } + /// @brief Return a shared pointer to an object that stores metadata (file format, /// compression scheme, etc.) for use when reading from or writing to the given stream. OPENVDB_API SharedPtr getStreamMetadataPtr(std::ios_base&); diff --git a/openvdb/openvdb/unittest/TestFastSweeping.cc b/openvdb/openvdb/unittest/TestFastSweeping.cc index 9bddbf7fbf..226aa35cd8 100644 --- a/openvdb/openvdb/unittest/TestFastSweeping.cc +++ b/openvdb/openvdb/unittest/TestFastSweeping.cc @@ -252,7 +252,7 @@ TEST_F(TestFastSweeping, testMaskSdf) openvdb::initialize();//required whenever I/O of OpenVDB files is performed! const std::string path(TestFastSweeping_DATA_PATH); io::File file( path + "bunny.vdb" ); - file.open(false);//disable delayed loading + file.open(); FloatGrid::Ptr mask = openvdb::gridPtrCast(file.getGrids()->at(0)); //this->writeFile("/tmp/bunny_mask_input.vdb", grid); @@ -595,7 +595,7 @@ TEST_F(TestFastSweeping, velocityExtensionOfFogBunny) openvdb::initialize();//required whenever I/O of OpenVDB files is performed! const std::string path(TestFastSweeping_DATA_PATH); io::File file( path + "bunny.vdb" ); - file.open(false);//disable delayed loading + file.open(); auto grid = openvdb::gridPtrCast(file.getGrids()->at(0)); tools::sdfToFogVolume(*grid); writeFile("/tmp/bunny1_fog_in.vdb", grid); @@ -619,7 +619,7 @@ TEST_F(TestFastSweeping, velocityExtensionOfSdfBunny) using namespace openvdb; const std::string path(TestFastSweeping_DATA_PATH); io::File file( path + "bunny.vdb" ); - file.open(false);//disable delayed loading + file.open(); auto grid = openvdb::gridPtrCast(file.getGrids()->at(0)); writeFile("/tmp/bunny2_sdf_in.vdb", grid); auto bbox = grid->evalActiveVoxelBoundingBox(); diff --git a/openvdb/openvdb/unittest/TestMultiResGrid.cc b/openvdb/openvdb/unittest/TestMultiResGrid.cc index 32735d508f..d30cb0a28d 100644 --- a/openvdb/openvdb/unittest/TestMultiResGrid.cc +++ b/openvdb/openvdb/unittest/TestMultiResGrid.cc @@ -280,7 +280,7 @@ TEST_F(TestMultiResGrid, testModels) << "\" =====================" << std::endl; std::cerr << "Reading \"" << filenames[i] << "\" ..."; io::File file( path + filenames[i] ); - file.open(false);//disable delayed loading + file.open(); FloatGrid::Ptr model = gridPtrCast(file.getGrids()->at(0)); std::cerr << " done\nProcessing \"" << filenames[i] << "\" ..."; timer.start("\nMultiResGrid processing"); diff --git a/openvdb/openvdb/unittest/TestTools.cc b/openvdb/openvdb/unittest/TestTools.cc index df17050a06..4ffc1fccd1 100644 --- a/openvdb/openvdb/unittest/TestTools.cc +++ b/openvdb/openvdb/unittest/TestTools.cc @@ -1762,7 +1762,7 @@ TEST_F(TestTools, testPrune) util::CpuTimer timer; initialize();//required whenever I/O of OpenVDB files is performed! io::File sourceFile("/usr/pic1/Data/OpenVDB/LevelSetModels/crawler.vdb"); - sourceFile.open(false);//disable delayed loading + sourcefile.open(); FloatGrid::Ptr grid = gridPtrCast(sourceFile.getGrids()->at(0)); const Index64 leafCount = grid->tree().leafCount(); @@ -1775,7 +1775,7 @@ TEST_F(TestTools, testPrune) util::CpuTimer timer; initialize();//required whenever I/O of OpenVDB files is performed! io::File sourceFile("/usr/pic1/Data/OpenVDB/LevelSetModels/crawler.vdb"); - sourceFile.open(false);//disable delayed loading + sourcefile.open(); FloatGrid::Ptr grid = gridPtrCast(sourceFile.getGrids()->at(0)); const Index64 leafCount = grid->tree().leafCount(); diff --git a/openvdb_cmd/vdb_render/main.cc b/openvdb_cmd/vdb_render/main.cc index 127d8720b7..2281405978 100644 --- a/openvdb_cmd/vdb_render/main.cc +++ b/openvdb_cmd/vdb_render/main.cc @@ -768,7 +768,7 @@ main(int argc, char *argv[]) } } else { // If no grid was specified by name, retrieve the first float grid from the file. - file.open(/*delayLoad=*/false); + file.open(); openvdb::io::File::NameIterator it = file.beginName(); openvdb::GridPtrVecPtr grids = file.readAllGridMetadata(); for (size_t i = 0; i < grids->size(); ++i, ++it) { diff --git a/openvdb_cmd/vdb_tool/include/Geometry.h b/openvdb_cmd/vdb_tool/include/Geometry.h index ebd497b3bb..627bf6213e 100644 --- a/openvdb_cmd/vdb_tool/include/Geometry.h +++ b/openvdb_cmd/vdb_tool/include/Geometry.h @@ -824,7 +824,7 @@ void Geometry::readVDB(const std::string &fileName) { initialize(); io::File file(fileName); - file.open();// enables delayed loading by default + file.open(); GridPtrVecPtr meta = file.readAllGridMetadata(); for (auto m : *meta) { if (m->isType()) { diff --git a/openvdb_cmd/vdb_tool/include/Tool.h b/openvdb_cmd/vdb_tool/include/Tool.h index e558be1923..b6d5e3aaef 100644 --- a/openvdb_cmd/vdb_tool/include/Tool.h +++ b/openvdb_cmd/vdb_tool/include/Tool.h @@ -370,8 +370,7 @@ void Tool::init() mParser.addAction( "read", "i", "Read one or more geometry or VDB files from disk or STDIN.", {{"files", "", "{file|stdin}.{abc|obj|ply|stl|off|vdb}", "list of files or the input stream, e.g. file.vdb,stdin.vdb. Note that \"files=\" is optional since any argument without \"=\" is intrepreted as a file and appended to \"files\""}, - {"grids", "*", "*|grid_name,...", "list of VDB grids name to be imported (defaults to \"*\", i.e. import all available grids)"}, - {"delayed", "true", "1|0|true|false", "toggle delayed loading of VDB grids (enabled by default). This option is ignored by other file types"}}, + {"grids", "*", "*|grid_name,...", "list of VDB grids name to be imported (defaults to \"*\", i.e. import all available grids)"}}, [](){}, [&](){this->read();}, 0);// anonymous options are treated as to the first option,i.e. "files" mParser.addAction( @@ -1023,7 +1022,7 @@ void Tool::readVDB(const std::string &fileName) } else { if (mParser.verbose) mTimer.start("Reading VDB grid(s) from file named \""+fileName+"\""); io::File file(fileName); - file.open(mParser.get("delayed")); + file.open(); grids = file.getGrids(); } const size_t count = mGrid.size(); diff --git a/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc b/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc index 54802cd36c..38bc5e1b25 100644 --- a/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc +++ b/openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc @@ -108,7 +108,7 @@ GEO_VDBTranslator::fileStat(const char *filename, GA_Stat &stat, uint /*level*/) try { openvdb::io::File file(filename); - file.open(/*delayLoad*/false); + file.open(); int nprim = 0; UT_BoundingBox bbox; @@ -204,7 +204,7 @@ GEO_VDBTranslator::fileLoad(GEO_Detail *geogdp, UT_IStream &is, bool /*ate_magic try { // Create and open a VDB file, but don't read any grids yet. - openvdb::io::Stream file(*stdstream, /*delayLoad*/false); + openvdb::io::Stream file(*stdstream); // Read the file-level metadata into global attributes. openvdb::MetaMap::Ptr fileMetadata = file.getMetadata(); diff --git a/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc b/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc index 13df7c3fbf..76ef00eae8 100644 --- a/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc +++ b/openvdb_houdini/openvdb_houdini/reference/GEO_PrimVDB.cc @@ -3112,7 +3112,7 @@ GEO_PrimVDB::loadVDB(UT_JSONParser &p, bool as_shmem) try { SYS_SharedMemoryInputStream is_shm(*shmem); - openvdb::io::Stream vis(is_shm, /*delayLoad*/false); + openvdb::io::Stream vis(is_shm); openvdb::GridPtrVecPtr grids = vis.getGrids(); int count = (grids ? grids->size() : 0); @@ -3152,7 +3152,7 @@ GEO_PrimVDB::loadVDB(UT_JSONParser &p, bool as_shmem) { UT_JSONParser::TiledStream is(p); - openvdb::io::Stream vis(is, /*delayLoad*/false); + openvdb::io::Stream vis(is); openvdb::GridPtrVecPtr grids = vis.getGrids(); From 4c8551bc1ddc7e9e56a8d74d023aee54d0b7fd19 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 30 Jan 2026 14:16:16 -0800 Subject: [PATCH 04/80] Adapt all tooling to remove support for out-of-core data Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 26 +-- openvdb/openvdb/io/Archive.cc | 20 --- openvdb/openvdb/io/Archive.h | 8 +- openvdb/openvdb/io/File.cc | 14 +- openvdb/openvdb/io/Stream.cc | 7 +- openvdb/openvdb/io/io.h | 3 - openvdb/openvdb/points/AttributeArray.cc | 3 - openvdb/openvdb/points/AttributeArray.h | 158 ++---------------- openvdb/openvdb/points/AttributeGroup.cc | 8 - openvdb/openvdb/points/AttributeSet.cc | 11 -- openvdb/openvdb/points/AttributeSet.h | 6 +- openvdb/openvdb/points/PointConversion.h | 44 ++++- openvdb/openvdb/points/PointCount.h | 19 ++- openvdb/openvdb/points/PointDataGrid.h | 79 +-------- openvdb/openvdb/points/StreamCompression.cc | 20 --- openvdb/openvdb/points/StreamCompression.h | 12 +- .../openvdb/points/impl/PointConversionImpl.h | 86 ++++++---- openvdb/openvdb/points/impl/PointCountImpl.h | 31 +++- openvdb/openvdb/points/impl/PointMoveImpl.h | 4 - .../points/impl/PointRasterizeFrustumImpl.h | 2 +- .../impl/PrincipalComponentAnalysisImpl.h | 3 - openvdb/openvdb/tools/Count.h | 35 +--- openvdb/openvdb/tools/Merge.h | 8 +- openvdb/openvdb/tools/PointIndexGrid.h | 10 +- openvdb/openvdb/tools/VolumeToMesh.h | 4 +- openvdb/openvdb/tree/LeafBuffer.h | 79 ++------- openvdb/openvdb/tree/LeafNode.h | 26 +-- openvdb/openvdb/tree/LeafNodeBool.h | 12 +- openvdb/openvdb/tree/LeafNodeMask.h | 12 +- openvdb/openvdb/tree/Tree.h | 37 +--- openvdb/openvdb/unittest/TestCount.cc | 9 +- openvdb/openvdb/unittest/TestMeshToVolume.cc | 4 +- openvdb/openvdb/unittest/TestPointCount.cc | 15 +- .../unittest/TestPointRasterizeFrustum.cc | 1 - .../openvdb_ax/compiler/PointExecutable.cc | 4 +- openvdb_cmd/vdb_ax/main.cc | 4 +- openvdb_cmd/vdb_print/main.cc | 7 +- openvdb_cmd/vdb_tool/include/Tool.h | 2 +- .../openvdb_houdini/GR_PrimVDBPoints.cc | 27 +-- openvdb_houdini/openvdb_houdini/PointUtils.cc | 82 +++------ openvdb_houdini/openvdb_houdini/PointUtils.h | 4 +- .../SOP_OpenVDB_Points_Convert.cc | 13 -- .../SOP_OpenVDB_Rasterize_Frustum.cc | 68 +------- .../openvdb_houdini/SOP_OpenVDB_Read.cc | 57 ++----- .../openvdb_houdini/VRAY_OpenVDB_Points.cc | 16 +- 45 files changed, 275 insertions(+), 825 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index cb0a48e4e2..ecb18f9b66 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -457,12 +457,10 @@ class OPENVDB_API GridBase: public MetaMap virtual void readBuffers(std::istream&) = 0; /// Read all of this grid's data buffers that intersect the given index-space bounding box. virtual void readBuffers(std::istream&, const CoordBBox&) = 0; - /// @brief Read all of this grid's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this grid was read from a memory-mapped file, this operation - /// disconnects the grid from the file. - /// @sa io::File::open, io::MappedFile + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void readNonresidentBuffers() const = 0; + /// Write out all data buffers for this grid. virtual void writeBuffers(std::ostream&) const = 0; @@ -939,12 +937,10 @@ class Grid: public GridBase void readBuffers(std::istream&) override; /// Read all of this grid's data buffers that intersect the given index-space bounding box. void readBuffers(std::istream&, const CoordBBox&) override; - /// @brief Read all of this grid's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this grid was read from a memory-mapped file, this operation - /// disconnects the grid from the file. - /// @sa io::File::open, io::MappedFile - void readNonresidentBuffers() const override; + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void readNonresidentBuffers() const override { } + /// Write out all data buffers for this grid. void writeBuffers(std::ostream&) const override; @@ -1670,14 +1666,6 @@ Grid::readBuffers(std::istream& is, const CoordBBox& bbox) } -template -inline void -Grid::readNonresidentBuffers() const -{ - tree().readNonresidentBuffers(); -} - - template inline void Grid::writeBuffers(std::ostream& os) const diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 0e48fdae04..7ddd4f48b8 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -244,7 +244,6 @@ bool StreamMetadata::seekable() const { return mImpl->mSeekabl bool StreamMetadata::delayedLoadMeta() const { return mImpl->mDelayedLoadMeta; } bool StreamMetadata::countingPasses() const { return mImpl->mCountingPasses; } uint32_t StreamMetadata::pass() const { return mImpl->mPass; } -uint64_t StreamMetadata::leaf() const { return mImpl->mLeaf; } MetaMap& StreamMetadata::gridMetadata() { return mImpl->mGridMetadata; } const MetaMap& StreamMetadata::gridMetadata() const { return mImpl->mGridMetadata; } uint32_t StreamMetadata::__test() const { return mImpl->mTest; } @@ -262,7 +261,6 @@ void StreamMetadata::setWriteGridStats(bool b) { mImpl->mWriteGridStats void StreamMetadata::setSeekable(bool b) { mImpl->mSeekable = b; } void StreamMetadata::setCountingPasses(bool b) { mImpl->mCountingPasses = b; } void StreamMetadata::setPass(uint32_t i) { mImpl->mPass = i; } -void StreamMetadata::setLeaf(uint64_t i) { mImpl->mLeaf = i; } void StreamMetadata::__setTest(uint32_t t) { mImpl->mTest = t; } std::string @@ -888,15 +886,6 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co //////////////////////////////////////// -//static -bool -Archive::isDelayedLoadingEnabled() -{ - return false; -} - - - namespace { struct NoBBox {}; @@ -941,12 +930,6 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const grid->readMeta(is); - // Add a description of the compression settings to the grid as metadata. - /// @todo Would this be useful? - //const uint32_t c = getDataCompression(is); - //grid->insertMeta(GridBase::META_FILE_COMPRESSION, - // StringMetadata(compressionToString(c))); - // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); @@ -956,9 +939,6 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const const GridClass gridClass = grid->getGridClass(); io::setGridClass(is, gridClass); - // reset leaf value to zero - streamMetadata->setLeaf(0); - grid->readTransform(is); if (!gd.isInstance()) { grid->readTopology(is); diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index 4e4e680273..87f31b14eb 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -91,12 +91,8 @@ class OPENVDB_API Archive /// @brief Write the grids in the given container to this archive's output stream. virtual void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const {} - /// @brief Return @c true if delayed loading is enabled. - /// @details If enabled, delayed loading can be disabled for individual files, - /// but not vice-versa. - /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD - /// to disable delayed loading unconditionally. - static bool isDelayedLoadingEnabled(); + /// @brief Return @c false (delayed loading has been removed). + static bool isDelayedLoadingEnabled() { return false; } protected: /// @brief Return @c true if the input stream contains grid offsets diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 218dcbb7e9..4cbda52eb6 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -28,8 +28,6 @@ namespace io { // Implementation details of the File class struct File::Impl { - enum { DEFAULT_COPY_MAX_BYTES = 500000000 }; // 500 MB - struct NoBBox {}; // Common implementation of the various File::readGrid() overloads, @@ -64,16 +62,6 @@ struct File::Impl file.Archive::readGrid(grid, gd, file.inputStream(), worldBBox); } - static Index64 getDefaultCopyMaxBytes() - { - Index64 result = DEFAULT_COPY_MAX_BYTES; - if (const char* s = std::getenv("OPENVDB_DELAYED_LOAD_COPY_MAX_BYTES")) { - char* endptr = nullptr; - result = std::strtoul(s, &endptr, /*base=*/10); - } - return result; - } - std::string mFilename; // The file-level metadata MetaMap::Ptr mMeta; @@ -247,7 +235,7 @@ File::open() } mImpl->mInStream.reset(); - // Open the file. + // Open the file using standard I/O (delayed loading has been removed) std::unique_ptr newStream; newStream.reset(new std::ifstream( filename().c_str(), std::ios_base::in | std::ios_base::binary)); diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index e3ff05c2ed..be3a003515 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -170,9 +170,7 @@ MetaMap::Ptr Stream::getMetadata() const { MetaMap::Ptr result; - if (mImpl->mFile) { - result = mImpl->mFile->getMetadata(); - } else if (mImpl->mMeta) { + if (mImpl->mMeta) { // Return a deep copy of the file-level metadata // that was read when this object was constructed. result.reset(new MetaMap(*mImpl->mMeta)); @@ -184,9 +182,6 @@ Stream::getMetadata() const GridPtrVecPtr Stream::getGrids() { - if (mImpl->mFile) { - return mImpl->mFile->getGrids(); - } return mImpl->mGrids; } diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index cd1c460aaf..f2ac342ff7 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -76,9 +76,6 @@ class OPENVDB_API StreamMetadata uint32_t pass() const; void setPass(uint32_t); - uint64_t leaf() const; - void setLeaf(uint64_t); - //@{ /// @brief Return a (reference to a) copy of the metadata of the grid /// currently being read or written. diff --git a/openvdb/openvdb/points/AttributeArray.cc b/openvdb/openvdb/points/AttributeArray.cc index a24fc5a2bd..7d83051b52 100644 --- a/openvdb/openvdb/points/AttributeArray.cc +++ b/openvdb/openvdb/points/AttributeArray.cc @@ -190,9 +190,6 @@ AttributeArray::setConstantStride(bool state) bool AttributeArray::operator==(const AttributeArray& other) const { - this->loadData(); - other.loadData(); - if (this->mUsePagedRead != other.mUsePagedRead || this->mFlags != other.mFlags) return false; return this->isEqual(other); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index 46d0b02318..bbfba7b156 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -130,7 +130,7 @@ class OPENVDB_API AttributeArray template friend class AttributeHandle; - AttributeArray(): mPageHandle() { mOutOfCore = 0; } + AttributeArray(): mPageHandle() { } virtual ~AttributeArray() { // if this AttributeArray has been partially read, zero the compressed bytes, @@ -189,10 +189,7 @@ class OPENVDB_API AttributeArray /// Return the number of bytes of memory used by this attribute. virtual size_t memUsage() const = 0; - /// Return the number of bytes of memory used by this attribute array once it - /// has been deserialized (this may be different to memUsage() if delay-loading - /// is in use). Note that this method does NOT consider the fact that a - /// uniform attribute could be expanded and only deals with delay-loading. + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual size_t memUsageIfLoaded() const = 0; /// Create a new attribute array of the given (registered) type, length and stride. @@ -315,10 +312,10 @@ class OPENVDB_API AttributeArray /// @param outputTransient if true, write out transient attributes virtual void writePagedBuffers(compression::PagedOutputStream&, bool outputTransient) const = 0; - /// Ensures all data is in-core + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void loadData() const = 0; - /// Return @c true if all data has been loaded + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual bool isDataLoaded() const = 0; /// Check the compressed bytes and flags. If they are equal, perform a deeper @@ -366,7 +363,7 @@ class OPENVDB_API AttributeArray mutable tbb::spin_mutex mMutex; uint8_t mFlags = 0; uint8_t mUsePagedRead = 0; - std::atomic mOutOfCore; // interpreted as bool + std::atomic mOutOfCore{0}; // interpreted as bool /// used for out-of-core, paged reading union { compression::PageHandle::Ptr mPageHandle; @@ -611,11 +608,8 @@ class TypedAttributeArray final: public AttributeArray /// Return the number of bytes of memory used by this attribute. size_t memUsage() const override; - /// Return the number of bytes of memory used by this attribute array once it - /// has been deserialized (this may be different to memUsage() if delay-loading - /// is in use). Note that this method does NOT consider the fact that a - /// uniform attribute could be expanded and only deals with delay-loading. - size_t memUsageIfLoaded() const override; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + size_t memUsageIfLoaded() const override { return memUsage(); } /// Return the value at index @a n (assumes in-core) ValueType getUnsafe(Index n) const; @@ -696,14 +690,14 @@ class TypedAttributeArray final: public AttributeArray /// @param outputTransient if true, write out transient attributes void writePagedBuffers(compression::PagedOutputStream& os, bool outputTransient) const override; - /// Return @c true if this buffer's values have not yet been read from disk. - inline bool isOutOfCore() const; + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isOutOfCore() const { return false; } - /// Ensures all data is in-core - void loadData() const override; + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void loadData() const override { } - /// Return @c true if all data has been loaded - bool isDataLoaded() const override; + OPENVDB_DEPRECATED_MESSAGE("Always returns true. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isDataLoaded() const override { return true; } /// Return the raw data buffer inline const StorageType* constData() const { return this->data(); } @@ -716,18 +710,13 @@ class TypedAttributeArray final: public AttributeArray inline const StorageType* data() const { OPENVDB_ASSERT(validData()); return mData.get(); } /// Verify that data is not out-of-core or in a partially-read state - inline bool validData() const { return !(isOutOfCore() || (flags() & PARTIALREAD)); } + inline bool validData() const { return !((flags() & PARTIALREAD)); } private: friend class ::TestAttributeArray; TypedAttributeArray(const TypedAttributeArray&, const tbb::spin_mutex::scoped_lock&); - /// Load data from memory-mapped file. - inline void doLoad() const; - /// Load data from memory-mapped file (unsafe as this function is not protected by a mutex). - inline void doLoadUnsafe() const; - /// Toggle out-of-core state inline void setOutOfCore(const bool); @@ -962,8 +951,6 @@ void AttributeArray::doCopyValues(const AttributeArray& sourceArray, const IterT { // ensure both arrays have float-float or integer-integer value types OPENVDB_ASSERT(sourceArray.valueTypeIsFloatingPoint() == this->valueTypeIsFloatingPoint()); - // ensure both arrays have been loaded from disk (if delay-loaded) - OPENVDB_ASSERT(sourceArray.isDataLoaded() && this->isDataLoaded()); // ensure storage size * stride matches on both arrays OPENVDB_ASSERT(this->storageTypeSize()*this->stride() == sourceArray.storageTypeSize()*sourceArray.stride()); @@ -1024,10 +1011,6 @@ void AttributeArray::copyValues(const AttributeArray& sourceArray, const IterT& OPENVDB_THROW(TypeError, "Cannot copy array data due to mis-match in storage type sizes."); } - // ensure both arrays have been loaded from disk - sourceArray.loadData(); - this->loadData(); - // if the target array is uniform, expand it first this->expand(); @@ -1206,8 +1189,6 @@ template size_t TypedAttributeArray::arrayMemUsage() const { - if (this->isOutOfCore()) return 0; - return (mIsUniform ? 1 : this->dataSize()) * sizeof(StorageType); } @@ -1232,11 +1213,6 @@ template void TypedAttributeArray::deallocate() { - // detach from file if delay-loaded - if (this->isOutOfCore()) { - this->setOutOfCore(false); - this->mPageHandle.reset(); - } if (mData) mData.reset(); } @@ -1304,14 +1280,6 @@ TypedAttributeArray::memUsage() const } -template -size_t -TypedAttributeArray::memUsageIfLoaded() const -{ - return sizeof(*this) + (mIsUniform ? 1 : this->dataSize()) * sizeof(StorageType); -} - - template typename TypedAttributeArray::ValueType TypedAttributeArray::getUnsafe(Index n) const @@ -1329,7 +1297,6 @@ typename TypedAttributeArray::ValueType TypedAttributeArray::get(Index n) const { if (n >= this->dataSize()) OPENVDB_THROW(IndexError, "Out-of-range access."); - if (this->isOutOfCore()) this->doLoad(); return this->getUnsafe(n); } @@ -1366,7 +1333,6 @@ void TypedAttributeArray::setUnsafe(Index n, const ValueType& val) { OPENVDB_ASSERT(n < this->dataSize()); - OPENVDB_ASSERT(!this->isOutOfCore()); OPENVDB_ASSERT(!this->isUniform()); // this unsafe method assumes the data is not uniform, however if it is, this redirects the index @@ -1381,7 +1347,6 @@ void TypedAttributeArray::set(Index n, const ValueType& val) { if (n >= this->dataSize()) OPENVDB_THROW(IndexError, "Out-of-range access."); - if (this->isOutOfCore()) this->doLoad(); if (this->isUniform()) this->expand(); this->setUnsafe(n, val); @@ -1486,12 +1451,6 @@ template void TypedAttributeArray::fill(const ValueType& value) { - if (this->isOutOfCore()) { - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); - this->allocate(); - } - const Index size = mIsUniform ? 1 : this->dataSize(); for (Index i = 0; i < size; ++i) { Codec::encode(value, this->data()[i]); @@ -1507,54 +1466,6 @@ TypedAttributeArray::fill(AttributeArray* array, const Value } -template -bool -TypedAttributeArray::isOutOfCore() const -{ - return mOutOfCore; -} - - -template -void -TypedAttributeArray::setOutOfCore(const bool b) -{ - mOutOfCore = b; -} - - -template -void -TypedAttributeArray::doLoad() const -{ - if (!(this->isOutOfCore())) return; - - TypedAttributeArray* self = - const_cast*>(this); - - // This lock will be contended at most once, after which this buffer - // will no longer be out-of-core. - tbb::spin_mutex::scoped_lock lock(self->mMutex); - this->doLoadUnsafe(); -} - - -template -void -TypedAttributeArray::loadData() const -{ - this->doLoad(); -} - - -template -bool -TypedAttributeArray::isDataLoaded() const -{ - return !this->isOutOfCore(); -} - - template void TypedAttributeArray::read(std::istream& is) @@ -1679,9 +1590,7 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu this->deallocate(); - this->setOutOfCore(false); is.read(mPageHandle, std::streamsize(mPageHandle->size()), false); - std::unique_ptr buffer = mPageHandle->read(); mData.reset(reinterpret_cast(buffer.release())); mPageHandle.reset(); @@ -1727,9 +1636,6 @@ TypedAttributeArray::writeMetadata(std::ostream& os, bool ou bool bloscCompression = io::getDataCompression(os) & io::COMPRESS_BLOSC; - // any compressed data needs to be loaded if out-of-core - if (bloscCompression) this->doLoad(); - size_t compressedBytes = 0; if (!strideOfOne) @@ -1778,8 +1684,6 @@ TypedAttributeArray::writeBuffers(std::ostream& os, bool out OPENVDB_THROW(IoError, "Cannot write out a partially-read AttributeArray."); } - this->doLoad(); - if (this->isUniform()) { os.write(reinterpret_cast(this->data()), sizeof(StorageType)); } @@ -1827,37 +1731,10 @@ TypedAttributeArray::writePagedBuffers(compression::PagedOut OPENVDB_THROW(IoError, "Cannot write out a partially-read AttributeArray."); } - this->doLoad(); - os.write(reinterpret_cast(this->data()), this->arrayMemUsage()); } -template -void -TypedAttributeArray::doLoadUnsafe() const -{ - if (!(this->isOutOfCore())) return; - - // this function expects the mutex to already be locked - - auto* self = const_cast*>(this); - - OPENVDB_ASSERT(self->mPageHandle); - OPENVDB_ASSERT(!(self->mFlags & PARTIALREAD)); - - std::unique_ptr buffer = self->mPageHandle->read(); - - self->mData.reset(reinterpret_cast(buffer.release())); - - self->mPageHandle.reset(); - - // clear all write and out-of-core flags - - self->mOutOfCore = false; -} - - template AttributeArray::AccessorBasePtr TypedAttributeArray::getAccessor() const @@ -1884,9 +1761,6 @@ TypedAttributeArray::isEqual(const AttributeArray& other) co this->mIsUniform != otherT->mIsUniform || this->attributeType() != this->attributeType()) return false; - this->doLoad(); - otherT->doLoad(); - const StorageType *target = this->data(), *source = otherT->data(); if (!target && !source) return true; if (!target || !source) return false; @@ -1978,10 +1852,6 @@ AttributeHandle::AttributeHandle(const AttributeArray& arr OPENVDB_THROW(TypeError, "Cannot bind handle due to incompatible type of AttributeArray."); } - // load data if delay-loaded - - mArray->loadData(); - // bind getter and setter methods AttributeArray::AccessorBasePtr accessor = mArray->getAccessor(); diff --git a/openvdb/openvdb/points/AttributeGroup.cc b/openvdb/openvdb/points/AttributeGroup.cc index 4f022bae7d..431a5ac73a 100644 --- a/openvdb/openvdb/points/AttributeGroup.cc +++ b/openvdb/openvdb/points/AttributeGroup.cc @@ -23,10 +23,6 @@ GroupHandle::GroupHandle(const GroupAttributeArray& array, const GroupType& offs , mBitMask(static_cast(1 << offset)) { OPENVDB_ASSERT(isGroup(mArray)); - - // load data if delay-loaded - - mArray.loadData(); } @@ -36,10 +32,6 @@ GroupHandle::GroupHandle(const GroupAttributeArray& array, const GroupType& bitM , mBitMask(bitMask) { OPENVDB_ASSERT(isGroup(mArray)); - - // load data if delay-loaded - - mArray.loadData(); } diff --git a/openvdb/openvdb/points/AttributeSet.cc b/openvdb/openvdb/points/AttributeSet.cc index 1eebe436ff..71cc065151 100644 --- a/openvdb/openvdb/points/AttributeSet.cc +++ b/openvdb/openvdb/points/AttributeSet.cc @@ -140,17 +140,6 @@ AttributeSet::memUsage() const } -size_t -AttributeSet::memUsageIfLoaded() const -{ - size_t bytes = sizeof(*this) + mDescr->memUsage(); - for (const auto& attr : mAttrs) { - bytes += attr->memUsageIfLoaded(); - } - return bytes; -} - - size_t AttributeSet::find(const std::string& name) const { diff --git a/openvdb/openvdb/points/AttributeSet.h b/openvdb/openvdb/points/AttributeSet.h index 3137eb7413..84cbeb3eaa 100644 --- a/openvdb/openvdb/points/AttributeSet.h +++ b/openvdb/openvdb/points/AttributeSet.h @@ -114,10 +114,8 @@ class OPENVDB_API AttributeSet /// Return the number of bytes of memory used by this attribute set. size_t memUsage() const; - /// Return the number of bytes of memory used by this attribute set once it - /// has been deserialized (this may be different to memUsage() if delay-loading - /// is in use). - size_t memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + size_t memUsageIfLoaded() const { return memUsage(); } /// @brief Return the position of the attribute array whose name is @a name, /// or @c INVALID_POS if no match is found. diff --git a/openvdb/openvdb/points/PointConversion.h b/openvdb/openvdb/points/PointConversion.h index dbdfd97b6d..a77df2f267 100644 --- a/openvdb/openvdb/points/PointConversion.h +++ b/openvdb/openvdb/points/PointConversion.h @@ -134,7 +134,6 @@ populateAttribute( PointDataTreeT& tree, /// @param pointOffsets a vector of cumulative point offsets for each leaf /// @param startOffset a value to shift all the point offsets by /// @param filter an index filter -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored /// template @@ -143,8 +142,17 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, const PointDataGridT& grid, const std::vector& pointOffsets, const Index64 startOffset, - const FilterT& filter = NullFilter(), - const bool inCoreOnly = false); + const FilterT& filter = NullFilter()); + +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridPosition() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline void +convertPointDataGridPosition( PositionAttribute& positionAttribute, + const PointDataGridT& grid, + const std::vector& pointOffsets, + const Index64 startOffset, + const FilterT& filter, + const bool /*inCoreOnly*/); /// @brief Convert the attribute from a PointDataGrid @@ -156,7 +164,6 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, /// @param arrayIndex the index in the Descriptor of the array to be converted. /// @param stride the stride of the attribute /// @param filter an index filter -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored template inline void convertPointDataGridAttribute( TypedAttribute& attribute, @@ -165,8 +172,19 @@ convertPointDataGridAttribute( TypedAttribute& attribute, const Index64 startOffset, const unsigned arrayIndex, const Index stride = 1, - const FilterT& filter = NullFilter(), - const bool inCoreOnly = false); + const FilterT& filter = NullFilter()); + +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridAttribute() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline void +convertPointDataGridAttribute( TypedAttribute& attribute, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const unsigned arrayIndex, + const Index stride, + const FilterT& filter, + const bool /*inCoreOnly*/); /// @brief Convert the group from a PointDataGrid @@ -177,18 +195,26 @@ convertPointDataGridAttribute( TypedAttribute& attribute, /// @param startOffset a value to shift all the point offsets by /// @param index the group index to be converted. /// @param filter an index filter -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored /// +template +inline void +convertPointDataGridGroup( Group& group, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const AttributeSet::Descriptor::GroupIndex index, + const FilterT& filter = NullFilter()); template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridGroup() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void convertPointDataGridGroup( Group& group, const PointDataTreeT& tree, const std::vector& pointOffsets, const Index64 startOffset, const AttributeSet::Descriptor::GroupIndex index, - const FilterT& filter = NullFilter(), - const bool inCoreOnly = false); + const FilterT& filter, + const bool /*inCoreOnly*/); // for internal use only - this traits class extracts T::value_type if defined, // otherwise falls back to using Vec3R diff --git a/openvdb/openvdb/points/PointCount.h b/openvdb/openvdb/points/PointCount.h index d3fef689db..32ae569b5a 100644 --- a/openvdb/openvdb/points/PointCount.h +++ b/openvdb/openvdb/points/PointCount.h @@ -28,28 +28,39 @@ namespace points { /// @brief Count the total number of points in a PointDataTree /// @param tree the PointDataTree in which to count the points /// @param filter an optional index filter -/// @param inCoreOnly if true, points in out-of-core leaf nodes are not counted /// @param threaded enable or disable threading (threading is enabled by default) template inline Index64 pointCount( const PointDataTreeT& tree, const FilterT& filter = NullFilter(), - const bool inCoreOnly = false, const bool threaded = true); +template +OPENVDB_DEPRECATED_MESSAGE("Use pointCount() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline Index64 pointCount( const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded); + /// @brief Populate an array of cumulative point offsets per leaf node. /// @param pointOffsets array of offsets to be populated /// @param tree the PointDataTree from which to populate the offsets /// @param filter an optional index filter -/// @param inCoreOnly if true, points in out-of-core leaf nodes are ignored /// @param threaded enable or disable threading (threading is enabled by default) /// @return The final cumulative point offset. template inline Index64 pointOffsets(std::vector& pointOffsets, const PointDataTreeT& tree, const FilterT& filter = NullFilter(), - const bool inCoreOnly = false, const bool threaded = true); +template +OPENVDB_DEPRECATED_MESSAGE("Use pointOffsets() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +inline Index64 pointOffsets(std::vector& pointOffsets, + const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded); + /// @brief Generate a new grid with voxel values to store the number of points per voxel /// @param grid the PointDataGrid to use to compute the count grid /// @param filter an optional index filter diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index f3f72eab56..fcb10bdb2a 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -218,19 +218,15 @@ makeDescriptorUnique(PointDataTreeT& tree); /// /// @note Multiple threads cannot safely access the same AttributeArray when using streaming. template +OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void -setStreamingMode(PointDataTreeT& tree, bool on = true); +setStreamingMode(PointDataTreeT&, bool /*on*/ = true) { } -/// @brief Sequentially pre-fetch all delayed-load voxel and attribute data from disk in order -/// to accelerate subsequent random access. -/// -/// @param tree the PointDataTree. -/// @param position if enabled, prefetch the position attribute (default is on) -/// @param otherAttributes if enabled, prefetch all other attributes (default is on) template +OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void -prefetch(PointDataTreeT& tree, bool position = true, bool otherAttributes = true); +prefetch(PointDataTreeT&, bool /*position*/ = true, bool /*otherAttributes*/ = true) { } //////////////////////////////////////// @@ -504,7 +500,8 @@ class PointDataLeafNode : public tree::LeafNode, io::MultiPass { Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } void evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels = true) const; @@ -1522,13 +1519,6 @@ PointDataLeafNode::memUsage() const return BaseLeaf::memUsage() + mAttributeSet->memUsage(); } -template -inline Index64 -PointDataLeafNode::memUsageIfLoaded() const -{ - return BaseLeaf::memUsageIfLoaded() + mAttributeSet->memUsageIfLoaded(); -} - template inline void PointDataLeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const @@ -1598,63 +1588,6 @@ makeDescriptorUnique(PointDataTreeT& tree) } -template -inline void -setStreamingMode(PointDataTreeT& tree, bool on) -{ - auto leafIter = tree.beginLeaf(); - for (; leafIter; ++leafIter) { - for (size_t i = 0; i < leafIter->attributeSet().size(); i++) { - leafIter->attributeArray(i).setStreaming(on); - } - } -} - - -template -inline void -prefetch(PointDataTreeT& tree, bool position, bool otherAttributes) -{ - // NOTE: the following is intentionally not multi-threaded, as the I/O - // is faster if done in the order in which it is stored in the file - - auto leaf = tree.cbeginLeaf(); - if (!leaf) return; - - const auto& attributeSet = leaf->attributeSet(); - - // pre-fetch leaf data - - for ( ; leaf; ++leaf) { - leaf->buffer().data(); - } - - // pre-fetch position attribute data (position will typically have index 0) - - size_t positionIndex = attributeSet.find("P"); - - if (position && positionIndex != AttributeSet::INVALID_POS) { - for (leaf = tree.cbeginLeaf(); leaf; ++leaf) { - OPENVDB_ASSERT(leaf->hasAttribute(positionIndex)); - leaf->constAttributeArray(positionIndex).loadData(); - } - } - - // pre-fetch other attribute data - - if (otherAttributes) { - const size_t attributes = attributeSet.size(); - for (size_t attributeIndex = 0; attributeIndex < attributes; attributeIndex++) { - if (attributeIndex == positionIndex) continue; - for (leaf = tree.cbeginLeaf(); leaf; ++leaf) { - OPENVDB_ASSERT(leaf->hasAttribute(attributeIndex)); - leaf->constAttributeArray(attributeIndex).loadData(); - } - } - } -} - - namespace internal { /// @brief Global registration of point data-related types diff --git a/openvdb/openvdb/points/StreamCompression.cc b/openvdb/openvdb/points/StreamCompression.cc index fde60d9f93..5760f3b4ae 100644 --- a/openvdb/openvdb/points/StreamCompression.cc +++ b/openvdb/openvdb/points/StreamCompression.cc @@ -279,13 +279,6 @@ bloscDecompress(const char*, const size_t, const bool) //////////////////////////////////////// -void -Page::load() const -{ - this->doLoad(); -} - - long Page::uncompressedBytes() const { @@ -345,13 +338,6 @@ Page::readBuffers(std::istream&is, bool delayed) } -bool -Page::isOutOfCore() const -{ - return false; -} - - void Page::copy(const std::unique_ptr& temp, int pageSize) { @@ -374,12 +360,6 @@ Page::decompress(const std::unique_ptr& temp) } -void -Page::doLoad() const -{ -} - - //////////////////////////////////////// diff --git a/openvdb/openvdb/points/StreamCompression.h b/openvdb/openvdb/points/StreamCompression.h index d63e9c1116..d983ea2404 100644 --- a/openvdb/openvdb/points/StreamCompression.h +++ b/openvdb/openvdb/points/StreamCompression.h @@ -114,7 +114,6 @@ class OPENVDB_API Page private: struct Info { - SharedPtr meta; std::streamoff filepos; long compressedBytes; long uncompressedBytes; @@ -125,8 +124,8 @@ class OPENVDB_API Page Page() = default; - /// @brief load the Page into memory - void load() const; + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + void load() const { } /// @brief Uncompressed bytes of the Paged data, available /// when the header has been read. @@ -143,8 +142,8 @@ class OPENVDB_API Page /// pointers will be stored to load the data lazily. void readBuffers(std::istream&, bool delayed); - /// @brief Test if the data is out-of-core - bool isOutOfCore() const; + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isOutOfCore() const { return false; } private: /// @brief Convenience method to store a copy of the supplied buffer @@ -153,9 +152,6 @@ class OPENVDB_API Page /// @brief Decompress and store the supplied data void decompress(const std::unique_ptr& temp); - /// @brief Thread-safe loading of the data - void doLoad() const; - std::unique_ptr mInfo = std::unique_ptr(new Info); std::unique_ptr mData; tbb::spin_mutex mMutex; diff --git a/openvdb/openvdb/points/impl/PointConversionImpl.h b/openvdb/openvdb/points/impl/PointConversionImpl.h index 22fc09a5a0..bab3771610 100644 --- a/openvdb/openvdb/points/impl/PointConversionImpl.h +++ b/openvdb/openvdb/points/impl/PointConversionImpl.h @@ -133,15 +133,13 @@ struct ConvertPointDataGridPositionOp { const Index64 startOffset, const math::Transform& transform, const size_t index, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) : mAttribute(attribute) , mPointOffsets(pointOffsets) , mStartOffset(startOffset) , mTransform(transform) , mIndex(index) , mFilter(filter) - , mInCoreOnly(inCoreOnly) { // only accept Vec3f as ValueType static_assert(VecTraits::Size == 3 && @@ -169,8 +167,6 @@ struct ConvertPointDataGridPositionOp { OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size()); - if (mInCoreOnly && leaf->buffer().isOutOfCore()) continue; - Index64 offset = mStartOffset; if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1]; @@ -196,7 +192,6 @@ struct ConvertPointDataGridPositionOp { const math::Transform& mTransform; const size_t mIndex; const FilterT& mFilter; - const bool mInCoreOnly; }; // ConvertPointDataGridPositionOp @@ -215,15 +210,13 @@ struct ConvertPointDataGridAttributeOp { const Index64 startOffset, const size_t index, const Index stride, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) : mAttribute(attribute) , mPointOffsets(pointOffsets) , mStartOffset(startOffset) , mIndex(index) , mStride(stride) - , mFilter(filter) - , mInCoreOnly(inCoreOnly) { } + , mFilter(filter) { } template void convert(IterT& iter, HandleT& targetHandle, @@ -257,8 +250,6 @@ struct ConvertPointDataGridAttributeOp { OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size()); - if (mInCoreOnly && leaf->buffer().isOutOfCore()) continue; - Index64 offset = mStartOffset; if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1]; @@ -284,7 +275,6 @@ struct ConvertPointDataGridAttributeOp { const size_t mIndex; const Index mStride; const FilterT& mFilter; - const bool mInCoreOnly; }; // ConvertPointDataGridAttributeOp template @@ -299,14 +289,12 @@ struct ConvertPointDataGridGroupOp { const std::vector& pointOffsets, const Index64 startOffset, const AttributeSet::Descriptor::GroupIndex index, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) : mGroup(group) , mPointOffsets(pointOffsets) , mStartOffset(startOffset) , mIndex(index) - , mFilter(filter) - , mInCoreOnly(inCoreOnly) { } + , mFilter(filter) { } template void convert(IterT& iter, const GroupAttributeArray& groupArray, Index64& offset) const @@ -337,8 +325,6 @@ struct ConvertPointDataGridGroupOp { OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size()); - if (mInCoreOnly && leaf->buffer().isOutOfCore()) continue; - Index64 offset = mStartOffset; if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1]; @@ -365,7 +351,6 @@ struct ConvertPointDataGridGroupOp { const Index64 mStartOffset; const GroupIndex mIndex; const FilterT& mFilter; - const bool mInCoreOnly; }; // ConvertPointDataGridGroupOp template @@ -582,8 +567,7 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, const PointDataGridT& grid, const std::vector& pointOffsets, const Index64 startOffset, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) { using TreeType = typename PointDataGridT::TreeType; using LeafManagerT = typename tree::LeafManager; @@ -601,12 +585,26 @@ convertPointDataGridPosition( PositionAttribute& positionAttribute, LeafManagerT leafManager(tree); ConvertPointDataGridPositionOp convert( positionAttribute, pointOffsets, startOffset, grid.transform(), positionIndex, - filter, inCoreOnly); + filter); tbb::parallel_for(leafManager.leafRange(), convert); positionAttribute.compact(); } +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridPosition() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +void +convertPointDataGridPosition( PositionAttribute& positionAttribute, + const PointDataGridT& grid, + const std::vector& pointOffsets, + const Index64 startOffset, + const FilterT& filter, + const bool /*inCoreOnly*/) +{ + convertPointDataGridPosition(positionAttribute, grid, pointOffsets, startOffset, filter); +} + + //////////////////////////////////////// @@ -618,8 +616,7 @@ convertPointDataGridAttribute( TypedAttribute& attribute, const Index64 startOffset, const unsigned arrayIndex, const Index stride, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) { using LeafManagerT = typename tree::LeafManager; @@ -633,12 +630,28 @@ convertPointDataGridAttribute( TypedAttribute& attribute, LeafManagerT leafManager(tree); ConvertPointDataGridAttributeOp convert( attribute, pointOffsets, startOffset, arrayIndex, stride, - filter, inCoreOnly); + filter); tbb::parallel_for(leafManager.leafRange(), convert); attribute.compact(); } +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridAttribute() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +void +convertPointDataGridAttribute( TypedAttribute& attribute, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const unsigned arrayIndex, + const Index stride, + const FilterT& filter, + const bool /*inCoreOnly*/) +{ + convertPointDataGridAttribute(attribute, tree, pointOffsets, startOffset, arrayIndex, stride, filter); +} + + //////////////////////////////////////// @@ -649,8 +662,7 @@ convertPointDataGridGroup( Group& group, const std::vector& pointOffsets, const Index64 startOffset, const AttributeSet::Descriptor::GroupIndex index, - const FilterT& filter, - const bool inCoreOnly) + const FilterT& filter) { using LeafManagerT= typename tree::LeafManager; @@ -662,7 +674,7 @@ convertPointDataGridGroup( Group& group, LeafManagerT leafManager(tree); ConvertPointDataGridGroupOp convert( group, pointOffsets, startOffset, index, - filter, inCoreOnly); + filter); tbb::parallel_for(leafManager.leafRange(), convert); // must call this after modifying point groups in parallel @@ -670,6 +682,22 @@ convertPointDataGridGroup( Group& group, group.finalize(); } + +template +OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridGroup() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +void +convertPointDataGridGroup( Group& group, + const PointDataTreeT& tree, + const std::vector& pointOffsets, + const Index64 startOffset, + const AttributeSet::Descriptor::GroupIndex index, + const FilterT& filter, + const bool /*inCoreOnly*/) +{ + convertPointDataGridGroup(group, tree, pointOffsets, startOffset, index, filter); +} + + template inline float computeVoxelSize( const PositionWrapper& positions, diff --git a/openvdb/openvdb/points/impl/PointCountImpl.h b/openvdb/openvdb/points/impl/PointCountImpl.h index 29c998b9c8..9f913a5ce0 100644 --- a/openvdb/openvdb/points/impl/PointCountImpl.h +++ b/openvdb/openvdb/points/impl/PointCountImpl.h @@ -17,16 +17,14 @@ namespace points { template Index64 pointCount(const PointDataTreeT& tree, const FilterT& filter, - const bool inCoreOnly, const bool threaded) { using LeafManagerT = tree::LeafManager; using LeafRangeT = typename LeafManagerT::LeafRange; auto countLambda = - [&filter, &inCoreOnly] (const LeafRangeT& range, Index64 sum) -> Index64 { + [&filter] (const LeafRangeT& range, Index64 sum) -> Index64 { for (const auto& leaf : range) { - if (inCoreOnly && leaf.buffer().isOutOfCore()) continue; auto state = filter.state(leaf); if (state == index::ALL) { sum += leaf.pointCount(); @@ -48,11 +46,21 @@ Index64 pointCount(const PointDataTreeT& tree, } +template +OPENVDB_DEPRECATED_MESSAGE("Use pointCount() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +Index64 pointCount(const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded) +{ + return pointCount(tree, filter, threaded); +} + + template Index64 pointOffsets( std::vector& pointOffsets, const PointDataTreeT& tree, const FilterT& filter, - const bool inCoreOnly, const bool threaded) { using LeafT = typename PointDataTreeT::LeafNodeType; @@ -67,8 +75,7 @@ Index64 pointOffsets( std::vector& pointOffsets, LeafManagerT leafManager(tree); leafManager.foreach( - [&pointOffsets, &filter, &inCoreOnly](const LeafT& leaf, size_t pos) { - if (inCoreOnly && leaf.buffer().isOutOfCore()) return; + [&pointOffsets, &filter](const LeafT& leaf, size_t pos) { auto state = filter.state(leaf); if (state == index::ALL) { pointOffsets[pos] = leaf.pointCount(); @@ -90,6 +97,18 @@ Index64 pointOffsets( std::vector& pointOffsets, } +template +OPENVDB_DEPRECATED_MESSAGE("Use pointOffsets() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +Index64 pointOffsets( std::vector& pointOffsets, + const PointDataTreeT& tree, + const FilterT& filter, + const bool /*inCoreOnly*/, + const bool threaded) +{ + return pointOffsets(pointOffsets, tree, filter, threaded); +} + + template typename GridT::Ptr pointCountGrid( const PointDataGridT& points, diff --git a/openvdb/openvdb/points/impl/PointMoveImpl.h b/openvdb/openvdb/points/impl/PointMoveImpl.h index 7d86a66d1f..87f2b35a4a 100644 --- a/openvdb/openvdb/points/impl/PointMoveImpl.h +++ b/openvdb/openvdb/points/impl/PointMoveImpl.h @@ -319,7 +319,6 @@ struct GlobalMovePointsOp // extract target array and ensure data is out-of-core and non-uniform auto& targetArray = leaf.attributeArray(mAttributeIndex); - targetArray.loadData(); targetArray.expand(); // perform the copy @@ -344,7 +343,6 @@ struct GlobalMovePointsOp const LeafT& sourceLeaf = mSourceLeafManager.leaf(sourceLeafIndex); const auto& sourceArray = sourceLeaf.constAttributeArray(mAttributeIndex); - sourceArray.loadData(); targetArray.copyValuesUnsafe(sourceArray, copyIterator); @@ -427,12 +425,10 @@ struct LocalMovePointsOp const Index sourceLeafOffset(mSourceIndices[idx]); LeafT& sourceLeaf = mSourceLeafManager.leaf(sourceLeafOffset); const auto& sourceArray = sourceLeaf.constAttributeArray(mAttributeIndex); - sourceArray.loadData(); // extract target array and ensure data is out-of-core and non-uniform auto& targetArray = leaf.attributeArray(mAttributeIndex); - targetArray.loadData(); targetArray.expand(); // perform the copy diff --git a/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h b/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h index 3578b44e83..88ee47f078 100644 --- a/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h +++ b/openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h @@ -948,7 +948,7 @@ class GridToRasterize // generate leaf offsets (if necessary) if (mLeafOffsets.empty()) { openvdb::points::pointOffsets(mLeafOffsets, mGrid->constTree(), resolvedFilter, - /*inCoreOnly=*/false, mSettings.threaded); + mSettings.threaded); } // set streaming arbitrary attribute array flags diff --git a/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h b/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h index f2ce326efd..3b2a66e1c9 100644 --- a/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h +++ b/openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h @@ -310,10 +310,7 @@ struct WeightPosSumsTransfer auto& leaf = this->mManager.leaf(idx); { - // @todo add API to get the array from the group handle. The handle - // calls loadData but not expand. auto& array = leaf.attributeArray(this->mIndices.mEllipsesGroupIndex.first); - array.loadData(); // so we can call setUnsafe/getUnsafe array.expand(); } diff --git a/openvdb/openvdb/tools/Count.h b/openvdb/openvdb/tools/Count.h index 03dc719b86..c1a3a9f9f9 100644 --- a/openvdb/openvdb/tools/Count.h +++ b/openvdb/openvdb/tools/Count.h @@ -60,19 +60,13 @@ Index64 countActiveTiles(const TreeT& tree, bool threaded = true); /// @brief Return the total amount of memory in bytes occupied by this tree. -/// @details This method returns the total in-core memory usage which can be -/// different to the maximum possible memory usage for trees which have not -/// been fully deserialized (via delay-loading). Thus, this is the current -/// true memory consumption. template Index64 memUsage(const TreeT& tree, bool threaded = true); -/// @brief Return the deserialized memory usage of this tree. This is not -/// necessarily equal to the current memory usage (returned by tools::memUsage) -/// if delay-loading is enabled. See File::open. template -Index64 memUsageIfLoaded(const TreeT& tree, bool threaded = true); +OPENVDB_DEPRECATED_MESSAGE("Use tools::memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") +Index64 memUsageIfLoaded(const TreeT& tree, bool threaded = true) { return tools::memUsage(tree, threaded); } /// @brief Return the minimum and maximum active values in this tree. @@ -299,9 +293,9 @@ struct MemUsageOp using RootT = typename TreeType::RootNodeType; using LeafT = typename TreeType::LeafNodeType; - MemUsageOp(const bool inCoreOnly) : mInCoreOnly(inCoreOnly) {} - MemUsageOp(const MemUsageOp& other) : mCount(0), mInCoreOnly(other.mInCoreOnly) {} - MemUsageOp(const MemUsageOp& other, tbb::split) : MemUsageOp(other) {} + MemUsageOp() = default; + MemUsageOp(const MemUsageOp& other) = default; + MemUsageOp(const MemUsageOp&, tbb::split) : mCount(0) {} // accumulate size of the root node in bytes bool operator()(const RootT& root, size_t) @@ -323,8 +317,7 @@ struct MemUsageOp // accumulate size of leaf node in bytes bool operator()(const LeafT& leaf, size_t) { - if (mInCoreOnly) mCount += leaf.memUsage(); - else mCount += leaf.memUsageIfLoaded(); + mCount += leaf.memUsage(); return false; } @@ -334,7 +327,6 @@ struct MemUsageOp } openvdb::Index64 mCount{0}; - const bool mInCoreOnly; }; // struct MemUsageOp /// @brief A DynamicNodeManager operator to find the minimum and maximum active values in this tree. @@ -492,25 +484,12 @@ Index64 countActiveTiles(const TreeT& tree, bool threaded) template Index64 memUsage(const TreeT& tree, bool threaded) { - count_internal::MemUsageOp op(true); + count_internal::MemUsageOp op; tree::DynamicNodeManager nodeManager(tree); nodeManager.reduceTopDown(op, threaded); return op.mCount + sizeof(tree); } -template -Index64 memUsageIfLoaded(const TreeT& tree, bool threaded) -{ - /// @note For numeric (non-point) grids this really doesn't need to - /// traverse the tree and could instead be computed from the node counts. - /// We do so anyway as it ties this method into the tree data structure - /// which makes sure that changes to the tree/nodes are reflected/kept in - /// sync here. - count_internal::MemUsageOp op(false); - tree::DynamicNodeManager nodeManager(tree); - nodeManager.reduceTopDown(op, threaded); - return op.mCount + sizeof(tree); -} template math::MinMax minMax(const TreeT& tree, bool threaded) diff --git a/openvdb/openvdb/tools/Merge.h b/openvdb/openvdb/tools/Merge.h index a7afd4d055..54fbf927b4 100644 --- a/openvdb/openvdb/tools/Merge.h +++ b/openvdb/openvdb/tools/Merge.h @@ -581,16 +581,14 @@ struct UnallocatedBuffer static void allocateAndFill(BufferT& buffer, const ValueT& background) { if (buffer.empty()) { - if (!buffer.isOutOfCore()) { - buffer.allocate(); - buffer.fill(background); - } + buffer.allocate(); + buffer.fill(background); } } static bool isPartiallyConstructed(const BufferT& buffer) { - return !buffer.isOutOfCore() && buffer.empty(); + return buffer.empty(); } }; // struct AllocateAndFillBuffer diff --git a/openvdb/openvdb/tools/PointIndexGrid.h b/openvdb/openvdb/tools/PointIndexGrid.h index 3db213ea38..245b93ed5d 100644 --- a/openvdb/openvdb/tools/PointIndexGrid.h +++ b/openvdb/openvdb/tools/PointIndexGrid.h @@ -1491,7 +1491,8 @@ struct PointIndexLeafNode : public tree::LeafNode Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } //////////////////////////////////////// @@ -1798,13 +1799,6 @@ PointIndexLeafNode::memUsage() const return BaseLeaf::memUsage() + Index64((sizeof(T)*mIndices.capacity()) + sizeof(mIndices)); } -template -inline Index64 -PointIndexLeafNode::memUsageIfLoaded() const -{ - return BaseLeaf::memUsageIfLoaded() + Index64((sizeof(T)*mIndices.capacity()) + sizeof(mIndices)); -} - } // namespace tools diff --git a/openvdb/openvdb/tools/VolumeToMesh.h b/openvdb/openvdb/tools/VolumeToMesh.h index 509fc1187b..b3c268072a 100644 --- a/openvdb/openvdb/tools/VolumeToMesh.h +++ b/openvdb/openvdb/tools/VolumeToMesh.h @@ -616,8 +616,8 @@ template<> inline bool isInsideValue(bool value, bool /*isovalue*/) { return value; } -/// @brief Minor wrapper around the Leaf API to avoid atomic access with -/// delayed loading. +/// @brief Minor wrapper around the Leaf API. This was originally added to avoid +// atomic access with delayed loading, however delayed loading is no longer supported. template ::value> struct LeafBufferAccessor diff --git a/openvdb/openvdb/tree/LeafBuffer.h b/openvdb/openvdb/tree/LeafBuffer.h index 671781a129..80da87c721 100644 --- a/openvdb/openvdb/tree/LeafBuffer.h +++ b/openvdb/openvdb/tree/LeafBuffer.h @@ -37,27 +37,20 @@ class LeafBuffer static const Index SIZE = 1 << 3 * Log2Dim; /// Default constructor - inline LeafBuffer(): mData(new ValueType[SIZE]) - { - } + inline LeafBuffer(): mData(new ValueType[SIZE]) {} /// Construct a buffer populated with the specified value. explicit inline LeafBuffer(const ValueType&); /// Copy constructor inline LeafBuffer(const LeafBuffer&); /// Construct a buffer but don't allocate memory for the full array of values. - LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) - { - } + LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) {} /// Destructor inline ~LeafBuffer(); - /// Return @c true if this buffer's values have not yet been read from disk. - bool isOutOfCore() const - { - return false; - } + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") + bool isOutOfCore() const { return false; } /// Return @c true if memory for this buffer has not yet been allocated. - bool empty() const { return !mData || this->isOutOfCore(); } + bool empty() const { return !mData; } /// Allocate memory for this buffer if it has not already been allocated. bool allocate() { if (mData == nullptr) mData = new ValueType[SIZE]; return true; } @@ -86,7 +79,8 @@ class LeafBuffer /// Return the memory footprint of this buffer in bytes. inline Index memUsage() const; - inline Index memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + inline Index memUsageIfLoaded() const { return memUsage(); } /// Return the number of values contained in this buffer. static Index size() { return SIZE; } @@ -112,24 +106,10 @@ class LeafBuffer bool deallocate(); - inline void setOutOfCore(bool b) - { - (void) b; - } - // To facilitate inlining in the common case in which the buffer is in-core, - // the loading logic is split into a separate function, doLoad(). - inline void loadValues() const - { - } - inline void doLoad() const; - inline bool detachFromFile(); - - using FlagsType = std::atomic; - - ValueType* mData; - FlagsType mOutOfCore; // interpreted as bool; extra bits reserved for future use - tbb::spin_mutex mMutex; // 1 byte - //int8_t mReserved[3]; // padding for alignment + ValueType* mData = nullptr; + // Deprecated members kept for ABI compatibility + std::atomic mDeprecatedAtomic{0}; + tbb::spin_mutex mDeprecatedSpinMutex; friend class ::TestLeaf; // Allow the parent LeafNode to access this buffer's data pointer. @@ -177,7 +157,6 @@ inline void LeafBuffer::setValue(Index i, const ValueType& val) { OPENVDB_ASSERT(i < SIZE); - this->loadValues(); if (mData) mData[i] = val; } @@ -198,11 +177,11 @@ LeafBuffer::operator=(const LeafBuffer& other) return *this; } + template inline void LeafBuffer::fill(const ValueType& val) { - this->detachFromFile(); if (mData != nullptr) { ValueType* target = mData; Index n = SIZE; @@ -215,8 +194,6 @@ template inline bool LeafBuffer::operator==(const LeafBuffer& other) const { - this->loadValues(); - other.loadValues(); const ValueType *target = mData, *source = other.mData; if (!target && !source) return true; if (!target || !source) return false; @@ -244,21 +221,10 @@ LeafBuffer::memUsage() const } -template -inline Index -LeafBuffer::memUsageIfLoaded() const -{ - size_t n = sizeof(*this); - n += SIZE * sizeof(ValueType); - return static_cast(n); -} - - template inline const typename LeafBuffer::ValueType* LeafBuffer::data() const { - this->loadValues(); if (mData == nullptr) { LeafBuffer* self = const_cast(this); if (mData == nullptr) self->mData = new ValueType[SIZE]; @@ -270,7 +236,6 @@ template inline typename LeafBuffer::ValueType* LeafBuffer::data() { - this->loadValues(); if (mData == nullptr) { if (mData == nullptr) mData = new ValueType[SIZE]; } @@ -284,7 +249,6 @@ LeafBuffer::at(Index i) const { static const ValueType sZero = zeroVal(); OPENVDB_ASSERT(i < SIZE); - this->loadValues(); // We can't use the ternary operator here, otherwise Visual C++ returns // a reference to a temporary. if (mData) return mData[i]; else return sZero; @@ -295,7 +259,6 @@ template inline bool LeafBuffer::deallocate() { - if (mData != nullptr) { delete[] mData; mData = nullptr; @@ -305,21 +268,6 @@ LeafBuffer::deallocate() } -template -inline void -LeafBuffer::doLoad() const -{ -} - - -template -inline bool -LeafBuffer::detachFromFile() -{ - return false; -} - - //////////////////////////////////////// @@ -364,7 +312,8 @@ class LeafBuffer void swap(LeafBuffer& other) { if (&other != this) std::swap(mData, other.mData); } Index memUsage() const { return sizeof(*this); } - Index memUsageIfLoaded() const { return sizeof(*this); } + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index memUsageIfLoaded() const { return memUsage(); } static Index size() { return SIZE; } /// @brief Return a pointer to the C-style array of words encoding the bits. diff --git a/openvdb/openvdb/tree/LeafNode.h b/openvdb/openvdb/tree/LeafNode.h index 6894d17391..8318f1dd5f 100644 --- a/openvdb/openvdb/tree/LeafNode.h +++ b/openvdb/openvdb/tree/LeafNode.h @@ -152,13 +152,14 @@ class LeafNode /// Return @c true if this node contains only active voxels. bool isDense() const { return mValueMask.isOn(); } /// Return @c true if memory for this node's buffer has been allocated. - bool isAllocated() const { return !mBuffer.isOutOfCore() && !mBuffer.empty(); } + bool isAllocated() const { return !mBuffer.empty(); } /// Allocate memory for this node's buffer if it has not already been allocated. bool allocate() { return mBuffer.allocate(); } /// Return the memory in bytes occupied by this node. Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } /// Expand the given bounding box so that it includes this leaf node's active voxels. /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all @@ -466,7 +467,6 @@ class LeafNode template void modifyValue(Index offset, const ModifyOp& op) { - mBuffer.loadValues(); if (!mBuffer.empty()) { // in-place modify value ValueType& val = const_cast(mBuffer[offset]); @@ -487,7 +487,6 @@ class LeafNode template void modifyValueAndActiveState(const Coord& xyz, const ModifyOp& op) { - mBuffer.loadValues(); if (!mBuffer.empty()) { const Index offset = this->coordToOffset(xyz); bool state = mValueMask.isOn(offset); @@ -1262,8 +1261,6 @@ template inline void LeafNode::copyToDense(const CoordBBox& bbox, DenseT& dense) const { - mBuffer.loadValues(); - using DenseValueType = typename DenseT::ValueType; const size_t xStride = dense.xStride(), yStride = dense.yStride(), zStride = dense.zStride(); @@ -1390,11 +1387,9 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo // This node lies completely outside the clipping region. skipCompressedValues(seekable, is, fromHalf); mValueMask.setOff(); - mBuffer.setOutOfCore(false); } else { mBuffer.allocate(); io::readCompressedValues(is, mBuffer.mData, SIZE, mValueMask, fromHalf); - mBuffer.setOutOfCore(false); // Get this tree's background value. T background = zeroVal(); @@ -1417,9 +1412,6 @@ LeafNode::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo } } } - - // increment the leaf number - if (meta) meta->setLeaf(meta->leaf() + 1); } @@ -1430,8 +1422,6 @@ LeafNode::writeBuffers(std::ostream& os, bool toHalf) const // Write out the value mask. mValueMask.save(os); - mBuffer.loadValues(); - io::writeCompressedValues(os, mBuffer.mData, SIZE, mValueMask, /*childMask=*/NodeMaskType(), toHalf); } @@ -1460,16 +1450,6 @@ LeafNode::memUsage() const } -template -inline Index64 -LeafNode::memUsageIfLoaded() const -{ - // Use sizeof(*this) to capture alignment-related padding - // (but note that sizeof(*this) includes sizeof(mBuffer)). - return sizeof(*this) + mBuffer.memUsageIfLoaded() - sizeof(mBuffer); -} - - template inline void LeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const diff --git a/openvdb/openvdb/tree/LeafNodeBool.h b/openvdb/openvdb/tree/LeafNodeBool.h index 80dd31cb87..8caccdbbc6 100644 --- a/openvdb/openvdb/tree/LeafNodeBool.h +++ b/openvdb/openvdb/tree/LeafNodeBool.h @@ -145,7 +145,8 @@ class LeafNode /// Return the memory in bytes occupied by this node. Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } /// Expand the given bounding box so that it includes this leaf node's active voxels. /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all @@ -898,15 +899,6 @@ LeafNode::memUsage() const } -template -inline Index64 -LeafNode::memUsageIfLoaded() const -{ - // Use sizeof(*this) to capture alignment-related padding - return sizeof(*this); -} - - template inline void LeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const diff --git a/openvdb/openvdb/tree/LeafNodeMask.h b/openvdb/openvdb/tree/LeafNodeMask.h index 1a5e9a2bfc..41474874bd 100644 --- a/openvdb/openvdb/tree/LeafNodeMask.h +++ b/openvdb/openvdb/tree/LeafNodeMask.h @@ -144,7 +144,8 @@ class LeafNode /// Return the memory in bytes occupied by this node. Index64 memUsage() const; - Index64 memUsageIfLoaded() const; + OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") + Index64 memUsageIfLoaded() const { return memUsage(); } /// Expand the given bounding box so that it includes this leaf node's active voxels. /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all @@ -884,15 +885,6 @@ LeafNode::memUsage() const } -template -inline Index64 -LeafNode::memUsageIfLoaded() const -{ - // Use sizeof(*this) to capture alignment-related padding - return sizeof(*this); -} - - template inline void LeafNode::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const diff --git a/openvdb/openvdb/tree/Tree.h b/openvdb/openvdb/tree/Tree.h index e0e99881ba..a3371c71d7 100644 --- a/openvdb/openvdb/tree/Tree.h +++ b/openvdb/openvdb/tree/Tree.h @@ -95,13 +95,7 @@ class OPENVDB_API TreeBase virtual void getIndexRange(CoordBBox& bbox) const = 0; - /// @brief Replace with background tiles any nodes whose voxel buffers - /// have not yet been allocated. - /// @details Typically, unallocated nodes are leaf nodes whose voxel buffers - /// are not yet resident in memory because delayed loading is in effect. - /// @sa readNonresidentBuffers, io::File::open virtual void clipUnallocatedNodes() = 0; - /// Return the total number of unallocated leaf nodes residing in this tree. #if OPENVDB_ABI_VERSION_NUMBER >= 12 virtual Index64 unallocatedLeafCount() const = 0; #else @@ -167,11 +161,8 @@ class OPENVDB_API TreeBase virtual void readBuffers(std::istream&, bool saveFloatAsHalf = false) = 0; /// Read all of this tree's data buffers that intersect the given bounding box. virtual void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) = 0; - /// @brief Read all of this tree's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this tree was read from a memory-mapped file, this operation - /// disconnects the tree from the file. - /// @sa clipUnallocatedNodes, io::File::open, io::MappedFile + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void readNonresidentBuffers() const = 0; /// Write out all the data buffers for this tree. virtual void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const = 0; @@ -339,12 +330,10 @@ class Tree: public TreeBase void readBuffers(std::istream&, bool saveFloatAsHalf = false) override; /// Read all of this tree's data buffers that intersect the given bounding box. void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) override; - /// @brief Read all of this tree's data buffers that are not yet resident in memory - /// (because delayed loading is in effect). - /// @details If this tree was read from a memory-mapped file, this operation - /// disconnects the tree from the file. - /// @sa clipUnallocatedNodes, io::File::open, io::MappedFile - void readNonresidentBuffers() const override; + + OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") + void readNonresidentBuffers() const override { } + /// Write out all data buffers for this tree. void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const override; @@ -494,9 +483,6 @@ class Tree: public TreeBase void clip(const CoordBBox&); /// @brief Replace with background tiles any nodes whose voxel buffers /// have not yet been allocated. - /// @details Typically, unallocated nodes are leaf nodes whose voxel buffers - /// are not yet resident in memory because delayed loading is in effect. - /// @sa readNonresidentBuffers, io::File::open void clipUnallocatedNodes() override; /// Return the total number of unallocated leaf nodes residing in this tree. @@ -1315,17 +1301,6 @@ Tree::readBuffers(std::istream &is, const CoordBBox& bbox, bool sa } -template -inline void -Tree::readNonresidentBuffers() const -{ - for (LeafCIter it = this->cbeginLeaf(); it; ++it) { - // Retrieving the value of a leaf voxel forces loading of the leaf node's voxel buffer. - it->getValue(Index(0)); - } -} - - template inline void Tree::writeBuffers(std::ostream &os, bool saveFloatAsHalf) const diff --git a/openvdb/openvdb/unittest/TestCount.cc b/openvdb/openvdb/unittest/TestCount.cc index 0509b71359..201ab6efa6 100644 --- a/openvdb/openvdb/unittest/TestCount.cc +++ b/openvdb/openvdb/unittest/TestCount.cc @@ -223,8 +223,7 @@ TEST_F(TestCount, testMemUsage) internalNodeMemUsage += sizeof(Coord); for (auto leafIter = internal2Iter->cbeginChildOn(); leafIter; ++leafIter) { - EXPECT_EQ(leafIter->memUsage(), leafIter->memUsageIfLoaded()); - expectedMaxMem += leafIter->memUsageIfLoaded(); + expectedMaxMem += leafIter->memUsage(); ++leafCount; } } @@ -232,11 +231,9 @@ TEST_F(TestCount, testMemUsage) expectedMaxMem += internalNodeMemUsage; - Index64 inCoreMemUsage = tools::memUsage(grid->tree()); - Index64 memUsageIfLoaded = tools::memUsageIfLoaded(grid->tree()); + Index64 memUsage = tools::memUsage(grid->tree()); - EXPECT_EQ(expectedMaxMem, inCoreMemUsage); - EXPECT_EQ(expectedMaxMem, memUsageIfLoaded); + EXPECT_EQ(expectedMaxMem, memUsage); } diff --git a/openvdb/openvdb/unittest/TestMeshToVolume.cc b/openvdb/openvdb/unittest/TestMeshToVolume.cc index 5c4ff2ffdf..587c092cf2 100644 --- a/openvdb/openvdb/unittest/TestMeshToVolume.cc +++ b/openvdb/openvdb/unittest/TestMeshToVolume.cc @@ -211,8 +211,8 @@ TEST_F(TestMeshToVolume, testInterrupt) // Should have returned _something_ EXPECT_TRUE(grid); - // Expect to interrupt in under a second + // Expect to interrupt in under two seconds const auto duration = std::chrono::duration_cast(end - start); - EXPECT_LT(duration.count(), 1000); + EXPECT_LT(duration.count(), 2000); } diff --git a/openvdb/openvdb/unittest/TestPointCount.cc b/openvdb/openvdb/unittest/TestPointCount.cc index f02661456c..0d2332674b 100644 --- a/openvdb/openvdb/unittest/TestPointCount.cc +++ b/openvdb/openvdb/unittest/TestPointCount.cc @@ -313,17 +313,14 @@ TEST_F(TestPointCount, testGroup) GroupFilter groupFilter("test", attributeSet); - bool inCoreOnly; - inCoreOnly = false; - - EXPECT_EQ(pointCount(inputTree, NullFilter(), inCoreOnly), Index64(4)); - EXPECT_EQ(pointCount(inputTree, ActiveFilter(), inCoreOnly), Index64(3)); - EXPECT_EQ(pointCount(inputTree, InactiveFilter(), inCoreOnly), Index64(1)); - EXPECT_EQ(pointCount(inputTree, groupFilter, inCoreOnly), Index64(2)); + EXPECT_EQ(pointCount(inputTree, NullFilter()), Index64(4)); + EXPECT_EQ(pointCount(inputTree, ActiveFilter()), Index64(3)); + EXPECT_EQ(pointCount(inputTree, InactiveFilter()), Index64(1)); + EXPECT_EQ(pointCount(inputTree, groupFilter), Index64(2)); EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, ActiveFilter()), inCoreOnly), Index64(1)); + groupFilter, ActiveFilter())), Index64(1)); EXPECT_EQ(pointCount(inputTree, BinaryFilter( - groupFilter, InactiveFilter()), inCoreOnly), Index64(1)); + groupFilter, InactiveFilter())), Index64(1)); } std::remove(filename.c_str()); diff --git a/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc b/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc index 572b66dd2b..7d8997888f 100644 --- a/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc +++ b/openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc @@ -2333,7 +2333,6 @@ TEST_F(TestPointRasterizeFrustum, testStreaming) Vec3SGrid::Ptr velocity1, velocity2, velocity3; const size_t mb = 1024*1024; - const size_t tinyMemory = static_cast(0.1*mb); { // memory test 1 - retain caches and streaming disabled Rasterizer rasterizer(settings); diff --git a/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc b/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc index 3598fb3adb..7b704e8524 100644 --- a/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc +++ b/openvdb_ax/openvdb_ax/compiler/PointExecutable.cc @@ -362,7 +362,6 @@ struct PointFunctionArguments // @todo if the array is shared we should probably make it unique? if (mData.mUseBufferKernel) { - const_cast(array).loadData(); const char* data = array.constDataAsByteArray(); void* ptr = static_cast(const_cast(data)); mHandlesOrBuffers.emplace_back(ptr); @@ -387,14 +386,13 @@ struct PointFunctionArguments array.expand(); if (mData.mUseBufferKernel) { - array.loadData(); const char* data = array.constDataAsByteArray(); void* ptr = static_cast(const_cast(data)); mHandlesOrBuffers.emplace_back(ptr); const codegen::Codec* codec = codegen::getCodec(ast::tokens::tokenFromTypeString(array.valueType()), array.codecType()); if (codec) flag |= codec->flag(); - OPENVDB_ASSERT(array.isDataLoaded() && !array.isUniform()); + OPENVDB_ASSERT(!array.isUniform()); } else { typename WriteHandle::UniquePtr handle(new WriteHandle(leaf, Index(pos))); diff --git a/openvdb_cmd/vdb_ax/main.cc b/openvdb_cmd/vdb_ax/main.cc index 9543d9fe6c..b62c4559a7 100644 --- a/openvdb_cmd/vdb_ax/main.cc +++ b/openvdb_cmd/vdb_ax/main.cc @@ -791,9 +791,7 @@ main(int argc, char *argv[]) if (opts.mMode.get() == openvdb::ax::VDB_AX_MODE::Execute) { // read vdb file data for - axlog("[INFO] Reading VDB data" - << (openvdb::io::Archive::isDelayedLoadingEnabled() ? - " (delay-load)" : "") << '\n'); + axlog("[INFO] Reading VDB data" << '\n'); for (const auto& filename : opts.mInputVDBFiles.get()) { openvdb::io::File file(filename); try { diff --git a/openvdb_cmd/vdb_print/main.cc b/openvdb_cmd/vdb_print/main.cc index e093838768..3a329d4089 100644 --- a/openvdb_cmd/vdb_print/main.cc +++ b/openvdb_cmd/vdb_print/main.cc @@ -209,14 +209,9 @@ printShortListing(const StringVec& filenames, bool metadata) // Print the grid's size, in bytes - // no support for memUsageIfLoaded until ABI >= 10 for points::PointDataGrid types using ListT = openvdb::GridTypes; grid->apply([&](const auto& typed){ - // @todo combine these methods to avoid iterating across the tree twice - const openvdb::Index64 incore = openvdb::tools::memUsage(typed.tree()); - const openvdb::Index64 total = openvdb::tools::memUsageIfLoaded(typed.tree()); - - std::cout << " " << std::right << std::setw(6) << bytesAsString(incore) << " (In Core)"; + const openvdb::Index64 total = openvdb::tools::memUsage(typed.tree()); std::cout << " " << std::right << std::setw(6) << bytesAsString(total) << " (Total)"; }); diff --git a/openvdb_cmd/vdb_tool/include/Tool.h b/openvdb_cmd/vdb_tool/include/Tool.h index b6d5e3aaef..10ba883187 100644 --- a/openvdb_cmd/vdb_tool/include/Tool.h +++ b/openvdb_cmd/vdb_tool/include/Tool.h @@ -127,7 +127,7 @@ class Tool using FilterT = std::unique_ptr>; struct Points;// defined below struct Header;// defined below - + mutable util::CpuTimer mTimer; std::string mCmdName;// name of this command-line tool std::list mGeom;// list of geometries owned by this tool diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index 9608005294..b1fa29ba0a 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -563,10 +563,10 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, int numPoints = 0; if (useGroup) { GroupFilter filter(groupName, iter->attributeSet()); - numPoints = static_cast(pointCount(tree, filter, /*inCoreOnly=*/true)); + numPoints = static_cast(pointCount(tree, filter)); } else { NullFilter filter; - numPoints = static_cast(pointCount(tree, filter, /*inCoreOnly=*/true)); + numPoints = static_cast(pointCount(tree, filter)); } if (numPoints == 0) return; @@ -596,13 +596,13 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, MultiGroupFilter filter(includeGroups, excludeGroups, iter->attributeSet()); std::vector offsets; - pointOffsets(offsets, grid.tree(), filter, /*inCoreOnly=*/true); + pointOffsets(offsets, grid.tree(), filter); UT_UniquePtr pdata(new UT_Vector3F[numPoints]); PositionAttribute positionAttribute(pdata.get(), mCentroid); convertPointDataGridPosition(positionAttribute, grid, offsets, - /*startOffset=*/ 0, filter, /*inCoreOnly=*/true); + /*startOffset=*/ 0, filter); posGeo->setArray(r, pdata.get()); posGeo->setCacheVersion(version); @@ -647,20 +647,12 @@ GR_PrimVDBPoints::updateWireBuffer(RE_Render *r, using LeafNode = TreeType::LeafNodeType; const TreeType& tree = grid.tree(); + size_t leafCount = tree.leafCount(); if (tree.leafCount() == 0) return; - // count up total points ignoring any leaf nodes that are out of core - - size_t outOfCoreLeaves = 0; - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - if (iter->buffer().isOutOfCore()) outOfCoreLeaves++; - } - - if (outOfCoreLeaves == 0) return; - // Initialize the number of points for the wireframe box per leaf. - int numPoints = static_cast(outOfCoreLeaves*8*3); + int numPoints = static_cast(leafCount*8*3); myWire->setNumPoints(numPoints); // fetch wireframe position, if its cache version matches, no upload is required. @@ -680,9 +672,6 @@ GR_PrimVDBPoints::updateWireBuffer(RE_Render *r, for (auto iter = tree.cbeginLeaf(); iter; ++iter) { const LeafNode& leaf = *iter; - // skip in-core leaf nodes (for use when delay loading VDBs) - if (!leaf.buffer().isOutOfCore()) continue; - coords.push_back(leaf.origin()); } @@ -826,12 +815,12 @@ GR_PrimVDBPoints::updateVec3Buffer(RE_Render* r, MultiGroupFilter filter(includeGroups, excludeGroups, iter->attributeSet()); std::vector offsets; - pointOffsets(offsets, grid.tree(), filter, /*inCoreOnly=*/true); + pointOffsets(offsets, grid.tree(), filter); VectorAttribute typedAttribute(data.get()); convertPointDataGridAttribute(typedAttribute, grid.tree(), offsets, /*startOffset=*/ 0, static_cast(index), /*stride=*/1, - filter, /*inCoreOnly=*/true); + filter); } bufferGeo->setArray(r, data.get()); diff --git a/openvdb_houdini/openvdb_houdini/PointUtils.cc b/openvdb_houdini/openvdb_houdini/PointUtils.cc index e8070f0265..fdae57ccf1 100644 --- a/openvdb_houdini/openvdb_houdini/PointUtils.cc +++ b/openvdb_houdini/openvdb_houdini/PointUtils.cc @@ -1073,8 +1073,7 @@ convertPointDataGridToHoudini( const PointDataGrid& grid, const std::vector& attributes, const std::vector& includeGroups, - const std::vector& excludeGroups, - const bool inCoreOnly) + const std::vector& excludeGroups) { using namespace openvdb::math; @@ -1096,14 +1095,14 @@ convertPointDataGridToHoudini( // obtain cumulative point offsets and total points std::vector offsets; MultiGroupFilter filter(includeGroups, excludeGroups, leafIter->attributeSet()); - const Index64 total = pointOffsets(offsets, tree, filter, inCoreOnly); + const Index64 total = pointOffsets(offsets, tree, filter); // a block's global offset is needed to transform its point offsets to global offsets const Index64 startOffset = detail.appendPointBlock(total); HoudiniWriteAttribute positionAttribute(*detail.getP()); convertPointDataGridPosition(positionAttribute, grid, offsets, startOffset, - filter, inCoreOnly); + filter); // add other point attributes to the hdk detail const AttributeSet::Descriptor::NameToPosMap& nameToPosMap = descriptor.map(); @@ -1186,87 +1185,87 @@ convertPointDataGridToHoudini( if (valueType == "string") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "bool") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int8") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int16") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int32") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "int64") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "float") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "double") { HoudiniWriteAttribute attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "vec3i") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "vec3s") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "vec3d") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "quats") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "quatd") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat3s") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat3d") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat4s") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else if (valueType == "mat4d") { HoudiniWriteAttribute > attribute(*attributeRef.getAttribute()); convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride, - filter, inCoreOnly); + filter); } else { throw std::runtime_error("Unknown Attribute Type for Conversion: " + valueType); @@ -1288,7 +1287,7 @@ convertPointDataGridToHoudini( attributeSet.groupIndex(name); HoudiniGroup group(*pointGroup, startOffset, total); - convertPointDataGridGroup(group, tree, offsets, startOffset, index, filter, inCoreOnly); + convertPointDataGridGroup(group, tree, offsets, startOffset, index, filter); } } @@ -1615,32 +1614,7 @@ collectPointInfo(const PointDataGrid& grid, const PointDataTree& tree = grid.constTree(); - // iterate through all leaf nodes to find out if all are out-of-core - bool allOutOfCore = true; - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - if (!iter->buffer().isOutOfCore()) { - allOutOfCore = false; - break; - } - } - - openvdb::Index64 totalPointCount = 0; - - // it is more technically correct to rely on the voxel count as this may be - // out of sync with the attribute size, however for faster node preview when - // the voxel buffers are all out-of-core, count up the sizes of the first - // attribute array instead - - if (allOutOfCore) { - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - if (iter->attributeSet().size() > 0) { - totalPointCount += iter->constAttributeArray(0).size(); - } - } - } - else { - totalPointCount = openvdb::points::pointCount(tree); - } + openvdb::Index64 totalPointCount = openvdb::points::pointCount(tree); std::ostringstream os; os << openvdb::util::formattedInt(totalPointCount); @@ -1674,16 +1648,8 @@ collectPointInfo(const PointDataGrid& grid, os << it.first << "("; - // for faster node preview when all the voxel buffers are out-of-core, - // don't load the group arrays to display the group sizes, just print - // "out-of-core" instead @todo - put the group sizes into the grid - // metadata on write for this use case - - if (allOutOfCore) os << "out-of-core"; - else { - const openvdb::points::GroupFilter filter(it.first, attributeSet); - os << openvdb::util::formattedInt(pointCount(tree, filter)); - } + const openvdb::points::GroupFilter filter(it.first, attributeSet); + os << openvdb::util::formattedInt(pointCount(tree, filter)); os << ")"; } diff --git a/openvdb_houdini/openvdb_houdini/PointUtils.h b/openvdb_houdini/openvdb_houdini/PointUtils.h index 57dbf6306e..89fe54d941 100644 --- a/openvdb_houdini/openvdb_houdini/PointUtils.h +++ b/openvdb_houdini/openvdb_houdini/PointUtils.h @@ -109,7 +109,6 @@ convertHoudiniToPointDataGrid( /// (empty vector defaults to all) /// @param excludeGroups a vector of VDB Points groups to be excluded /// (empty vector defaults to none) -/// @param inCoreOnly true if out-of-core leaf nodes are to be ignored OPENVDB_HOUDINI_API void convertPointDataGridToHoudini( @@ -117,8 +116,7 @@ convertPointDataGridToHoudini( const openvdb::points::PointDataGrid& grid, const std::vector& attributes = {}, const std::vector& includeGroups = {}, - const std::vector& excludeGroups = {}, - const bool inCoreOnly = false); + const std::vector& excludeGroups = {}); /// @brief Populate VDB Points grid metadata from Houdini detail attributes diff --git a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc index e5abf786d3..665882b9ae 100644 --- a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc +++ b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc @@ -693,23 +693,10 @@ SOP_OpenVDB_Points_Convert::Cache::cookVDBSop(OP_Context& context) // all attributes should be converted const std::vector emptyNameVector; - // if all point data is being converted, sequentially pre-fetch any out-of-core - // data for faster performance when using delayed-loading - - const bool allData = emptyNameVector.empty() && - includeGroups.empty() && - excludeGroups.empty(); - for (const PointDataGrid::ConstPtr &grid : pointGrids) { GU_Detail geo; - // if all the data is being loaded, prefetch it for faster load performance - - if (allData) { - prefetch(grid->tree()); - } - // perform conversion hvdb::convertPointDataGridToHoudini( diff --git a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc index 2dcaf0aa26..d1a11f007c 100644 --- a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc +++ b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc @@ -516,59 +516,24 @@ struct GridsToRasterize struct Grid { - private: - static bool isTreeOutOfCore(const TreeType& tree) - { - using LeafManagerT = openvdb::tree::LeafManager; - using LeafRangeT = typename LeafManagerT::LeafRange; - LeafManagerT leafManager(tree); - return tbb::parallel_reduce(leafManager.leafRange(), true, - [] (const LeafRangeT& range, bool result) -> bool { - for (const auto& leaf : range) { - if (!leaf.buffer().isOutOfCore()) return false; - } - return result; - }, - [] (bool n, bool m) -> bool { return n || m; }); - } - public: explicit Grid(const ConstPtr& grid) - : mGrid(grid) - , mOutOfCore(isTreeOutOfCore(grid->constTree())) { } - - inline bool isOutOfCore() const { return mOutOfCore; } + : mGrid(grid) { } template void addToRasterizer(RasterizerT& rasterizer) { - if (mOutOfCore) { - auto newGrid = mGrid->deepCopy(); - rasterizer.addPoints(newGrid, /*stream=*/true); - } - else { - rasterizer.addPoints(mGrid); - } + rasterizer.addPoints(mGrid); } private: ConstPtr mGrid; - const bool mOutOfCore; }; // struct Grid void push_back(ConstPtr& grid) { mGrids.emplace_back(grid); } size_t size() const { return mGrids.size(); } bool empty() const { return mGrids.empty(); } - // return true if any of the grids is out-of-core - bool streaming() const - { - for (auto& grid : mGrids) { - if (grid.isOutOfCore()) return true; - } - return false; - } - template void addGridToRasterizer(RasterizerT& rasterizer, size_t index) { @@ -885,8 +850,7 @@ Rasterizing can be performed into cartesian and frustum grids (otherwise referre grids). For frustum grids, approximations are used by default to accelerate the rasterization.\n\ This can be disabled to achieve a more accurate result.\n\ \n\ -This node supports streaming of VDB points and attributes if one or more delayed-load VDB is\n\ -provided as an input.\n\ +This node supports streaming of VDB points and attributes\n\ \n\ @related\n\ - [OpenVDB Rasterize Points|Node:sop/DW_OpenVDBRasterizePoints]\n\ @@ -1257,8 +1221,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) iterations = 1; } - bool streaming = pointGrids.streaming(); - for (size_t i = 0; i < iterations; i++) { if (!mergeVDBPoints) { @@ -1274,12 +1236,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) if (std::find(vectorAttribNames.begin(), vectorAttribNames.end(), velocityAttribute) != vectorAttribNames.end()) { velocity = rasterizer.rasterizeAttribute(velocityAttribute, vectorMode, reduceMemory, scale, rasterGroups); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } // rasterize density @@ -1287,12 +1243,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) if (createDensity) { auto density = rasterizer.rasterizeDensity(densityAttribute, densityMode, reduceMemory, densityScale, rasterGroups); outputGrids.push_back(density); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } // rasterize mask @@ -1307,12 +1257,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) for (const auto& name : scalarAttribNames) { auto scalar = rasterizer.rasterizeAttribute(name, scalarMode, reduceMemory, scale, rasterGroups); outputGrids.push_back(scalar); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } // rasterize vector attributes @@ -1327,12 +1271,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context) else { auto vector = rasterizer.rasterizeAttribute(name, vectorMode, reduceMemory, scale, rasterGroups); outputGrids.push_back(vector); - - // need to deep-copy input grids again if caches are being discarded - if (streaming && reduceMemory) { - if (mergeVDBPoints) pointGrids.addGridsToRasterizer(rasterizer); - else pointGrids.addGridToRasterizer(rasterizer, i); - } } } } diff --git a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc index 7b11986b40..887a0a6e11 100644 --- a/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc +++ b/openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Read.cc @@ -204,40 +204,20 @@ newSopOperator(OP_OperatorTable* table) .setCallbackFunc(&reloadCB) .setTooltip("Reread the VDB file.")); - parms.add(hutil::ParmFactory(PRM_SEPARATOR, "sep1", "Sep")); - - // Delayed loading - parms.add(hutil::ParmFactory(PRM_TOGGLE, "delayload", "Delay Loading") - .setDefault(PRMoneDefaults) - .setTooltip( - "Don't allocate memory for or read voxel values until the values" - " are actually accessed.\n\n" - "Delayed loading can significantly lower memory usage, but\n" - "note that viewport visualization of a volume usually requires\n" - "the entire volume to be loaded into memory.")); - - // Localization file size slider - parms.add(hutil::ParmFactory(PRM_FLT_J, "copylimit", "Copy If Smaller Than") + // Obsolete parameters + hutil::ParmList obsoleteParms; + obsoleteParms.add(hutil::ParmFactory(PRM_SEPARATOR, "sep1", "Sep")); + obsoleteParms.add(hutil::ParmFactory(PRM_TOGGLE, "delayload", "Delay Loading") + .setDefault(PRMoneDefaults)); + obsoleteParms.add(hutil::ParmFactory(PRM_FLT_J, "copylimit", "Copy If Smaller Than") .setTypeExtended(PRM_TYPE_JOIN_PAIR) - .setDefault(0.5f) - .setRange(PRM_RANGE_RESTRICTED, 0, PRM_RANGE_UI, 10) - .setTooltip( - "When delayed loading is enabled, a file must not be modified on disk before\n" - "it has been fully read. For safety, files smaller than the given size (in GB)\n" - "will be copied to a private, temporary location (either $OPENVDB_TEMP_DIR,\n" - "$TMPDIR or a system default temp directory).") - .setDocumentation( - "When delayed loading is enabled, a file must not be modified on disk before" - " it has been fully read. For safety, files smaller than the given size (in GB)" - " will be copied to a private, temporary location (either `$OPENVDB_TEMP_DIR`," - " `$TMPDIR` or a system default temp directory).")); - - parms.add(hutil::ParmFactory(PRM_LABEL, "copylimitlabel", "GB") - .setDocumentation(nullptr)); + .setDefault(0.5f)); + obsoleteParms.add(hutil::ParmFactory(PRM_LABEL, "copylimitlabel", "GB")); // Register this operator. hvdb::OpenVDBOpFactory("VDB Read", SOP_OpenVDB_Read::factory, parms, *table) .setNativeName("") + .setObsoleteParms(obsoleteParms) .addOptionalInput("Optional Bounding Geometry") .setDocumentation("\ #icon: COMMON/openvdb\n\ @@ -248,13 +228,7 @@ newSopOperator(OP_OperatorTable* table) @overview\n\ \n\ This node reads VDB volumes from a `.vdb` file.\n\ -It is usually preferable to use Houdini's native [File|Node:sop/file] node,\n\ -however unlike the native node, this node allows one to take advantage of\n\ -delayed loading, meaning that only those portions of a volume that are\n\ -actually accessed in a scene get loaded into memory.\n\ -Delayed loading can significantly reduce memory usage when working\n\ -with large volumes (but note that viewport visualization of a volume\n\ -usually requires the entire volume to be loaded into memory).\n\ +It is usually preferable to use Houdini's native [File|Node:sop/file] node.\n\ \n\ @related\n\ - [OpenVDB Write|Node:sop/DW_OpenVDBWrite]\n\ @@ -297,10 +271,6 @@ SOP_OpenVDB_Read::updateParmsFlags() changed |= enableParm("group", bool(evalInt("enable_grouping", 0, t))); - const bool delayedLoad = evalInt("delayload", 0, t); - changed |= enableParm("copylimit", delayedLoad); - changed |= enableParm("copylimitlabel", delayedLoad); - return changed; } @@ -344,10 +314,6 @@ SOP_OpenVDB_Read::cookVDBSop(OP_Context& context) //} } - const bool delayedLoad = evalInt("delayload", 0, t); - const openvdb::Index64 copyMaxBytes = - openvdb::Index64(1.0e9 * evalFloat("copylimit", 0, t)); - openvdb::BBoxd clipBBox; bool clip = evalInt("clip", 0, t); if (clip) { @@ -370,8 +336,7 @@ SOP_OpenVDB_Read::cookVDBSop(OP_Context& context) openvdb::MetaMap::Ptr fileMetadata; try { // Open the VDB file, but don't read any grids yet. - file.setCopyMaxBytes(copyMaxBytes); - file.open(delayedLoad); + file.open(); // Read the file-level metadata. fileMetadata = file.getMetadata(); diff --git a/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc b/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc index 89c808f406..a14b7b514a 100644 --- a/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc +++ b/openvdb_houdini/openvdb_houdini/VRAY_OpenVDB_Points.cc @@ -351,7 +351,7 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) { struct Local { - static GridVecPtr loadGrids(const std::string& filename, const bool stream) + static GridVecPtr loadGrids(const std::string& filename) { GridVecPtr grids; @@ -367,11 +367,6 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) if (baseGrid->isType()) { auto grid = StaticPtrCast(file.readGrid(*iter)); assert(grid); - if (stream) { - // enable streaming mode to auto-collapse attributes - // on read for improved memory efficiency - points::setStreamingMode(grid->tree(), /*on=*/true); - } grids.push_back(grid); } } @@ -389,8 +384,6 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) import("file", mFilename); - int streamData; - import("streamdata", &streamData, 1); import("attrmask", mAttrStr); float fps; @@ -430,7 +423,7 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) } } - mGridPtrs = Local::loadGrids(mFilename.toStdString(), streamData ? true : false); + mGridPtrs = Local::loadGrids(mFilename.toStdString()); // extract which groups to include and exclude UT_StringHolder groupStr; @@ -447,11 +440,6 @@ VRAY_OpenVDB_Points::initialize(const UT_BoundingBox *) static_cast(vdbBox.max().y()), static_cast(vdbBox.max().z())); - // if streaming the data, re-open the file now that the bounding box has been computed - if (streamData) { - mGridPtrs = Local::loadGrids(mFilename.toStdString(), true); - } - return 1; } From 0f0143f464c6cf1d212850ebcfb724647ae4f536 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 11 Feb 2026 18:01:51 -0800 Subject: [PATCH 05/80] Delete wireframe rendering, this is only useful for delayed-loading Signed-off-by: Dan Bailey --- .../openvdb_houdini/GR_PrimVDBPoints.cc | 183 +----------------- 1 file changed, 1 insertion(+), 182 deletions(-) diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index b1fa29ba0a..4f93d05033 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -48,7 +48,6 @@ static RE_ShaderHandle theMarkerDecorShader("decor/GL32/point_marker.prog"); static RE_ShaderHandle theNormalDecorShader("decor/GL32/point_normal.prog"); static RE_ShaderHandle theVelocityDecorShader("decor/GL32/user_point_vector3.prog"); -static RE_ShaderHandle theLineShader("basic/GL32/wire_color.prog"); static RE_ShaderHandle thePixelShader("particle/GL32/pixel.prog"); static RE_ShaderHandle thePointShader("particle/GL32/point.prog"); @@ -145,10 +144,6 @@ class GR_PrimVDBPoints : public GR_Primitive const openvdb::points::PointDataGrid& grid, const RE_CacheVersion& version); - void updateWireBuffer(RE_Render* r, - const openvdb::points::PointDataGrid& grid, - const RE_CacheVersion& version); - bool updateVec3Buffer(RE_Render* r, const openvdb::points::PointDataGrid& grid, const std::string& attributeName, @@ -164,7 +159,6 @@ class GR_PrimVDBPoints : public GR_Primitive private: UT_UniquePtr myGeo; - UT_UniquePtr myWire; bool mDefaultPointColor = true; openvdb::Vec3f mCentroid{0, 0, 0}; openvdb::BBoxd mBbox; @@ -354,84 +348,6 @@ GR_PrimVDBPoints::acceptPrimitive(GT_PrimitiveType, return GR_NOT_PROCESSED; } -namespace gr_primitive_internal -{ - -struct FillGPUBuffersLeafBoxes -{ - FillGPUBuffersLeafBoxes(UT_Vector3H* buffer, - const std::vector& coords, - const openvdb::math::Transform& transform, - const openvdb::Vec3f& positionOffset) - : mBuffer(buffer) - , mCoords(coords) - , mTransform(transform) - , mPositionOffset(positionOffset) { } - - void operator()(const tbb::blocked_range& range) const - { - std::vector corners; - corners.reserve(8); - - for (size_t n = range.begin(), N = range.end(); n != N; ++n) { - const openvdb::Coord& origin = mCoords[n]; - - // define 8 corners - - corners.clear(); - - const openvdb::Vec3f pos000 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 0.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos000.x(), pos000.y(), pos000.z()); - const openvdb::Vec3f pos001 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 0.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos001.x(), pos001.y(), pos001.z()); - const openvdb::Vec3f pos010 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 8.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos010.x(), pos010.y(), pos010.z()); - const openvdb::Vec3f pos011 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 8.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos011.x(), pos011.y(), pos011.z()); - const openvdb::Vec3f pos100 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 0.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos100.x(), pos100.y(), pos100.z()); - const openvdb::Vec3f pos101 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 0.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos101.x(), pos101.y(), pos101.z()); - const openvdb::Vec3f pos110 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 8.0, 0.0)) - mPositionOffset; - corners.emplace_back(pos110.x(), pos110.y(), pos110.z()); - const openvdb::Vec3f pos111 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 8.0, 8.0)) - mPositionOffset; - corners.emplace_back(pos111.x(), pos111.y(), pos111.z()); - - openvdb::Index64 offset = n*8*3; - - // Z axis - - mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[1]; - mBuffer[offset++] = corners[2]; mBuffer[offset++] = corners[3]; - mBuffer[offset++] = corners[4]; mBuffer[offset++] = corners[5]; - mBuffer[offset++] = corners[6]; mBuffer[offset++] = corners[7]; - - // Y axis - - mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[2]; - mBuffer[offset++] = corners[1]; mBuffer[offset++] = corners[3]; - mBuffer[offset++] = corners[4]; mBuffer[offset++] = corners[6]; - mBuffer[offset++] = corners[5]; mBuffer[offset++] = corners[7]; - - // X axis - - mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[4]; - mBuffer[offset++] = corners[1]; mBuffer[offset++] = corners[5]; - mBuffer[offset++] = corners[2]; mBuffer[offset++] = corners[6]; - mBuffer[offset++] = corners[3]; mBuffer[offset++] = corners[7]; - } - } - - ////////// - - UT_Vector3H* mBuffer; - const std::vector& mCoords; - const openvdb::math::Transform& mTransform; - const openvdb::Vec3f mPositionOffset; -}; // class FillGPUBuffersLeafBoxes - -} // namespace gr_primitive_internal - void GR_PrimVDBPoints::computeCentroid(const openvdb::points::PointDataGrid& grid) { @@ -631,78 +547,6 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, myGeo->connectAllPrims(r, RE_GEO_WIRE_IDX, primType, nullptr, true); } -void -GR_PrimVDBPoints::updateWireBuffer(RE_Render *r, - const openvdb::points::PointDataGrid& grid, - const RE_CacheVersion& version) -{ - const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); - - // Initialize the geometry with the proper name for the GL cache - if (!myWire) myWire.reset(new RE_Geometry); - myWire->cacheBuffers(getCacheName()); - - using GridType = openvdb::points::PointDataGrid; - using TreeType = GridType::TreeType; - using LeafNode = TreeType::LeafNodeType; - - const TreeType& tree = grid.tree(); - size_t leafCount = tree.leafCount(); - if (tree.leafCount() == 0) return; - - // Initialize the number of points for the wireframe box per leaf. - - int numPoints = static_cast(leafCount*8*3); - myWire->setNumPoints(numPoints); - - // fetch wireframe position, if its cache version matches, no upload is required. - - RE_VertexArray* posWire = myWire->findCachedAttrib(r, "P", RE_GPU_FLOAT16, 3, RE_ARRAY_POINT, true); - - if (posWire->getCacheVersion() != version) - { - using gr_primitive_internal::FillGPUBuffersLeafBoxes; - - // fill the wire data - - UT_UniquePtr data(new UT_Vector3H[numPoints]); - - std::vector coords; - - for (auto iter = tree.cbeginLeaf(); iter; ++iter) { - const LeafNode& leaf = *iter; - - coords.push_back(leaf.origin()); - } - - FillGPUBuffersLeafBoxes fill(data.get(), coords, grid.transform(), mCentroid); - const tbb::blocked_range range(0, coords.size()); - tbb::parallel_for(range, fill); - - posWire->setArray(r, data.get()); - posWire->setCacheVersion(version); - } - - if (gl3) - { - // Extra constant inputs for the GL3 default shaders we are using. - - fpreal32 uv[2] = { 0.0, 0.0 }; - fpreal32 alpha = 1.0; - fpreal32 pnt = 0.0; - UT_Matrix4F instance; - instance.identity(); - - myWire->createConstAttribute(r, "uv", RE_GPU_FLOAT32, 2, uv); - myWire->createConstAttribute(r, "Alpha", RE_GPU_FLOAT32, 1, &alpha); - myWire->createConstAttribute(r, "pointSelection", RE_GPU_FLOAT32, 1,&pnt); - myWire->createConstAttribute(r, "instmat", RE_GPU_MATRIX4, 1, - instance.data()); - } - - myWire->connectAllPrims(r, RE_GEO_WIRE_IDX, RE_PRIM_LINES, nullptr, true); -} - void GR_PrimVDBPoints::update(RE_RenderContext r, const GT_PrimitiveHandle &primh, @@ -710,7 +554,6 @@ GR_PrimVDBPoints::update(RE_RenderContext r, { // patch the point shaders at run-time to add an offset (does nothing if already patched) - patchShaderVertexOffset(r, theLineShader); patchShaderVertexOffset(r, thePixelShader); patchShaderVertexOffset(r, thePointShader); @@ -738,7 +581,6 @@ GR_PrimVDBPoints::update(RE_RenderContext r, computeCentroid(pointDataGrid); computeBbox(pointDataGrid); updatePosBuffer(r, pointDataGrid, p.geo_version); - updateWireBuffer(r, pointDataGrid, p.geo_version); mDefaultPointColor = !updateVec3Buffer(r, pointDataGrid, "Cd", "Cd", p.geo_version); } @@ -856,7 +698,7 @@ GR_PrimVDBPoints::removeBuffer(const std::string& name) void GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_DrawParms dp) { - if (!myGeo && !myWire) return; + if (!myGeo) return; const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); @@ -900,29 +742,6 @@ GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_D r->popShader(); } - - // draw leaf bboxes - - if (myWire && myWire->getNumPoints() > 0) { - - // bind the shader - - r->pushShader(); - r->bindShader(theLineShader); - - // bind the position offset - - UT_Vector3F positionOffset(mCentroid.x(), mCentroid.y(), mCentroid.z()); - theLineShader->bindVector(r, "offset", positionOffset); - - fpreal32 constcol[3] = { 0.6f, 0.6f, 0.6f }; - myWire->createConstAttribute(r, "Cd", RE_GPU_FLOAT32, 3, constcol); - - r->pushLineWidth(commonOpts.wireWidth()); - myWire->draw(r, RE_GEO_WIRE_IDX); - r->popLineWidth(); - r->popShader(); - } } From 759482d158666bef5c8993b8c88873cb865a92d7 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 11 Feb 2026 18:58:39 -0800 Subject: [PATCH 06/80] Fix the patching of marker decor shaders for recent Houdini releases Signed-off-by: Dan Bailey --- openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index 4f93d05033..6ffbed0fa8 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -246,10 +246,15 @@ bool patchShader(RE_Render* r, RE_ShaderHandle& shader, RE_ShaderType type, shader->getShaderSource(r, source, type); const int version = shader->getCodeVersion(); + // normalize whitespace (collapse tabs and runs of spaces to single space) + + source.substitute("\t", " "); + while (source.substitute(" ", " ")) {} + // patch the shader to replace the strings for (const auto& stringPair : stringReplacements) { - source.substitute(stringPair.first.c_str(), stringPair.second.c_str(), /*all=*/true); + source.substitute(stringPair.first.c_str(), stringPair.second.c_str()); } // patch the shader to insert the strings @@ -306,9 +311,6 @@ void patchShaderNoRedeclarations(RE_Render* r, RE_ShaderHandle& shader) { static const std::vector stringReplacements { - StringPair("\t", " "), - StringPair(" ", " "), - StringPair(" ", " "), StringPair("uniform vec2 glH_DepthProject;", "//uniform vec2 glH_DepthProject;"), StringPair("uniform vec2 glH_ScreenSize", "//uniform vec2 glH_ScreenSize") }; From eac8cc0208785b6615b277847d8a0be70bb2d1e3 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 12 Feb 2026 15:27:29 -0800 Subject: [PATCH 07/80] Remove support for Houdini < 20.0 Signed-off-by: Dan Bailey --- .../openvdb_houdini/GR_PrimVDBPoints.cc | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index 6ffbed0fa8..a86869e993 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -7,8 +7,6 @@ /// /// @brief GR Render Hook and Primitive for VDB PointDataGrid -#include - #include #include #include @@ -39,10 +37,6 @@ #include #include -#if UT_VERSION_INT < 0x14000000 // Below 20.0, there is no RE_RenderContext -#define RE_RenderContext RE_Render * -#endif - //////////////////////////////////////// static RE_ShaderHandle theMarkerDecorShader("decor/GL32/point_marker.prog"); @@ -119,11 +113,8 @@ class GR_PrimVDBPoints : public GR_Primitive /// return true if the primitive is in or overlaps the view frustum. /// always returning true will effectively disable frustum culling. - bool inViewFrustum(const UT_Matrix4D &objviewproj -#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later - , const UT_BoundingBoxD *bbox -#endif - ) override; + bool inViewFrustum(const UT_Matrix4D &objviewproj, + const UT_BoundingBoxD *bbox) override; /// Called whenever the primitive is required to render, which may be more /// than one time per viewport redraw (beauty, shadow passes, wireframe-over) @@ -589,20 +580,13 @@ GR_PrimVDBPoints::update(RE_RenderContext r, } bool -GR_PrimVDBPoints::inViewFrustum(const UT_Matrix4D& objviewproj -#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later - , const UT_BoundingBoxD *passed_bbox -#endif - ) +GR_PrimVDBPoints::inViewFrustum(const UT_Matrix4D& objviewproj, + const UT_BoundingBoxD *passed_bbox) { const UT_BoundingBoxD bbox( mBbox.min().x(), mBbox.min().y(), mBbox.min().z(), mBbox.max().x(), mBbox.max().y(), mBbox.max().z()); -#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later - return GR_Utils::inViewFrustum(passed_bbox ? *passed_bbox :bbox, + return GR_Utils::inViewFrustum(passed_bbox ? *passed_bbox : bbox, objviewproj); -#else - return GR_Utils::inViewFrustum(bbox, objviewproj); -#endif } bool From 94fafd52a585e9a7de4481a5f27decdc0b71340a Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 12 Feb 2026 15:29:19 -0800 Subject: [PATCH 08/80] Add Vulkan support Signed-off-by: Dan Bailey --- .../openvdb_houdini/CMakeLists.txt | 5 + .../openvdb_houdini/GR_PrimVDBPoints.cc | 517 +++++++++++++++++- .../glsl/openvdb/VK/points.frag | 28 + .../glsl/openvdb/VK/points.prog | 6 + .../glsl/openvdb/VK/points.vert | 28 + .../glsl/openvdb/VK/velocity.frag | 26 + .../glsl/openvdb/VK/velocity.prog | 8 + .../glsl/openvdb/VK/velocity.tcs | 46 ++ .../glsl/openvdb/VK/velocity.tes | 23 + .../glsl/openvdb/VK/velocity.vert | 30 + 10 files changed, 713 insertions(+), 4 deletions(-) create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes create mode 100644 openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert diff --git a/openvdb_houdini/openvdb_houdini/CMakeLists.txt b/openvdb_houdini/openvdb_houdini/CMakeLists.txt index 50d9ed4cab..1a8ae34dd0 100644 --- a/openvdb_houdini/openvdb_houdini/CMakeLists.txt +++ b/openvdb_houdini/openvdb_houdini/CMakeLists.txt @@ -123,6 +123,8 @@ set(OPENVDB_HOUDINI_ICON_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/config CACHE PATH "Install path for the OpenVDB Houdini node icons") set(OPENVDB_HOUDINI_PYTHON_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/python2.7libs CACHE PATH "Install path for the OpenVDB Houdini startup script") +set(OPENVDB_HOUDINI_GLSL_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/glsl + CACHE PATH "Install path for the OpenVDB Houdini GLSL shaders") ######################################################################### @@ -395,6 +397,9 @@ install(FILES install(DIRECTORY help/ DESTINATION ${OPENVDB_HOUDINI_HELP_INSTALL_PREFIX}) +install(DIRECTORY glsl/ + DESTINATION ${OPENVDB_HOUDINI_GLSL_INSTALL_PREFIX}) + if(OPENVDB_INSTALL_HOUDINI_PYTHONRC) install(FILES pythonrc.py diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index a86869e993..c4f45b7272 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -28,6 +28,18 @@ #include #include #include +#include + +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#include +#include +#include +#include +#include +#include +#include +#include +#endif #include #include @@ -45,6 +57,11 @@ static RE_ShaderHandle theVelocityDecorShader("decor/GL32/user_point_vector3.pro static RE_ShaderHandle thePixelShader("particle/GL32/pixel.prog"); static RE_ShaderHandle thePointShader("particle/GL32/point.prog"); +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +static RV_ShaderProgram* theVkPointShader = nullptr; +static RV_ShaderProgram* theVkVelocityShader = nullptr; +#endif + /// @note An additional scale for velocity trails to accurately match /// the visualization of velocity for Houdini points #define VELOCITY_DECOR_SCALE -0.041f; @@ -148,11 +165,35 @@ class GR_PrimVDBPoints : public GR_Primitive void removeBuffer(const std::string& name); +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + void updatePosBufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const RE_CacheVersion& version); + + bool updateVec3BufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const std::string& attributeName, + const std::string& bufferName, + const RE_CacheVersion& version); +#endif + private: UT_UniquePtr myGeo; bool mDefaultPointColor = true; openvdb::Vec3f mCentroid{0, 0, 0}; openvdb::BBoxd mBbox; +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + UT_UniquePtr myGeoVk; + UT_UniquePtr myVkObjectSet; + UT_UniquePtr myVkObjectBlock; + UT_UniquePtr myVkDrawingSet; + UT_UniquePtr myVkGeoBlock; + // velocity shader uniform state (separate to avoid clobbering point shader) + UT_UniquePtr myVkVelObjectSet; + UT_UniquePtr myVkVelObjectBlock; + bool mHasNormals = false; + bool mHasVelocity = false; +#endif }; @@ -439,12 +480,49 @@ struct VectorAttribute UT_Vector3H* mBuffer; }; // struct VectorAttribute +struct VectorAttribute4H +{ + using ValueType = Vec3f; + + struct Handle + { + Handle(VectorAttribute4H& attribute) + : mBuffer(attribute.mBuffer) { } + + template + void set(openvdb::Index offset, + openvdb::Index /*stride*/, + const openvdb::math::Vec3& value) + { + mBuffer[offset] = UT_Vector4H( + fpreal16(float(value.x())), + fpreal16(float(value.y())), + fpreal16(float(value.z())), + fpreal16(0)); + } + + private: + UT_Vector4H* mBuffer; + }; // struct Handle + + VectorAttribute4H(UT_Vector4H* buffer) + : mBuffer(buffer) { } + + void expand() { } + void compact() { } + +private: + UT_Vector4H* mBuffer; +}; // struct VectorAttribute4H + void GR_PrimVDBPoints::updatePosBuffer(RE_Render* r, const openvdb::points::PointDataGrid& grid, const RE_CacheVersion& version) { - const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); + const GR_RenderVersion renderVersion = getRenderVersion(); + const bool gl3 = (renderVersion == GR_RENDER_GL3 + || renderVersion == GR_RENDER_GL4); // Initialize the geometry with the proper name for the GL cache if (!myGeo) myGeo.reset(new RE_Geometry); @@ -545,6 +623,36 @@ GR_PrimVDBPoints::update(RE_RenderContext r, const GT_PrimitiveHandle &primh, const GR_UpdateParms &p) { +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + if (r.isVulkan()) + { + if (p.reason & (GR_GEO_CHANGED | GR_GEO_TOPOLOGY_CHANGED)) + { + const GT_PrimVDB& gt_primVDB = + static_cast(*primh); + + const openvdb::GridBase* grid = + const_cast(gt_primVDB).getGrid(); + + const openvdb::points::PointDataGrid& pointDataGrid = + static_cast(*grid); + + computeCentroid(pointDataGrid); + computeBbox(pointDataGrid); + + RV_Render* rv = r.vkRender(); + updatePosBufferVk(rv, pointDataGrid, p.geo_version); + mDefaultPointColor = !updateVec3BufferVk( + rv, pointDataGrid, "Cd", "Cd", p.geo_version); + updateVec3BufferVk( + rv, pointDataGrid, "N", "N", p.geo_version); + updateVec3BufferVk( + rv, pointDataGrid, "v", "V", p.geo_version); + } + return; + } +#endif + // patch the point shaders at run-time to add an offset (does nothing if already patched) patchShaderVertexOffset(r, thePixelShader); @@ -681,14 +789,294 @@ GR_PrimVDBPoints::removeBuffer(const std::string& name) myGeo->clearAttribute(name.c_str()); } + +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + +void +GR_PrimVDBPoints::updatePosBufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const RE_CacheVersion& version) +{ + using GridType = openvdb::points::PointDataGrid; + using TreeType = GridType::TreeType; + using AttributeSet = openvdb::points::AttributeSet; + + const TreeType& tree = grid.tree(); + auto iter = tree.cbeginLeaf(); + if (!iter) return; + + const AttributeSet::Descriptor& descriptor = + iter->attributeSet().descriptor(); + + // check if group viewport is in use + + const openvdb::StringMetadata::ConstPtr s = + grid.getMetadata( + openvdb_houdini::META_GROUP_VIEWPORT); + + const std::string groupName = s ? s->value() : ""; + const bool useGroup = + !groupName.empty() && descriptor.hasGroup(groupName); + + // count up total points + + int numPoints = 0; + if (useGroup) { + GroupFilter filter(groupName, iter->attributeSet()); + numPoints = static_cast(pointCount(tree, filter)); + } else { + NullFilter filter; + numPoints = static_cast(pointCount(tree, filter)); + } + + if (numPoints == 0) return; + + const size_t positionIndex = descriptor.find("P"); + if (positionIndex == AttributeSet::INVALID_POS) return; + + // check for decoration attributes + + mHasNormals = (descriptor.find("N") != AttributeSet::INVALID_POS); + mHasVelocity = (descriptor.find("v") != AttributeSet::INVALID_POS); + + // (re)create the VK geometry + + myGeoVk.reset(new RV_Geometry); + myGeoVk->setName("vdb_points"); + myGeoVk->setNumPoints(numPoints); + myGeoVk->createAttribute("P", RV_GPU_FLOAT32, 3); + myGeoVk->createAttribute("Cd", RV_GPU_FLOAT16, 4); + if (mHasNormals) + myGeoVk->createAttribute("N", RV_GPU_FLOAT16, 4); + if (mHasVelocity) + myGeoVk->createAttribute("V", RV_GPU_FLOAT16, 4); + myGeoVk->createConstant("pointSelection", RV_GPU_UINT32, 1); + myGeoVk->connectAllPrims(0, RV_PRIM_POINTS); + if (mHasVelocity) + myGeoVk->connectAllPrims(1, RV_PRIM_PATCHES, /*patch_size=*/1); + myGeoVk->populateBuffers(r); + + // set pointSelection to zero (no selection) + + myGeoVk->setAttributeConstValue("pointSelection", 0.0); + + // fill the position buffer (world-space, no offset subtraction) + + { + std::vector includeGroups, excludeGroups; + if (useGroup) includeGroups.emplace_back(groupName); + + MultiGroupFilter filter( + includeGroups, excludeGroups, iter->attributeSet()); + + std::vector offsets; + pointOffsets(offsets, grid.tree(), filter); + + UT_UniquePtr pdata(new UT_Vector3F[numPoints]); + + // use zero offset so positions are stored in world space + PositionAttribute positionAttribute( + pdata.get(), Vec3f(0, 0, 0)); + convertPointDataGridPosition( + positionAttribute, grid, offsets, + /*startOffset=*/ 0, filter); + + const exint byteSize = + static_cast(numPoints) * sizeof(UT_Vector3F); + myGeoVk->getAttribute("P")->uploadData(r, pdata.get(), byteSize); + } + + // set a default Cd (will be overwritten if Cd attribute exists) + + myGeoVk->setAttributeConstVecValue( + "Cd", UT_Vector4F(0.6f, 0.6f, 0.5f, 1.0f)); + +} + + +bool +GR_PrimVDBPoints::updateVec3BufferVk(RV_Render* r, + const openvdb::points::PointDataGrid& grid, + const std::string& attributeName, + const std::string& bufferName, + const RE_CacheVersion& version) +{ + if (!myGeoVk) return false; + + using GridType = openvdb::points::PointDataGrid; + using TreeType = GridType::TreeType; + using AttributeSet = openvdb::points::AttributeSet; + + const TreeType& tree = grid.tree(); + auto iter = tree.cbeginLeaf(); + if (!iter) return false; + + const int numPoints = static_cast(myGeoVk->getNumPoints()); + if (numPoints == 0) return false; + + const AttributeSet::Descriptor& descriptor = + iter->attributeSet().descriptor(); + const size_t index = descriptor.find(attributeName); + + if (index == AttributeSet::INVALID_POS) return false; + + const openvdb::Name& type = descriptor.type(index).first; + if (type != "vec3s") return false; + + // check if group viewport is in use + + const openvdb::StringMetadata::ConstPtr s = + grid.getMetadata( + openvdb_houdini::META_GROUP_VIEWPORT); + + const std::string groupName = s ? s->value() : ""; + const bool useGroup = + !groupName.empty() && descriptor.hasGroup(groupName); + + std::vector includeGroups, excludeGroups; + if (useGroup) includeGroups.emplace_back(groupName); + + MultiGroupFilter filter( + includeGroups, excludeGroups, iter->attributeSet()); + + std::vector offsets; + pointOffsets(offsets, grid.tree(), filter); + + // read vec3s attribute data directly into 4-component float16 + // (VK_FORMAT_R16G16B16A16_SFLOAT is mandatory; 3-component is not) + + UT_UniquePtr data(new UT_Vector4H[numPoints]); + + VectorAttribute4H typedAttribute(data.get()); + convertPointDataGridAttribute(typedAttribute, grid.tree(), offsets, + /*startOffset=*/ 0, static_cast(index), + /*stride=*/1, filter); + + const exint byteSize = + static_cast(numPoints) * sizeof(UT_Vector4H); + RV_VKBuffer* buf = myGeoVk->getAttribute(bufferName.c_str()); + if (buf) { + buf->uploadData(r, data.get(), byteSize); + } + + return true; +} + +#endif // UT_VERSION_INT >= 0x15000000 + + void GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_DrawParms dp) { - if (!myGeo) return; +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + if (r.isVulkan()) + { + if (!myGeoVk) return; + + RV_Render* rv = r.vkRender(); + GR_Uniforms* uniforms = r.uniforms(); + + // initialize the VK point shader on first use + + if (!theVkPointShader) { + theVkPointShader = RV_ShaderProgram::loadShaderProgram( + rv->instance(), "openvdb/VK/points.prog"); + if (!theVkPointShader) return; + } - const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3); + // set the shader + + rv->setShader(theVkPointShader); + + // bind global uniform blocks (pass info, object transforms) + + if (uniforms) { + bool globalBound = uniforms->bindRVGlobalBlock(rv, theVkPointShader); + + // bind set 1 (glH_Object) for per-object transform uniforms + if (theVkPointShader->hasSet(1)) { + const auto* objBinding = + theVkPointShader->getBinding(1, 0); + if (objBinding) { + if (!myVkObjectBlock) { + myVkObjectBlock.reset( + RV_ShaderBlock::create( + rv->instance(), *objBinding)); + } + if (!myVkObjectSet || + !theVkPointShader->isSetCompatible( + *myVkObjectSet)) { + myVkObjectSet = + theVkPointShader->createSet( + rv->instance(), 1); + } + uniforms->assignRVBlock( + rv, myVkObjectBlock.get(), theVkPointShader); + // override DecorationScale with the user's + // point size from display options + const GR_CommonDispOption& commonOpts = + dp.opts->common(); + myVkObjectBlock->bindFloat( + "DecorationScale", + static_cast(commonOpts.pointSize())); + myVkObjectBlock->uploadBuffer(rv); + myVkObjectSet->attachBufferBlock( + rv->instance(), "Object", + myVkObjectBlock.get()); + rv->bindSet( + myVkObjectSet.get(), theVkPointShader); + } + } + + // bind set 2 (auto-injected Geometry uniform block) + if (theVkPointShader->hasSet(2)) { + const auto* geoBinding = + theVkPointShader->getBinding(2, 2); + if (geoBinding) { + if (!myVkGeoBlock) { + myVkGeoBlock.reset( + RV_ShaderBlock::create( + rv->instance(), *geoBinding)); + } + if (!myVkDrawingSet || + !theVkPointShader->isSetCompatible( + *myVkDrawingSet)) { + myVkDrawingSet = + theVkPointShader->createSet( + rv->instance(), 2); + } + myVkGeoBlock->uploadBuffer(rv); + myVkDrawingSet->attachBufferBlock( + rv->instance(), "Geometry", + myVkGeoBlock.get()); + rv->bindSet( + myVkDrawingSet.get(), theVkPointShader); + } + } + + } + + // draw - must be inside beginRendering/endRendering block + + bool beganRendering = false; + if (!rv->isRendering()) { + beganRendering = rv->beginRendering(); + } - if (!gl3) return; + rv->draw(myGeoVk.get(), 0); + + if (beganRendering) { + rv->endRendering(); + } + return; + } +#endif + + if (!myGeo) return; + + const GR_RenderVersion renderVersion = getRenderVersion(); + if (renderVersion != GR_RENDER_GL3 && renderVersion != GR_RENDER_GL4) + return; const GR_CommonDispOption& commonOpts = dp.opts->common(); @@ -734,6 +1122,127 @@ GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_D void GR_PrimVDBPoints::renderDecoration(RE_RenderContext r, GR_Decoration decor, const GR_DecorationParms& p) { +#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support + if (r.isVulkan()) + { + if (!myGeoVk) return; + + if (decor == GR_POINT_MARKER) + { + drawDecorationForGeo(r, myGeoVk.get(), decor, p.opts, + GR_DECOR_RENDER_FLAG_NONE, + /*overlay=*/false, /*override_vis=*/false, + /*instance_group=*/-1, GR_SELECT_NONE, + GR_DecorationRender::PRIM_POINT); + } + else if (decor == GR_POINT_NORMAL && mHasNormals) + { + drawDecorationForGeo(r, myGeoVk.get(), decor, p.opts, + GR_DECOR_RENDER_FLAG_NONE, + /*overlay=*/false, /*override_vis=*/false, + /*instance_group=*/-1, GR_SELECT_NONE, + GR_DecorationRender::PRIM_POINT); + } + else if (decor == GR_POINT_VELOCITY && mHasVelocity) + { + // GPU-based velocity trail rendering using tessellation. + // The VK decoration system requires GR_GeoRenderVK for its + // built-in tessellation shaders, which custom GR_Primitive + // subclasses don't have. Instead, we use a custom velocity + // shader that reads P and V as vertex inputs and generates + // isolines via tessellation on the GPU. + + RV_Render* rv = r.vkRender(); + GR_Uniforms* uniforms = r.uniforms(); + + // load the velocity tessellation shader on first use + + if (!theVkVelocityShader) { + theVkVelocityShader = + RV_ShaderProgram::loadShaderProgram( + rv->instance(), + "openvdb/VK/velocity.prog"); + if (!theVkVelocityShader) return; + } + + rv->setShader(theVkVelocityShader); + + // compute velocity scale and trail color + + const GR_CommonDispOption& commonOpts = + p.opts->common(); + const float velocityScale = + static_cast(commonOpts.vectorScale()) + * -0.041f; + + UT_Color trailColor = + commonOpts.getColor(GR_POINT_TRAIL_COLOR); + float cr, cg, cb; + trailColor.getRGB(&cr, &cg, &cb); + + // bind uniform blocks + + if (uniforms) { + uniforms->bindRVGlobalBlock( + rv, theVkVelocityShader); + + // bind set 1 (glH_Object) with velocity overrides + if (theVkVelocityShader->hasSet(1)) { + const auto* objBinding = + theVkVelocityShader->getBinding(1, 0); + if (objBinding) { + if (!myVkVelObjectBlock) { + myVkVelObjectBlock.reset( + RV_ShaderBlock::create( + rv->instance(), + *objBinding)); + } + if (!myVkVelObjectSet || + !theVkVelocityShader->isSetCompatible( + *myVkVelObjectSet)) { + myVkVelObjectSet = + theVkVelocityShader->createSet( + rv->instance(), 1); + } + // fill with viewport transforms + uniforms->assignRVBlock(rv, + myVkVelObjectBlock.get(), + theVkVelocityShader); + // override decoration scale and wire color + myVkVelObjectBlock->bindFloat( + "DecorationScale", velocityScale); + myVkVelObjectBlock->bindVector( + "WireColor", + UT_Vector4F(cr, cg, cb, 1.0f)); + myVkVelObjectBlock->uploadBuffer(rv); + myVkVelObjectSet->attachBufferBlock( + rv->instance(), "Object", + myVkVelObjectBlock.get()); + rv->bindSet(myVkVelObjectSet.get(), + theVkVelocityShader); + } + } + } + + // draw using PATCHES connection group (index 1) + + bool beganRendering = false; + if (!rv->isRendering()) { + beganRendering = rv->beginRendering(); + } + + rv->draw(myGeoVk.get(), 1); + + if (beganRendering) { + rv->endRendering(); + } + } + return; + } +#endif + + if (!myGeo) return; + // just render native GR_Primitive decorations if position not available const RE_VertexArray* const position = myGeo->getAttribute("P"); diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag new file mode 100644 index 0000000000..22229d6310 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.frag @@ -0,0 +1,28 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB Points fragment shader for Vulkan. +// Outputs the interpolated color from the vertex shader. +// Uses gl_PointCoord to discard fragments outside a circular disc +// with smooth anti-aliased edges. + +layout(location = 0) in vec4 pnt_color; + +layout(location = 0) out vec4 color_out; + +void main() +{ + if (pnt_color.a == 0.0) + discard; + + // circular point sprite with anti-aliased edge + vec2 coord = gl_PointCoord * 2.0 - 1.0; + float r2 = dot(coord, coord); + if (r2 > 1.0) + discard; + + // smooth edge over the outermost ~1 pixel + float alpha = 1.0 - smoothstep(0.8, 1.0, r2); + + color_out = vec4(pnt_color.rgb, pnt_color.a * alpha); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog new file mode 100644 index 0000000000..ee21815c92 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.prog @@ -0,0 +1,6 @@ +#name VDB Points +#version 450 +#output color 0 + +points.vert +points.frag diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert new file mode 100644 index 0000000000..5257e48d01 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/points.vert @@ -0,0 +1,28 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// Minimal VDB Points vertex shader for Vulkan. +// Transforms position P by the standard Houdini view/projection matrices, +// passes Cd (color) to the fragment shader. +// +// Uses direct vertex inputs (not ATTRIB macros) because the geometry is +// populated with createAttribute() which creates VBOs, not SSBOs. + +layout(location = 0) in vec3 P; +layout(location = 1) in vec3 Cd; + +layout(location = 0) out vec4 pnt_color; + +layout(set=0, binding=0) +#using glH_PassInfo + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + vec4 pos = glH_Object.ObjView * vec4(P, 1.0); + gl_Position = glH_PassInfo.Projection * pos; + gl_PointSize = glH_Object.DecorationScale; + pnt_color = vec4(Cd, 1.0); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag new file mode 100644 index 0000000000..16d45e8293 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.frag @@ -0,0 +1,26 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration fragment shader. +// Applies a fade effect along the velocity trail using the u parameter +// from the tessellation evaluation shader. Trail color comes from +// glH_Object.WireColor which is set by the C++ code to the trail color. + +layout(location = 0) in float fsU; + +layout(location = 0) out vec4 color_out; + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + vec4 col = glH_Object.WireColor; + + // cubic fade from opaque (tip) to transparent (tail) + float a = 1.0 - fsU; + a = 1.0 - a * a * a; + a *= col.a; + + color_out = vec4(col.rgb, a); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog new file mode 100644 index 0000000000..3470f6fba5 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.prog @@ -0,0 +1,8 @@ +#name VDB Velocity Lines +#version 450 +#output color 0 + +velocity.vert +velocity.tcs +velocity.tes +velocity.frag diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs new file mode 100644 index 0000000000..3bf7e2bfbb --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tcs @@ -0,0 +1,46 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration tessellation control shader. +// Takes a single control point (patch size 1) and computes the +// start and end positions of the velocity vector in clip space. +// Tessellation generates one isoline segment (2 vertices). + +layout(vertices = 1) out; + +layout(location = 0) in parms +{ + vec3 velocity; +} tsIn[]; + +layout(location = 0) out patch nparms +{ + vec4 startPos; + vec4 endPos; +} tsOut; + +layout(set=0, binding=0) +#using glH_PassInfo + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + vec3 vel = tsIn[0].velocity * glH_Object.DecorationScale; + + // start position in clip space + tsOut.startPos = glH_PassInfo.Projection * gl_in[0].gl_Position; + + // end position in clip space (add scaled velocity in view space) + tsOut.endPos = glH_PassInfo.Projection * + (gl_in[0].gl_Position + vec4(vel, 0.0)); + + // one isoline segment = 2 vertices + gl_TessLevelInner[0] = 0.0; + gl_TessLevelInner[1] = 0.0; + gl_TessLevelOuter[0] = 1.0; + gl_TessLevelOuter[1] = 1.0; + gl_TessLevelOuter[2] = 1.0; + gl_TessLevelOuter[3] = 1.0; +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes new file mode 100644 index 0000000000..0eea5f48e4 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.tes @@ -0,0 +1,23 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration tessellation evaluation shader. +// Generates isoline vertices by interpolating between start and end +// positions. The u parameter drives a fade effect in the fragment shader. + +layout(isolines) in; + +layout(location = 0) in patch nparms +{ + vec4 startPos; + vec4 endPos; +} tsIn; + +layout(location = 0) out float fsU; + +void main() +{ + float t = gl_TessCoord.x; + fsU = 1.0 - t; + gl_Position = mix(tsIn.startPos, tsIn.endPos, t); +} diff --git a/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert new file mode 100644 index 0000000000..bc2f0a0a16 --- /dev/null +++ b/openvdb_houdini/openvdb_houdini/glsl/openvdb/VK/velocity.vert @@ -0,0 +1,30 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 +// +// VDB velocity decoration vertex shader for Vulkan. +// Reads P (position) and V (velocity) as vertex inputs. +// Transforms position to view space. Velocity is passed through +// in view space for the tessellation stage to generate isolines. + +layout(location = 0) in vec3 P; +layout(location = 1) in vec3 V; + +layout(location = 0) out parms +{ + vec3 velocity; +} vsOut; + +layout(set=0, binding=0) +#using glH_PassInfo + +layout(set=1, binding=0) +#using glH_Object + +void main() +{ + // position in view space (projection applied in TCS) + gl_Position = glH_Object.ObjView * vec4(P, 1.0); + + // velocity direction in view space + vsOut.velocity = mat3(glH_Object.ObjView) * V; +} From d6a78e79a44ac5315b979210d131e01fd6ef38a2 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 12 Feb 2026 17:59:36 -0800 Subject: [PATCH 09/80] Add USE_VULKAN macro for Mac OSX Signed-off-by: Dan Bailey --- .../openvdb_houdini/GR_PrimVDBPoints.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc index c4f45b7272..7869ab94a7 100644 --- a/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc +++ b/openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc @@ -30,7 +30,7 @@ #include #include -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support #include #include #include @@ -57,7 +57,7 @@ static RE_ShaderHandle theVelocityDecorShader("decor/GL32/user_point_vector3.pro static RE_ShaderHandle thePixelShader("particle/GL32/pixel.prog"); static RE_ShaderHandle thePointShader("particle/GL32/point.prog"); -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support static RV_ShaderProgram* theVkPointShader = nullptr; static RV_ShaderProgram* theVkVelocityShader = nullptr; #endif @@ -165,7 +165,7 @@ class GR_PrimVDBPoints : public GR_Primitive void removeBuffer(const std::string& name); -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support void updatePosBufferVk(RV_Render* r, const openvdb::points::PointDataGrid& grid, const RE_CacheVersion& version); @@ -182,7 +182,7 @@ class GR_PrimVDBPoints : public GR_Primitive bool mDefaultPointColor = true; openvdb::Vec3f mCentroid{0, 0, 0}; openvdb::BBoxd mBbox; -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support UT_UniquePtr myGeoVk; UT_UniquePtr myVkObjectSet; UT_UniquePtr myVkObjectBlock; @@ -623,7 +623,7 @@ GR_PrimVDBPoints::update(RE_RenderContext r, const GT_PrimitiveHandle &primh, const GR_UpdateParms &p) { -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support if (r.isVulkan()) { if (p.reason & (GR_GEO_CHANGED | GR_GEO_TOPOLOGY_CHANGED)) @@ -790,7 +790,7 @@ GR_PrimVDBPoints::removeBuffer(const std::string& name) } -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support void GR_PrimVDBPoints::updatePosBufferVk(RV_Render* r, @@ -962,13 +962,13 @@ GR_PrimVDBPoints::updateVec3BufferVk(RV_Render* r, return true; } -#endif // UT_VERSION_INT >= 0x15000000 +#endif // UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) void GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_DrawParms dp) { -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support if (r.isVulkan()) { if (!myGeoVk) return; @@ -1122,7 +1122,7 @@ GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_D void GR_PrimVDBPoints::renderDecoration(RE_RenderContext r, GR_Decoration decor, const GR_DecorationParms& p) { -#if UT_VERSION_INT >= 0x15000000 // 21.0 or later - Vulkan support +#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support if (r.isVulkan()) { if (!myGeoVk) return; From c48b99f759a8542838da29ce723ab61cec41ee1f Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 12 Feb 2026 17:14:07 -0800 Subject: [PATCH 10/80] Remove unused class members and methods in future ABI=14 Signed-off-by: Dan Bailey --- .github/workflows/weekly.yml | 1 + openvdb/openvdb/Grid.h | 4 ++++ openvdb/openvdb/tree/LeafBuffer.h | 6 +++++- openvdb/openvdb/tree/Tree.h | 4 ++++ openvdb/openvdb/unittest/TestGrid.cc | 2 ++ 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index 614cb37de9..ee35ee2315 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -154,6 +154,7 @@ jobs: - { name: 'asan', image: '2026', build: 'asan', components: 'core,test', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DNANOVDB_USE_OPENVDB=ON -DOPENVDB_AX_STATIC=OFF -DOPENVDB_CORE_STATIC=OFF -DUSE_BLOSC=OFF' } # We never called blosc_destroy(), so disable blosc to silence these errors - { name: 'ubsan', image: '2026', build: 'ubsan', components: 'core,test', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations" ' } - { name: 'c++20', image: '2026', build: 'Release', components: 'core,test', cmake: '-DCMAKE_CXX_STANDARD=20' } + - { name: 'abi14', image: '2026', build: 'Release', components: 'core,test', cmake: '-DOPENVDB_USE_FUTURE_ABI_14=ON -DOPENVDB_ENABLE_ASSERTS=ON' } - { name: 'conf', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON' } fail-fast: false steps: diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index ecb18f9b66..b364e9efc0 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -458,8 +458,10 @@ class OPENVDB_API GridBase: public MetaMap /// Read all of this grid's data buffers that intersect the given index-space bounding box. virtual void readBuffers(std::istream&, const CoordBBox&) = 0; +#if OPENVDB_ABI_VERSION_NUMBER < 14 OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void readNonresidentBuffers() const = 0; +#endif /// Write out all data buffers for this grid. virtual void writeBuffers(std::ostream&) const = 0; @@ -938,8 +940,10 @@ class Grid: public GridBase /// Read all of this grid's data buffers that intersect the given index-space bounding box. void readBuffers(std::istream&, const CoordBBox&) override; +#if OPENVDB_ABI_VERSION_NUMBER < 14 OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") void readNonresidentBuffers() const override { } +#endif /// Write out all data buffers for this grid. void writeBuffers(std::ostream&) const override; diff --git a/openvdb/openvdb/tree/LeafBuffer.h b/openvdb/openvdb/tree/LeafBuffer.h index 80da87c721..bb6222cf84 100644 --- a/openvdb/openvdb/tree/LeafBuffer.h +++ b/openvdb/openvdb/tree/LeafBuffer.h @@ -8,9 +8,11 @@ #include // for io::readCompressedValues(), etc #include #include +#if OPENVDB_ABI_VERSION_NUMBER < 14 #include -#include // for std::swap #include +#endif +#include // for std::swap #include // for offsetof() #include #include @@ -107,9 +109,11 @@ class LeafBuffer bool deallocate(); ValueType* mData = nullptr; +#if OPENVDB_ABI_VERSION_NUMBER < 14 // Deprecated members kept for ABI compatibility std::atomic mDeprecatedAtomic{0}; tbb::spin_mutex mDeprecatedSpinMutex; +#endif friend class ::TestLeaf; // Allow the parent LeafNode to access this buffer's data pointer. diff --git a/openvdb/openvdb/tree/Tree.h b/openvdb/openvdb/tree/Tree.h index a3371c71d7..9277c57b0f 100644 --- a/openvdb/openvdb/tree/Tree.h +++ b/openvdb/openvdb/tree/Tree.h @@ -162,8 +162,10 @@ class OPENVDB_API TreeBase /// Read all of this tree's data buffers that intersect the given bounding box. virtual void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) = 0; +#if OPENVDB_ABI_VERSION_NUMBER < 14 OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") virtual void readNonresidentBuffers() const = 0; +#endif /// Write out all the data buffers for this tree. virtual void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const = 0; @@ -331,8 +333,10 @@ class Tree: public TreeBase /// Read all of this tree's data buffers that intersect the given bounding box. void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) override; +#if OPENVDB_ABI_VERSION_NUMBER < 14 OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") void readNonresidentBuffers() const override { } +#endif /// Write out all data buffers for this tree. void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const override; diff --git a/openvdb/openvdb/unittest/TestGrid.cc b/openvdb/openvdb/unittest/TestGrid.cc index 147405c996..724109ce53 100644 --- a/openvdb/openvdb/unittest/TestGrid.cc +++ b/openvdb/openvdb/unittest/TestGrid.cc @@ -65,7 +65,9 @@ class ProxyTree: public openvdb::TreeBase void readBuffers(std::istream& is, const openvdb::CoordBBox&, bool /*saveFloatAsHalf*/=false) override { is.seekg(0); } +#if OPENVDB_ABI_VERSION_NUMBER < 14 void readNonresidentBuffers() const override {} +#endif void readBuffers(std::istream& is, bool /*saveFloatAsHalf*/=false) override { is.seekg(0); } void writeBuffers(std::ostream& os, bool /*saveFloatAsHalf*/=false) const override { os.seekp(0, std::ios::beg); } From 6e55069e92c470e686ebc560b03caefea4d42cb8 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 17 Feb 2026 11:27:01 -0800 Subject: [PATCH 11/80] Remove TreeBase readTopology() and writeTopology() Signed-off-by: Dan Bailey --- openvdb/openvdb/tree/Tree.h | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/openvdb/openvdb/tree/Tree.h b/openvdb/openvdb/tree/Tree.h index 9277c57b0f..07ed98e5db 100644 --- a/openvdb/openvdb/tree/Tree.h +++ b/openvdb/openvdb/tree/Tree.h @@ -151,11 +151,11 @@ class OPENVDB_API TreeBase /// @brief Read the tree topology from a stream. /// /// This will read the tree structure and tile values, but not voxel data. - virtual void readTopology(std::istream&, bool saveFloatAsHalf = false); + virtual void readTopology(std::istream&, bool saveFloatAsHalf = false) = 0; /// @brief Write the tree topology to a stream. /// /// This will write the tree structure and tile values, but not voxel data. - virtual void writeTopology(std::ostream&, bool saveFloatAsHalf = false) const; + virtual void writeTopology(std::ostream&, bool saveFloatAsHalf = false) const = 0; /// Read all data buffers for this tree. virtual void readBuffers(std::istream&, bool saveFloatAsHalf = false) = 0; @@ -1130,23 +1130,6 @@ struct Tree5 { //////////////////////////////////////// -inline void -TreeBase::readTopology(std::istream& is, bool /*saveFloatAsHalf*/) -{ - int32_t bufferCount; - is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); - if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported"); -} - - -inline void -TreeBase::writeTopology(std::ostream& os, bool /*saveFloatAsHalf*/) const -{ - int32_t bufferCount = 1; - os.write(reinterpret_cast(&bufferCount), sizeof(int32_t)); -} - - inline void TreeBase::print(std::ostream& os, int /*verboseLevel*/) const { @@ -1273,7 +1256,9 @@ void Tree::readTopology(std::istream& is, bool saveFloatAsHalf) { this->clearAllAccessors(); - TreeBase::readTopology(is, saveFloatAsHalf); + int32_t bufferCount; + is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); + if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported"); mRoot.readTopology(is, saveFloatAsHalf); } @@ -1282,7 +1267,8 @@ template void Tree::writeTopology(std::ostream& os, bool saveFloatAsHalf) const { - TreeBase::writeTopology(os, saveFloatAsHalf); + int32_t bufferCount = 1; + os.write(reinterpret_cast(&bufferCount), sizeof(int32_t)); mRoot.writeTopology(os, saveFloatAsHalf); } From bdaceec3ef7fe0d22c5573917f32678610ec8cd0 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 28 Nov 2025 16:06:59 -0800 Subject: [PATCH 12/80] Update all existing unit tests to use Grid I/O methods instead of Leaf I/O methods Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestLeafBool.cc | 50 ++++++-- openvdb/openvdb/unittest/TestLeafIO.cc | 109 +++++++++++++---- openvdb/openvdb/unittest/TestLeafMask.cc | 50 ++++++-- openvdb/openvdb/unittest/TestPointDataLeaf.cc | 113 ++++++++++-------- openvdb/openvdb/unittest/TestTree.cc | 47 ++++---- pendingchanges/removeleafiotests.txt | 3 + 6 files changed, 252 insertions(+), 120 deletions(-) create mode 100644 pendingchanges/removeleafiotests.txt diff --git a/openvdb/openvdb/unittest/TestLeafBool.cc b/openvdb/openvdb/unittest/TestLeafBool.cc index 6acae26ea2..79f9a00a83 100644 --- a/openvdb/openvdb/unittest/TestLeafBool.cc +++ b/openvdb/openvdb/unittest/TestLeafBool.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include "util.h" // for unittest_util::makeSphere() #include #include @@ -293,26 +294,49 @@ TEST_F(TestLeafBool, testIO) leaf.setValueOn(openvdb::Coord(0, 1, 0)); leaf.setValueOn(openvdb::Coord(1, 0, 0)); - std::ostringstream ostr(std::ios_base::binary); + // read and write topology to disk - leaf.writeBuffers(ostr); + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> BoolGrid; + BoolGrid::Ptr grid = BoolGrid::create(); + grid->setName("bool_leaf"); + grid->tree().addLeaf(new LeafType(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + // write to file + { + openvdb::io::File file("leaf_bool.vdb"); + file.write(grids); + file.close(); + } - leaf.setValueOff(openvdb::Coord(0, 1, 0)); - leaf.setValueOn(openvdb::Coord(0, 1, 1)); + // read grid from file + BoolGrid::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_bool.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("bool_leaf"); + file.close(); - std::istringstream istr(ostr.str(), std::ios_base::binary); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(istr); + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } - leaf.readBuffers(istr); + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); - EXPECT_EQ(origin, leaf.origin()); + // check topology and values match - EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(0, 1, 0))); - EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(1, 0, 0))); + EXPECT_EQ(origin, leaf2->origin()); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(0, 1, 0))); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(1, 0, 0))); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); - EXPECT_TRUE(leaf.onVoxelCount() == 2); + remove("leaf_bool.vdb"); + } } diff --git a/openvdb/openvdb/unittest/TestLeafIO.cc b/openvdb/openvdb/unittest/TestLeafIO.cc index 3b35259f53..a67cd9fdf6 100644 --- a/openvdb/openvdb/unittest/TestLeafIO.cc +++ b/openvdb/openvdb/unittest/TestLeafIO.cc @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include #include #include +#include #include #include // for toupper() @@ -21,35 +23,65 @@ template void TestLeafIO::testBuffer() { - openvdb::tree::LeafNode leaf(openvdb::Coord(0, 0, 0)); + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); + const openvdb::Coord origin = leaf.origin(); leaf.setValueOn(openvdb::Coord(0, 1, 0), T(1)); leaf.setValueOn(openvdb::Coord(1, 0, 0), T(1)); - std::ostringstream ostr(std::ios_base::binary); + // read and write topology to disk - leaf.writeBuffers(ostr); + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> GridType; + if (!GridType::isRegistered()) GridType::registerGrid(); - leaf.setValueOn(openvdb::Coord(0, 1, 0), T(0)); - leaf.setValueOn(openvdb::Coord(0, 1, 1), T(1)); + typename GridType::Ptr grid = GridType::create(); + grid->setName("leaf_io"); + grid->tree().addLeaf(new LeafT(leaf)); - std::istringstream istr(ostr.str(), std::ios_base::binary); + openvdb::GridCPtrVec grids; + grids.push_back(grid); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(istr); + // write to file + { + openvdb::io::File file("leaf_io.vdb"); + file.write(grids); + file.close(); + } - leaf.readBuffers(istr); + // read grid from file + typename GridType::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_io.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_io"); + file.close(); - EXPECT_NEAR(T(1), leaf.getValue(openvdb::Coord(0, 1, 0)), /*tolerance=*/0); - EXPECT_NEAR(T(1), leaf.getValue(openvdb::Coord(1, 0, 0)), /*tolerance=*/0); + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } - EXPECT_TRUE(leaf.onVoxelCount() == 2); + LeafT* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); + + // check topology and values match + + EXPECT_NEAR(T(1), leaf2->getValue(openvdb::Coord(0, 1, 0)), /*tolerance=*/0); + EXPECT_NEAR(T(1), leaf2->getValue(openvdb::Coord(1, 0, 0)), /*tolerance=*/0); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); + + remove("leaf_io.vdb"); + } } class TestLeafIOTest: public ::testing::Test { +public: + void SetUp() override { openvdb::initialize(); } + void TearDown() override { openvdb::uninitialize(); } }; @@ -62,28 +94,53 @@ TEST_F(TestLeafIOTest, testBufferByte) { TestLeafIO::testBuffer() TEST_F(TestLeafIOTest, testBufferVec3R) { - openvdb::tree::LeafNode leaf(openvdb::Coord(0, 0, 0)); + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); + const openvdb::Coord origin = leaf.origin(); leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(1, 1, 1)); leaf.setValueOn(openvdb::Coord(1, 0, 0), openvdb::Vec3R(1, 1, 1)); - std::ostringstream ostr(std::ios_base::binary); + // read and write topology to disk + + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> GridType; + GridType::Ptr grid = GridType::create(); + grid->setName("leaf_vec3r"); + grid->tree().addLeaf(new LeafT(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); - leaf.writeBuffers(ostr); + // write to file + { + openvdb::io::File file("leaf_vec3r.vdb"); + file.write(grids); + file.close(); + } - leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(0, 0, 0)); - leaf.setValueOn(openvdb::Coord(0, 1, 1), openvdb::Vec3R(1, 1, 1)); + // read grid from file + GridType::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_vec3r.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_vec3r"); + file.close(); - std::istringstream istr(ostr.str(), std::ios_base::binary); + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(istr); + LeafT* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); - leaf.readBuffers(istr); + // check topology and values match - EXPECT_TRUE(leaf.getValue(openvdb::Coord(0, 1, 0)) == openvdb::Vec3R(1, 1, 1)); - EXPECT_TRUE(leaf.getValue(openvdb::Coord(1, 0, 0)) == openvdb::Vec3R(1, 1, 1)); + EXPECT_TRUE(leaf2->getValue(openvdb::Coord(0, 1, 0)) == openvdb::Vec3R(1, 1, 1)); + EXPECT_TRUE(leaf2->getValue(openvdb::Coord(1, 0, 0)) == openvdb::Vec3R(1, 1, 1)); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); - EXPECT_TRUE(leaf.onVoxelCount() == 2); + remove("leaf_vec3r.vdb"); + } } diff --git a/openvdb/openvdb/unittest/TestLeafMask.cc b/openvdb/openvdb/unittest/TestLeafMask.cc index 7597158f45..ada92c2e12 100644 --- a/openvdb/openvdb/unittest/TestLeafMask.cc +++ b/openvdb/openvdb/unittest/TestLeafMask.cc @@ -6,6 +6,7 @@ #include #include #include +#include #include "util.h" // for unittest_util::makeSphere() #include #include @@ -290,26 +291,49 @@ TEST_F(TestLeafMask, testIO) leaf.setValueOn(openvdb::Coord(0, 1, 0)); leaf.setValueOn(openvdb::Coord(1, 0, 0)); - std::ostringstream ostr(std::ios_base::binary); + // read and write topology to disk - leaf.writeBuffers(ostr); + { + // create a grid with the leaf for topology testing + typedef openvdb::Grid, 5>>>> MaskGrid; + MaskGrid::Ptr grid = MaskGrid::create(); + grid->setName("leaf_mask"); + grid->tree().addLeaf(new LeafType(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + // write to file + { + openvdb::io::File file("leaf_mask.vdb"); + file.write(grids); + file.close(); + } - leaf.setValueOff(openvdb::Coord(0, 1, 0)); - leaf.setValueOn(openvdb::Coord(0, 1, 1)); + // read grid from file + MaskGrid::Ptr gridFromDisk; + { + openvdb::io::File file("leaf_mask.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_mask"); + file.close(); - std::istringstream istr(ostr.str(), std::ios_base::binary); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(istr); + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } - leaf.readBuffers(istr); + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(origin); + EXPECT_TRUE(leaf2); - EXPECT_EQ(origin, leaf.origin()); + // check topology and values match - EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(0, 1, 0))); - EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(1, 0, 0))); + EXPECT_EQ(origin, leaf2->origin()); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(0, 1, 0))); + EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(1, 0, 0))); + EXPECT_TRUE(leaf2->onVoxelCount() == 2); - EXPECT_TRUE(leaf.onVoxelCount() == 2); + remove("leaf_mask.vdb"); + } } diff --git a/openvdb/openvdb/unittest/TestPointDataLeaf.cc b/openvdb/openvdb/unittest/TestPointDataLeaf.cc index 651b3174d9..24fd1a679e 100644 --- a/openvdb/openvdb/unittest/TestPointDataLeaf.cc +++ b/openvdb/openvdb/unittest/TestPointDataLeaf.cc @@ -1058,76 +1058,93 @@ TEST_F(TestPointDataLeaf, testIO) // read and write topology to disk { - LeafType leaf2(openvdb::Coord(0, 0, 0)); + // create a grid with the leaf for topology testing + PointDataGrid::Ptr grid = PointDataGrid::create(); + grid->setName("points"); + grid->tree().addLeaf(new LeafType(leaf)); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); + + // write to file + { + io::File file("leaf_topology.vdb"); + file.write(grids); + file.close(); + } - std::ostringstream ostr(std::ios_base::binary); - leaf.writeTopology(ostr); + // read grid from file + PointDataGrid::Ptr gridFromDisk; + { + io::File file("leaf_topology.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); + file.close(); - std::istringstream istr(ostr.str(), std::ios_base::binary); - leaf2.readTopology(istr); + gridFromDisk = openvdb::gridPtrCast(baseGrid); + } + + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); + EXPECT_TRUE(leaf2); // check topology matches - EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); - EXPECT_TRUE(leaf2.isValueOn(4)); - EXPECT_TRUE(!leaf2.isValueOn(5)); + EXPECT_EQ(leaf.onVoxelCount(), leaf2->onVoxelCount()); + EXPECT_TRUE(leaf2->isValueOn(4)); + EXPECT_TRUE(!leaf2->isValueOn(5)); + + // check that values and attributes are correctly read - // check only topology (values and attributes still empty) + EXPECT_EQ(leaf2->getValue(4), ValueType(20)); + EXPECT_EQ(leaf2->attributeSet().size(), size_t(2)); - EXPECT_EQ(leaf2.getValue(4), ValueType(0)); - EXPECT_EQ(leaf2.attributeSet().size(), size_t(0)); + remove("leaf_topology.vdb"); } // read and write buffers to disk { - LeafType leaf2(openvdb::Coord(0, 0, 0)); - - io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); - - std::ostringstream ostr(std::ios_base::binary); - io::setStreamMetadataPtr(ostr, streamMetadata); - io::setDataCompression(ostr, io::COMPRESS_BLOSC); - leaf.writeTopology(ostr); - for (Index b = 0; b < leaf.buffers(); b++) { - uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); - streamMetadata->setPass(pass); - leaf.writeBuffers(ostr); - } - { // error checking - streamMetadata->setPass(1000); - leaf.writeBuffers(ostr); + // create a grid with the leaf for buffer testing + PointDataGrid::Ptr grid = PointDataGrid::create(); + grid->setName("points"); + grid->tree().addLeaf(new LeafType(leaf)); - io::StreamMetadata::Ptr meta; - io::setStreamMetadataPtr(ostr, meta); - EXPECT_THROW(leaf.writeBuffers(ostr), openvdb::IoError); - } + openvdb::GridCPtrVec grids; + grids.push_back(grid); - std::istringstream istr(ostr.str(), std::ios_base::binary); - io::setStreamMetadataPtr(istr, streamMetadata); - io::setDataCompression(istr, io::COMPRESS_BLOSC); + // write to file + { + io::File file("leaf_buffers.vdb"); + file.write(grids); + file.close(); + } - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - io::setCurrentVersion(istr); + // read grid from file + PointDataGrid::Ptr gridFromDisk; + { + io::File file("leaf_buffers.vdb"); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("points"); + file.close(); - leaf2.readTopology(istr); - for (Index b = 0; b < leaf.buffers(); b++) { - uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); - streamMetadata->setPass(pass); - leaf2.readBuffers(istr); + gridFromDisk = openvdb::gridPtrCast(baseGrid); } + LeafType* leaf2 = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0)); + EXPECT_TRUE(leaf2); + // check topology matches - EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); - EXPECT_TRUE(leaf2.isValueOn(4)); - EXPECT_TRUE(!leaf2.isValueOn(5)); + EXPECT_EQ(leaf.onVoxelCount(), leaf2->onVoxelCount()); + EXPECT_TRUE(leaf2->isValueOn(4)); + EXPECT_TRUE(!leaf2->isValueOn(5)); + + // check values and attributes are correctly read - // check only topology (values and attributes still empty) + EXPECT_EQ(leaf2->getValue(4), ValueType(20)); + EXPECT_EQ(leaf2->attributeSet().size(), size_t(2)); - EXPECT_EQ(leaf2.getValue(4), ValueType(20)); - EXPECT_EQ(leaf2.attributeSet().size(), size_t(2)); + remove("leaf_buffers.vdb"); } { // test multi-buffer IO diff --git a/openvdb/openvdb/unittest/TestTree.cc b/openvdb/openvdb/unittest/TestTree.cc index a5ac2afade..a132b2fd57 100644 --- a/openvdb/openvdb/unittest/TestTree.cc +++ b/openvdb/openvdb/unittest/TestTree.cc @@ -731,37 +731,44 @@ TEST_F(TestTree, testIterators) TEST_F(TestTree, testIO) { - const char* filename = "testIO.dbg"; + using TreeType = openvdb::tree::Tree; + using GridType = openvdb::Grid; + + const char* filename = "testIO.vdb"; openvdb::SharedPtr scopedFile(filename, ::remove); { ValueType background=5.0f; - RootNodeType root_node(background); - root_node.setValueOn(openvdb::Coord(5,10,20),0.234f); - root_node.setValueOn(openvdb::Coord(50000,20000,30000),4.5678f); + GridType::Ptr grid = GridType::create(background); + grid->setName("test_grid"); + grid->tree().setValueOn(openvdb::Coord(5,10,20),0.234f); + grid->tree().setValueOn(openvdb::Coord(50000,20000,30000),4.5678f); + + openvdb::GridCPtrVec grids; + grids.push_back(grid); - std::ofstream os(filename, std::ios_base::binary); - root_node.writeTopology(os); - root_node.writeBuffers(os); - EXPECT_TRUE(!os.fail()); + openvdb::io::File file(filename); + file.write(grids); + file.close(); } { ValueType background=2.0f; - RootNodeType root_node(background); - ASSERT_DOUBLES_EXACTLY_EQUAL(background, root_node.getValue(openvdb::Coord(5,10,20))); + GridType::Ptr grid = GridType::create(background); + ASSERT_DOUBLES_EXACTLY_EQUAL(background, grid->tree().getValue(openvdb::Coord(5,10,20))); + { - std::ifstream is(filename, std::ios_base::binary); - // Since the test file doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(is); - root_node.readTopology(is); - root_node.readBuffers(is); - EXPECT_TRUE(!is.fail()); + openvdb::io::File file(filename); + file.open(); + openvdb::GridBase::Ptr baseGrid = file.readGrid("test_grid"); + file.close(); + + grid = openvdb::gridPtrCast(baseGrid); + EXPECT_TRUE(grid.get() != nullptr); } - ASSERT_DOUBLES_EXACTLY_EQUAL(0.234f, root_node.getValue(openvdb::Coord(5,10,20))); - ASSERT_DOUBLES_EXACTLY_EQUAL(5.0f, root_node.getValue(openvdb::Coord(5,11,20))); + ASSERT_DOUBLES_EXACTLY_EQUAL(0.234f, grid->tree().getValue(openvdb::Coord(5,10,20))); + ASSERT_DOUBLES_EXACTLY_EQUAL(5.0f, grid->tree().getValue(openvdb::Coord(5,11,20))); ValueType sum=0.0f; - for (RootNodeType::ChildOnIter root_iter = root_node.beginChildOn(); + for (RootNodeType::ChildOnIter root_iter = grid->tree().root().beginChildOn(); root_iter.test(); ++root_iter) { for (InternalNodeType2::ChildOnIter internal_iter2 = root_iter->beginChildOn(); diff --git a/pendingchanges/removeleafiotests.txt b/pendingchanges/removeleafiotests.txt new file mode 100644 index 0000000000..481a910de8 --- /dev/null +++ b/pendingchanges/removeleafiotests.txt @@ -0,0 +1,3 @@ +OpenVDB: + Improvements: + - Remove direct testing of Tree I/O methods in favor of indirect testing using the Grid I/O. From 47cf7af05288ea3805e8d93b50867f410d5b41c7 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 10 Mar 2026 22:03:49 -0700 Subject: [PATCH 13/80] Add TreeIO methods for regression testing Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestLeafBool.cc | 29 ++++++ openvdb/openvdb/unittest/TestLeafIO.cc | 63 +++++++++++++ openvdb/openvdb/unittest/TestLeafMask.cc | 29 ++++++ openvdb/openvdb/unittest/TestPointDataLeaf.cc | 91 +++++++++++++++++++ openvdb/openvdb/unittest/TestTree.cc | 52 +++++++++++ 5 files changed, 264 insertions(+) diff --git a/openvdb/openvdb/unittest/TestLeafBool.cc b/openvdb/openvdb/unittest/TestLeafBool.cc index 79f9a00a83..3bb4522001 100644 --- a/openvdb/openvdb/unittest/TestLeafBool.cc +++ b/openvdb/openvdb/unittest/TestLeafBool.cc @@ -340,6 +340,35 @@ TEST_F(TestLeafBool, testIO) } +TEST_F(TestLeafBool, testTreeIO) +{ + LeafType leaf(openvdb::Coord(1, 3, 5)); + const openvdb::Coord origin = leaf.origin(); + + leaf.setValueOn(openvdb::Coord(0, 1, 0)); + leaf.setValueOn(openvdb::Coord(1, 0, 0)); + + std::ostringstream ostr(std::ios_base::binary); + + leaf.writeBuffers(ostr); + + leaf.setValueOff(openvdb::Coord(0, 1, 0)); + leaf.setValueOn(openvdb::Coord(0, 1, 1)); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + openvdb::io::setCurrentVersion(istr); + + leaf.readBuffers(istr); + + EXPECT_EQ(origin, leaf.origin()); + + EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(0, 1, 0))); + EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(1, 0, 0))); + + EXPECT_TRUE(leaf.onVoxelCount() == 2); +} + + TEST_F(TestLeafBool, testConstructors) { using openvdb::Coord; diff --git a/openvdb/openvdb/unittest/TestLeafIO.cc b/openvdb/openvdb/unittest/TestLeafIO.cc index a67cd9fdf6..7e4cf04575 100644 --- a/openvdb/openvdb/unittest/TestLeafIO.cc +++ b/openvdb/openvdb/unittest/TestLeafIO.cc @@ -17,6 +17,7 @@ class TestLeafIO { public: static void testBuffer(); + static void testTreeIO(); }; template @@ -77,6 +78,35 @@ TestLeafIO::testBuffer() } +template +void +TestLeafIO::testTreeIO() +{ + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); + + leaf.setValueOn(openvdb::Coord(0, 1, 0), T(1)); + leaf.setValueOn(openvdb::Coord(1, 0, 0), T(1)); + + std::ostringstream ostr(std::ios_base::binary); + + leaf.writeBuffers(ostr); + + leaf.setValueOn(openvdb::Coord(0, 1, 0), T(0)); + leaf.setValueOn(openvdb::Coord(0, 1, 1), T(1)); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + openvdb::io::setCurrentVersion(istr); + + leaf.readBuffers(istr); + + EXPECT_NEAR(T(1), leaf.getValue(openvdb::Coord(0, 1, 0)), /*tolerance=*/0); + EXPECT_NEAR(T(1), leaf.getValue(openvdb::Coord(1, 0, 0)), /*tolerance=*/0); + + EXPECT_TRUE(leaf.onVoxelCount() == 2); +} + + class TestLeafIOTest: public ::testing::Test { public: @@ -144,3 +174,36 @@ TEST_F(TestLeafIOTest, testBufferVec3R) remove("leaf_vec3r.vdb"); } } + +TEST_F(TestLeafIOTest, testTreeIOInt) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIOFloat) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIODouble) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIOBool) { TestLeafIO::testTreeIO(); } +TEST_F(TestLeafIOTest, testTreeIOByte) { TestLeafIO::testTreeIO(); } + + +TEST_F(TestLeafIOTest, testTreeIOVec3R) +{ + using LeafT = openvdb::tree::LeafNode; + LeafT leaf(openvdb::Coord(0, 0, 0)); + + leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(1, 1, 1)); + leaf.setValueOn(openvdb::Coord(1, 0, 0), openvdb::Vec3R(1, 1, 1)); + + std::ostringstream ostr(std::ios_base::binary); + + leaf.writeBuffers(ostr); + + leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(0, 0, 0)); + leaf.setValueOn(openvdb::Coord(0, 1, 1), openvdb::Vec3R(1, 1, 1)); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + openvdb::io::setCurrentVersion(istr); + + leaf.readBuffers(istr); + + EXPECT_TRUE(leaf.getValue(openvdb::Coord(0, 1, 0)) == openvdb::Vec3R(1, 1, 1)); + EXPECT_TRUE(leaf.getValue(openvdb::Coord(1, 0, 0)) == openvdb::Vec3R(1, 1, 1)); + + EXPECT_TRUE(leaf.onVoxelCount() == 2); +} diff --git a/openvdb/openvdb/unittest/TestLeafMask.cc b/openvdb/openvdb/unittest/TestLeafMask.cc index ada92c2e12..6545b3461a 100644 --- a/openvdb/openvdb/unittest/TestLeafMask.cc +++ b/openvdb/openvdb/unittest/TestLeafMask.cc @@ -337,6 +337,35 @@ TEST_F(TestLeafMask, testIO) } +TEST_F(TestLeafMask, testTreeIO) +{ + LeafType leaf(openvdb::Coord(1, 3, 5)); + const openvdb::Coord origin = leaf.origin(); + + leaf.setValueOn(openvdb::Coord(0, 1, 0)); + leaf.setValueOn(openvdb::Coord(1, 0, 0)); + + std::ostringstream ostr(std::ios_base::binary); + + leaf.writeBuffers(ostr); + + leaf.setValueOff(openvdb::Coord(0, 1, 0)); + leaf.setValueOn(openvdb::Coord(0, 1, 1)); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + openvdb::io::setCurrentVersion(istr); + + leaf.readBuffers(istr); + + EXPECT_EQ(origin, leaf.origin()); + + EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(0, 1, 0))); + EXPECT_TRUE(leaf.isValueOn(openvdb::Coord(1, 0, 0))); + + EXPECT_TRUE(leaf.onVoxelCount() == 2); +} + + TEST_F(TestLeafMask, testTopologyCopy) { using openvdb::Coord; diff --git a/openvdb/openvdb/unittest/TestPointDataLeaf.cc b/openvdb/openvdb/unittest/TestPointDataLeaf.cc index 24fd1a679e..c5956e2991 100644 --- a/openvdb/openvdb/unittest/TestPointDataLeaf.cc +++ b/openvdb/openvdb/unittest/TestPointDataLeaf.cc @@ -1276,6 +1276,97 @@ TEST_F(TestPointDataLeaf, testIO) } +TEST_F(TestPointDataLeaf, testTreeIO) +{ + using AttributeVec3s = TypedAttributeArray; + using AttributeF = TypedAttributeArray; + + using Descriptor = AttributeSet::Descriptor; + + Descriptor::Ptr descrA = Descriptor::create(AttributeVec3s::attributeType()); + + const size_t size = LeafType::NUM_VOXELS; + + LeafType leaf(openvdb::Coord(0, 0, 0)); + leaf.initializeAttributes(descrA, /*arrayLength=*/size/2); + + descrA = descrA->duplicateAppend("density", AttributeF::attributeType()); + leaf.appendAttribute(leaf.attributeSet().descriptor(), descrA, descrA->find("density")); + + leaf.setOffsetOn(1, 10); + leaf.setOffsetOn(4, 20); + leaf.setOffsetOn(7, 5); + + TypedAttributeArray& attr = + TypedAttributeArray::cast(leaf.attributeArray("density")); + + attr.set(0, 5.0f); + attr.set(50, 2.0f); + attr.set(51, 8.1f); + + { + LeafType leaf2(openvdb::Coord(0, 0, 0)); + + std::ostringstream ostr(std::ios_base::binary); + leaf.writeTopology(ostr); + + std::istringstream istr(ostr.str(), std::ios_base::binary); + leaf2.readTopology(istr); + + EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); + EXPECT_TRUE(leaf2.isValueOn(4)); + EXPECT_TRUE(!leaf2.isValueOn(5)); + + EXPECT_EQ(leaf2.getValue(4), ValueType(0)); + EXPECT_EQ(leaf2.attributeSet().size(), size_t(0)); + } + + { + LeafType leaf2(openvdb::Coord(0, 0, 0)); + + io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata); + + std::ostringstream ostr(std::ios_base::binary); + io::setStreamMetadataPtr(ostr, streamMetadata); + io::setDataCompression(ostr, io::COMPRESS_BLOSC); + leaf.writeTopology(ostr); + for (Index b = 0; b < leaf.buffers(); b++) { + uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); + streamMetadata->setPass(pass); + leaf.writeBuffers(ostr); + } + { + streamMetadata->setPass(1000); + leaf.writeBuffers(ostr); + + io::StreamMetadata::Ptr meta; + io::setStreamMetadataPtr(ostr, meta); + EXPECT_THROW(leaf.writeBuffers(ostr), openvdb::IoError); + } + + std::istringstream istr(ostr.str(), std::ios_base::binary); + io::setStreamMetadataPtr(istr, streamMetadata); + io::setDataCompression(istr, io::COMPRESS_BLOSC); + + io::setCurrentVersion(istr); + + leaf2.readTopology(istr); + for (Index b = 0; b < leaf.buffers(); b++) { + uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b); + streamMetadata->setPass(pass); + leaf2.readBuffers(istr); + } + + EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount()); + EXPECT_TRUE(leaf2.isValueOn(4)); + EXPECT_TRUE(!leaf2.isValueOn(5)); + + EXPECT_EQ(leaf2.getValue(4), ValueType(20)); + EXPECT_EQ(leaf2.attributeSet().size(), size_t(2)); + } +} + + TEST_F(TestPointDataLeaf, testSwap) { using AttributeVec3s = TypedAttributeArray; diff --git a/openvdb/openvdb/unittest/TestTree.cc b/openvdb/openvdb/unittest/TestTree.cc index a132b2fd57..bc08d087c6 100644 --- a/openvdb/openvdb/unittest/TestTree.cc +++ b/openvdb/openvdb/unittest/TestTree.cc @@ -790,6 +790,58 @@ TEST_F(TestTree, testIO) } +TEST_F(TestTree, testTreeIO) +{ + const char* filename = "testTreeIO.dbg"; + openvdb::SharedPtr scopedFile(filename, ::remove); + { + ValueType background=5.0f; + RootNodeType root_node(background); + root_node.setValueOn(openvdb::Coord(5,10,20),0.234f); + root_node.setValueOn(openvdb::Coord(50000,20000,30000),4.5678f); + + std::ofstream os(filename, std::ios_base::binary); + root_node.writeTopology(os); + root_node.writeBuffers(os); + EXPECT_TRUE(!os.fail()); + } + { + ValueType background=2.0f; + RootNodeType root_node(background); + ASSERT_DOUBLES_EXACTLY_EQUAL(background, root_node.getValue(openvdb::Coord(5,10,20))); + { + std::ifstream is(filename, std::ios_base::binary); + openvdb::io::setCurrentVersion(is); + root_node.readTopology(is); + root_node.readBuffers(is); + EXPECT_TRUE(!is.fail()); + } + + ASSERT_DOUBLES_EXACTLY_EQUAL(0.234f, root_node.getValue(openvdb::Coord(5,10,20))); + ASSERT_DOUBLES_EXACTLY_EQUAL(5.0f, root_node.getValue(openvdb::Coord(5,11,20))); + ValueType sum=0.0f; + for (RootNodeType::ChildOnIter root_iter = root_node.beginChildOn(); + root_iter.test(); ++root_iter) + { + for (InternalNodeType2::ChildOnIter internal_iter2 = root_iter->beginChildOn(); + internal_iter2.test(); ++internal_iter2) + { + for (InternalNodeType1::ChildOnIter internal_iter1 = + internal_iter2->beginChildOn(); internal_iter1.test(); ++internal_iter1) + { + for (LeafNodeType::ValueOnIter block_iter = + internal_iter1->beginValueOn(); block_iter.test(); ++block_iter) + { + sum += *block_iter; + } + } + } + } + ASSERT_DOUBLES_EXACTLY_EQUAL(sum, (0.234f + 4.5678f)); + } +} + + TEST_F(TestTree, testNegativeIndexing) { ValueType background=5.0f; From 6d3f320c2a5236826630b1f0ffe6ba733aaad84c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 15 Apr 2026 21:59:20 -0700 Subject: [PATCH 14/80] Clean up some custom initialization logic in the unit tests Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestFile.cc | 184 +------------------ openvdb/openvdb/unittest/TestMultiResGrid.cc | 3 +- openvdb/openvdb/unittest/TestStream.cc | 20 +- 3 files changed, 4 insertions(+), 203 deletions(-) diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 123408c524..d4e976fedc 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -44,7 +44,7 @@ class TestFile: public ::testing::Test { public: - void SetUp() override {} + void SetUp() override { openvdb::initialize(); } void TearDown() override { openvdb::uninitialize(); } void testHeader(); @@ -126,8 +126,6 @@ TestFile::testWriteGrid() tree.setValue(Coord(0, 0, 0), 5); // Add some metadata. - Metadata::clearRegistry(); - StringMetadata::registerType(); const std::string meta0Val, meta1Val("Hello, world."); Metadata::Ptr stringMetadata = Metadata::createMetadata(typeNameAsString()); EXPECT_TRUE(stringMetadata); @@ -157,22 +155,6 @@ TestFile::testWriteGrid() io::setCurrentVersion(istr); GridBase::Ptr gd2_grid; - EXPECT_THROW(gd2.read(istr), openvdb::LookupError); - - // Register the grid and the transform and the blocks. - GridBase::clearRegistry(); - GridType::registerGrid(); - - // Register transform maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - istr.seekg(0, std::ios_base::beg); EXPECT_NO_THROW(gd2_grid = gd2.read(istr)); @@ -207,10 +189,6 @@ TestFile::testWriteGrid() EXPECT_EQ( grid->baseTree().treeDepth(), gd2_grid->baseTree().treeDepth()); - //EXPECT_EQ(0.1, gd2_grid->getTransform()->getVoxelSizeX()); - //EXPECT_EQ(0.1, gd2_grid->getTransform()->getVoxelSizeY()); - //EXPECT_EQ(0.1, gd2_grid->getTransform()->getVoxelSizeZ()); - // Read in the data blocks. gd2.seekToBlocks(istr); gd2_grid->readBuffers(istr); @@ -220,10 +198,6 @@ TestFile::testWriteGrid() EXPECT_EQ(5, tree2->getValue(Coord(0, 0, 0))); EXPECT_EQ(1, tree2->getValue(Coord(1000, 1000, 16000))); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); remove("something.vdb2"); } @@ -276,20 +250,6 @@ TestFile::testWriteMultipleGrids() EXPECT_TRUE(gd2.getBlockPos() != 0); EXPECT_TRUE(gd2.getEndPos() != 0); - // register the grid - GridBase::clearRegistry(); - GridType::registerGrid(); - - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Read in the first grid descriptor. GridDescriptor gd_in; std::istringstream istr(ostr.str(), std::ios_base::binary); @@ -377,10 +337,6 @@ TestFile::testWriteMultipleGrids() EXPECT_EQ(10, grid2_in->getValue(Coord(0, 0, 0))); EXPECT_EQ(2, grid2_in->getValue(Coord(100000, 100000, 16000))); - // Clear registries. - GridBase::clearRegistry(); - - math::MapRegistry::clear(); remove("something.vdb2"); } TEST_F(TestFile, testWriteMultipleGrids) { testWriteMultipleGrids(); } @@ -394,12 +350,6 @@ TEST_F(TestFile, testWriteFloatAsHalf) using TreeType = Vec3STree; using GridType = Grid; - // Register all grid types. - initialize(); - // Ensure that the registry is cleared on exit. - struct Local { static void uninitialize(char*) { openvdb::uninitialize(); } }; - SharedPtr onExit(nullptr, Local::uninitialize); - // Create two test grids. GridType::Ptr grid1 = createGrid(/*bg=*/Vec3s(1, 1, 1)); TreeType& tree1 = grid1->tree(); @@ -460,9 +410,6 @@ TEST_F(TestFile, testWriteInstancedGrids) { using namespace openvdb; - // Register data types. - openvdb::initialize(); - // Remove something.vdb2 when done. We must declare this here before the // other grid smart_ptr's because we re-use them in the test several times. // We will not be able to remove something.vdb2 on Windows if the pointers @@ -680,19 +627,6 @@ TestFile::testReadGridDescriptors() file.writeGrid(gd, grid, ostr, /*seekable=*/true); file.writeGrid(gd2, grid2, ostr, /*seekable=*/true); - // Register the grid and the transform and the blocks. - GridBase::clearRegistry(); - GridType::registerGrid(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Read in the grid descriptors. File file2("something.vdb2"); std::istringstream istr(ostr.str(), std::ios_base::binary); @@ -716,10 +650,6 @@ TestFile::testReadGridDescriptors() EXPECT_EQ(gd2.getBlockPos(), file2gd.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), file2gd.getEndPos()); - // Clear registries. - GridBase::clearRegistry(); - math::MapRegistry::clear(); - remove("something.vdb2"); } TEST_F(TestFile, testReadGridDescriptors) { testReadGridDescriptors(); } @@ -732,9 +662,6 @@ TEST_F(TestFile, testGridNaming) using TreeType = Int32Tree; - // Register data types. - openvdb::initialize(); - logging::LevelScope suppressLogging{logging::Level::Fatal}; // Create several grids that share a single tree. @@ -947,19 +874,6 @@ TestFile::testEmptyGridIO() EXPECT_EQ(gd.getEndPos(), gd.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), gd2.getBlockPos()); - // Register the grid and the transform and the blocks. - GridBase::clearRegistry(); - GridType::registerGrid(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Read in the grid descriptors. File file2(filename); std::istringstream istr(ostr.str(), std::ios_base::binary); @@ -1002,10 +916,6 @@ TestFile::testEmptyGridIO() EXPECT_EQ(gd2.getGridPos(), file2gd.getGridPos()); EXPECT_EQ(gd2.getBlockPos(), file2gd.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), file2gd.getEndPos()); - - // Clear registries. - GridBase::clearRegistry(); - math::MapRegistry::clear(); } TEST_F(TestFile, testEmptyGridIO) { testEmptyGridIO(); } @@ -1055,23 +965,6 @@ void TestFile::testOpen() EXPECT_TRUE(meta.metaValue("author") == "Einstein"); EXPECT_EQ(2009, meta.metaValue("year")); - // Register grid and transform. - GridBase::clearRegistry(); - IntGrid::registerGrid(); - FloatGrid::registerGrid(); - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1120,11 +1013,6 @@ void TestFile::testOpen() EXPECT_THROW(vdbfile2.open(), openvdb::IoError); EXPECT_THROW(vdbfile2.inputStream(), openvdb::IoError); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - // Test closing the file. vdbfile.close(); EXPECT_TRUE(vdbfile.isOpen() == false); @@ -1166,11 +1054,6 @@ TEST_F(TestFile, testGetMetadata) meta.insertMeta("author", StringMetadata("Einstein")); meta.insertMeta("year", Int32Metadata(2009)); - // Adjust registry before writing. - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1187,9 +1070,6 @@ TEST_F(TestFile, testGetMetadata) EXPECT_TRUE(meta2->metaValue("author") == "Einstein"); EXPECT_EQ(2009, meta2->metaValue("year")); - // Clear registry. - Metadata::clearRegistry(); - remove("something.vdb2"); } @@ -1234,9 +1114,6 @@ TEST_F(TestFile, testReadAll) grids.push_back(grid1); grids.push_back(grid2); - // Register grid and transform. - openvdb::initialize(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1276,11 +1153,6 @@ TEST_F(TestFile, testReadAll) EXPECT_NEAR(10, temperature->getValue(Coord(0, 0, 0)), /*tolerance=*/0); EXPECT_NEAR(11, temperature->getValue(Coord(0, 100, 0)), /*tolerance=*/0); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - vdbfile2.close(); remove("something.vdb2"); @@ -1295,11 +1167,6 @@ TEST_F(TestFile, testWriteOpenFile) meta->insertMeta("author", StringMetadata("Einstein")); meta->insertMeta("year", Int32Metadata(2009)); - // Register metadata - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // Write the metadata out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(GridPtrVec(), *meta); @@ -1330,9 +1197,6 @@ TEST_F(TestFile, testWriteOpenFile) EXPECT_NO_THROW(vdbfile2.write(*grids)); - // Clear registries. - Metadata::clearRegistry(); - remove("something.vdb2"); } @@ -1341,8 +1205,6 @@ TEST_F(TestFile, testReadGridMetadata) { using namespace openvdb; - openvdb::initialize(); - const char* filename = "testReadGridMetadata.vdb2"; SharedPtr scopedFile(filename, ::remove); @@ -1512,9 +1374,6 @@ TEST_F(TestFile, testReadGrid) grids.push_back(grid); grids.push_back(grid2); - // Register grid and transform. - openvdb::initialize(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1549,11 +1408,6 @@ TEST_F(TestFile, testReadGrid) EXPECT_NEAR(5,typedDensity->getValue(Coord(0, 0, 0)), /*tolerance=*/0); EXPECT_NEAR(6,typedDensity->getValue(Coord(100, 0, 0)), /*tolerance=*/0); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - vdbfile2.close(); remove("something.vdb2"); @@ -1604,9 +1458,6 @@ TEST_F(TestFile, testReadClippedGrid) { using namespace openvdb; - // Register types. - openvdb::initialize(); - // World-space clipping region const BBoxd clipBox(Vec3d(4.0, 4.0, -6.0), Vec3d(4.9, 4.9, 6.0)); @@ -1958,23 +1809,6 @@ TEST_F(TestFile, testHasGrid) grids.push_back(grid); grids.push_back(grid2); - // Register grid and transform. - GridBase::clearRegistry(); - IntGrid::registerGrid(); - FloatGrid::registerGrid(); - Metadata::clearRegistry(); - StringMetadata::registerType(); - Int32Metadata::registerType(); - // register maps - math::MapRegistry::clear(); - math::AffineMap::registerMap(); - math::ScaleMap::registerMap(); - math::UniformScaleMap::registerMap(); - math::TranslationMap::registerMap(); - math::ScaleTranslateMap::registerMap(); - math::UniformScaleTranslateMap::registerMap(); - math::NonlinearFrustumMap::registerMap(); - // Write the vdb out to a file. io::File vdbfile("something.vdb2"); vdbfile.write(grids, meta); @@ -1990,11 +1824,6 @@ TEST_F(TestFile, testHasGrid) EXPECT_TRUE(!vdbfile2.hasGrid("Temperature")); EXPECT_TRUE(!vdbfile2.hasGrid("densitY")); - // Clear registries. - GridBase::clearRegistry(); - Metadata::clearRegistry(); - math::MapRegistry::clear(); - vdbfile2.close(); remove("something.vdb2"); @@ -2040,9 +1869,6 @@ TEST_F(TestFile, testNameIterator) grid->setName("level_set"); grids.push_back(grid); - // Register types. - openvdb::initialize(); - const char* filename = "testNameIterator.vdb2"; SharedPtr scopedFile(filename, ::remove); @@ -2090,9 +1916,6 @@ TEST_F(TestFile, testCompression) using IntGrid = openvdb::Int32Grid; - // Register types. - openvdb::initialize(); - // Create reference grids. IntGrid::Ptr intGrid = IntGrid::create(/*background=*/0); intGrid->fill(CoordBBox(Coord(0), Coord(49)), /*value=*/999, /*active=*/true); @@ -2302,9 +2125,6 @@ TEST_F(TestFile, testAsync) { using namespace openvdb; - // Register types. - openvdb::initialize(); - // Create a grid. FloatGrid::Ptr lsGrid = createLevelSet(); unittest_util::makeSphere(/*dim=*/Coord(100), /*ctr=*/Vec3f(50, 50, 50), /*r=*/20.0, @@ -2422,8 +2242,6 @@ TEST_F(TestFile, testAsync) // (see https://github.com/Blosc/c-blosc/pull/63). TEST_F(TestFile, testBlosc) { - openvdb::initialize(); - const unsigned char rawdata[] = { 0x93, 0xb0, 0x49, 0xaf, 0x62, 0xad, 0xe3, 0xaa, 0xe4, 0xa5, 0x43, 0x20, 0x24, 0x29, 0xc9, 0xaf, 0xee, 0xad, 0x0b, 0xac, 0x3d, 0xa8, 0x1f, 0x99, 0x53, 0x27, diff --git a/openvdb/openvdb/unittest/TestMultiResGrid.cc b/openvdb/openvdb/unittest/TestMultiResGrid.cc index d30cb0a28d..3930fbe047 100644 --- a/openvdb/openvdb/unittest/TestMultiResGrid.cc +++ b/openvdb/openvdb/unittest/TestMultiResGrid.cc @@ -208,6 +208,8 @@ TEST_F(TestMultiResGrid, testIO) { using namespace openvdb; + openvdb::initialize(); + const float radius = 1.0f; const Vec3f center(0.0f, 0.0f, 0.0f); const float voxelSize = 0.01f; @@ -243,7 +245,6 @@ TEST_F(TestMultiResGrid, testIO) outputFile.close(); // Read grids - openvdb::initialize(); openvdb::io::File file( filename ); file.open(); GridPtrVecPtr grids = file.getGrids(); diff --git a/openvdb/openvdb/unittest/TestStream.cc b/openvdb/openvdb/unittest/TestStream.cc index 8183317ea9..35850fab3d 100644 --- a/openvdb/openvdb/unittest/TestStream.cc +++ b/openvdb/openvdb/unittest/TestStream.cc @@ -36,25 +36,7 @@ class TestStream: public ::testing::Test void TestStream::SetUp() { - openvdb::uninitialize(); - - openvdb::Int32Grid::registerGrid(); - openvdb::FloatGrid::registerGrid(); - - openvdb::StringMetadata::registerType(); - openvdb::Int32Metadata::registerType(); - openvdb::Int64Metadata::registerType(); - openvdb::Vec3IMetadata::registerType(); - - // Register maps - openvdb::math::MapRegistry::clear(); - openvdb::math::AffineMap::registerMap(); - openvdb::math::ScaleMap::registerMap(); - openvdb::math::UniformScaleMap::registerMap(); - openvdb::math::TranslationMap::registerMap(); - openvdb::math::ScaleTranslateMap::registerMap(); - openvdb::math::UniformScaleTranslateMap::registerMap(); - openvdb::math::NonlinearFrustumMap::registerMap(); + openvdb::initialize(); } From 77162630c1f85146c7635f6c3c71130c6107e1ea Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 15 Apr 2026 22:42:42 -0700 Subject: [PATCH 15/80] Change testWriteHalf() to use a Stream when testing write as half Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestTree.cc | 74 +++++++++++++++++++++------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/openvdb/openvdb/unittest/TestTree.cc b/openvdb/openvdb/unittest/TestTree.cc index bc08d087c6..90a3de5e45 100644 --- a/openvdb/openvdb/unittest/TestTree.cc +++ b/openvdb/openvdb/unittest/TestTree.cc @@ -7,6 +7,7 @@ #include // for tools::setValueOnMin(), et al. #include #include // for io::RealToHalf +#include #include // for Abs() #include #include @@ -104,6 +105,10 @@ TEST_F(TestTree, testChangeBackground) TEST_F(TestTree, testHalf) { + // explicitly register these grid types as they are not registered by default + openvdb::Grid::registerGrid(); + openvdb::Grid::registerGrid(); + testWriteHalf(); testWriteHalf(); testWriteHalf(); @@ -125,27 +130,54 @@ TestTree::testWriteHalf() using GridType = openvdb::Grid; using ValueT = typename TreeType::ValueType; ValueT background(5); - GridType grid(background); + typename GridType::Ptr grid = GridType::create(background); + grid->setName("density"); unittest_util::makeSphere(openvdb::Coord(64, 64, 64), openvdb::Vec3f(35, 30, 40), - /*radius=*/10, grid, + /*radius=*/10, *grid, /*dx=*/1.0f, unittest_util::SPHERE_DENSE); - EXPECT_TRUE(!grid.tree().empty()); + EXPECT_TRUE(!grid->tree().empty()); + + // find stream and grid header position for save as float + + size_t headerSize = 0; + { + std::ostringstream outFull(std::ios_base::binary); + openvdb::io::Stream streamFull(outFull); + streamFull.write({}); + openvdb::io::GridDescriptor gd(grid->getName(), grid->type()); + gd.writeHeader(outFull); + headerSize = outFull.str().size(); + } + + // find stream and grid header position for save as half + + size_t halfHeaderSize = 0; + { + std::ostringstream outHalf(std::ios_base::binary); + openvdb::io::Stream streamHalf(outHalf); + streamHalf.write({}); + openvdb::io::GridDescriptor gdHalf(grid->getName(), grid->type(), /*half=*/true); + gdHalf.writeHeader(outHalf); + halfHeaderSize = outHalf.str().size(); + } // Write grid blocks in both float and half formats. std::ostringstream outFull(std::ios_base::binary); - grid.setSaveFloatAsHalf(false); - grid.writeBuffers(outFull); + openvdb::io::Stream streamFull(outFull); + grid->setSaveFloatAsHalf(false); + streamFull.write({grid}); outFull.flush(); - const size_t fullBytes = outFull.str().size(); + const size_t fullBytes = outFull.str().size() - headerSize; if (fullBytes == 0) FAIL() << "wrote empty full float buffers"; std::ostringstream outHalf(std::ios_base::binary); - grid.setSaveFloatAsHalf(true); - grid.writeBuffers(outHalf); + openvdb::io::Stream streamHalf(outHalf); + grid->setSaveFloatAsHalf(true); + streamHalf.write({grid}); outHalf.flush(); - const size_t halfBytes = outHalf.str().size(); + const size_t halfBytes = outHalf.str().size() - halfHeaderSize; if (halfBytes == 0) FAIL() << "wrote empty half float buffers"; if (openvdb::io::RealToHalf::isReal) { @@ -166,22 +198,26 @@ TestTree::testWriteHalf() // then write it out again in half float format. Verify that the resulting file // is identical to the original half float file. { - openvdb::Grid gridCopy(grid); - gridCopy.setSaveFloatAsHalf(true); std::istringstream is(outHalf.str(), std::ios_base::binary); + openvdb::io::Stream streamHalf2(is); - // Since the input stream doesn't include a VDB header with file format version info, - // tag the input stream explicitly with the current version number. - openvdb::io::setCurrentVersion(is); - - gridCopy.readBuffers(is); + openvdb::GridPtrVecPtr grids = streamHalf2.getGrids(); + openvdb::GridBase::Ptr gridCopy = (*grids)[0]; + gridCopy->setSaveFloatAsHalf(true); std::ostringstream outDiff(std::ios_base::binary); - gridCopy.writeBuffers(outDiff); + openvdb::io::Stream streamDiff(outDiff); + streamDiff.write({gridCopy}); outDiff.flush(); - if (outHalf.str() != outDiff.str()) { - FAIL() << "half-from-full and half-from-half buffers differ"; + // Compare the two buffers, skipping the header region + { + const std::string s1 = outHalf.str().substr(halfHeaderSize); + const std::string s2 = outDiff.str().substr(halfHeaderSize); + if (s1 != s2) + { + FAIL() << "half-from-full and half-from-half buffers differ"; + } } } } From 4042079c4de3e9f49b54052c32e178e8f8fd12cd Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 15 Apr 2026 23:02:04 -0700 Subject: [PATCH 16/80] Refactor GridDescriptor reading Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 6 +++-- openvdb/openvdb/io/GridDescriptor.cc | 27 ++++++++++++------- openvdb/openvdb/io/GridDescriptor.h | 10 +++++++ openvdb/openvdb/io/Stream.cc | 3 ++- openvdb/openvdb/unittest/TestFile.cc | 17 +++++++----- .../openvdb/unittest/TestGridDescriptor.cc | 7 +++-- 6 files changed, 47 insertions(+), 23 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 4cbda52eb6..00265f9839 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -281,7 +281,8 @@ File::open() const int32_t gridCount = readGridCount(inputStream()); for (int32_t i = 0; i < gridCount; ++i) { GridDescriptor gd; - gd.read(inputStream()); + gd.readHeader(inputStream()); + gd.readStreamPos(inputStream()); GridBase::Ptr grid = createGrid(gd); Archive::readGrid(grid, gd, inputStream()); @@ -584,7 +585,8 @@ File::readGridDescriptors(std::istream& is) for (int32_t i = 0, N = readGridCount(is); i < N; ++i) { // Read the grid descriptor. GridDescriptor gd; - gd.read(is); + gd.readHeader(is); + gd.readStreamPos(is); // Add the descriptor to the dictionary. gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); diff --git a/openvdb/openvdb/io/GridDescriptor.cc b/openvdb/openvdb/io/GridDescriptor.cc index 961f622008..9a59d22d1c 100644 --- a/openvdb/openvdb/io/GridDescriptor.cc +++ b/openvdb/openvdb/io/GridDescriptor.cc @@ -69,8 +69,8 @@ GridDescriptor::writeStreamPos(std::ostream &os) const os.write(reinterpret_cast(&mEndPos), sizeof(int64_t)); } -GridBase::Ptr -GridDescriptor::read(std::istream &is) +void +GridDescriptor::readHeader(std::istream &is) { checkFormatVersion(is); @@ -86,21 +86,28 @@ GridDescriptor::read(std::istream &is) } mInstanceParentName = readString(is); +} + +void +GridDescriptor::readStreamPos(std::istream &is) +{ + is.read(reinterpret_cast(&mGridPos), sizeof(int64_t)); + is.read(reinterpret_cast(&mBlockPos), sizeof(int64_t)); + is.read(reinterpret_cast(&mEndPos), sizeof(int64_t)); +} + +GridBase::Ptr +GridDescriptor::read(std::istream &is) +{ + readHeader(is); + readStreamPos(is); - // Create the grid of the type if it has been registered. if (!GridBase::isRegistered(mGridType)) { OPENVDB_THROW(LookupError, "Cannot read grid." << " Grid type " << mGridType << " is not registered."); } - // else GridBase::Ptr grid = GridBase::createGrid(mGridType); if (grid) grid->setSaveFloatAsHalf(mSaveFloatAsHalf); - - // Read in the offsets. - is.read(reinterpret_cast(&mGridPos), sizeof(int64_t)); - is.read(reinterpret_cast(&mBlockPos), sizeof(int64_t)); - is.read(reinterpret_cast(&mEndPos), sizeof(int64_t)); - return grid; } diff --git a/openvdb/openvdb/io/GridDescriptor.h b/openvdb/openvdb/io/GridDescriptor.h index 9ef03262e2..1aaf5f10a4 100644 --- a/openvdb/openvdb/io/GridDescriptor.h +++ b/openvdb/openvdb/io/GridDescriptor.h @@ -61,8 +61,18 @@ class OPENVDB_API GridDescriptor /// written out separately. void writeStreamPos(std::ostream&) const; + /// @brief Read this descriptor's header information (all data except for + /// stream offsets) from the given stream. + void readHeader(std::istream&); + + /// @brief Read stream positions (grid, block, and end offsets) from the + /// given stream. + void readStreamPos(std::istream&); + /// @brief Read a grid descriptor from the given stream. /// @return an empty grid of the type specified by the grid descriptor. + /// @deprecated Use readHeader() followed by readStreamPos() instead. + OPENVDB_DEPRECATED_MESSAGE("Use readHeader() followed by readStreamPos() instead.") GridBase::Ptr read(std::istream&); /// @brief Append the number @a n to the given name (separated by an ASCII diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index be3a003515..80326af693 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -72,7 +72,8 @@ Stream::Stream(std::istream& is): mImpl(new Impl) Archive::NamedGridMap namedGrids; for (int32_t i = 0; i < gridCount; ++i) { GridDescriptor gd; - gd.read(is); + gd.readHeader(is); + gd.readStreamPos(is); descriptors.push_back(gd); GridBase::Ptr grid = readGrid(gd, is); mImpl->mGrids->push_back(grid); diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index d4e976fedc..9120bed409 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -154,9 +154,12 @@ TestFile::testWriteGrid() // it doesn't have a header), set the file format version number explicitly. io::setCurrentVersion(istr); - GridBase::Ptr gd2_grid; istr.seekg(0, std::ios_base::beg); - EXPECT_NO_THROW(gd2_grid = gd2.read(istr)); + gd2.readHeader(istr); + gd2.readStreamPos(istr); + + // manually create the grid + GridBase::Ptr gd2_grid = GridBase::createGrid(gd2.gridType()); EXPECT_EQ(gd.gridName(), gd2.gridName()); EXPECT_EQ(GridType::gridType(), gd2_grid->type()); @@ -255,8 +258,9 @@ TestFile::testWriteMultipleGrids() std::istringstream istr(ostr.str(), std::ios_base::binary); io::setCurrentVersion(istr); - GridBase::Ptr gd_in_grid; - EXPECT_NO_THROW(gd_in_grid = gd_in.read(istr)); + gd_in.readHeader(istr); + gd_in.readStreamPos(istr); + GridBase::Ptr gd_in_grid = GridBase::createGrid(gd_in.gridType()); // Ensure read in the right values. EXPECT_EQ(gd.gridName(), gd_in.gridName()); @@ -300,8 +304,9 @@ TestFile::testWriteMultipleGrids() gd_in.seekToEnd(istr); GridDescriptor gd2_in; - GridBase::Ptr gd2_in_grid; - EXPECT_NO_THROW(gd2_in_grid = gd2_in.read(istr)); + gd2_in.readHeader(istr); + gd2_in.readStreamPos(istr); + GridBase::Ptr gd2_in_grid = GridBase::createGrid(gd2_in.gridType()); // Ensure that we read in the right values. EXPECT_EQ(gd2.gridName(), gd2_in.gridName()); diff --git a/openvdb/openvdb/unittest/TestGridDescriptor.cc b/openvdb/openvdb/unittest/TestGridDescriptor.cc index c45e8d0433..a39fde278e 100644 --- a/openvdb/openvdb/unittest/TestGridDescriptor.cc +++ b/openvdb/openvdb/unittest/TestGridDescriptor.cc @@ -42,16 +42,15 @@ TEST_F(TestGridDescriptor, testIO) GridDescriptor gd2; - EXPECT_THROW(gd2.read(istr), openvdb::LookupError); - // Register the grid. GridBase::clearRegistry(); GridType::registerGrid(); // seek back and read again. istr.seekg(0, std::ios_base::beg); - GridBase::Ptr grid; - EXPECT_NO_THROW(grid = gd2.read(istr)); + gd2.readHeader(istr); + gd2.readStreamPos(istr); + GridBase::Ptr grid = GridBase::createGrid(gd2.gridType()); EXPECT_EQ(gd.gridName(), gd2.gridName()); EXPECT_EQ(gd.uniqueName(), gd2.uniqueName()); From b2691d0797774237ee25bc59a6fe5e0988ff7f3e Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 19 Mar 2026 22:27:20 -0700 Subject: [PATCH 17/80] Move body of readGridDescriptors into sole call site and delete method Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 41 +++++++++++++------------------------- openvdb/openvdb/io/File.h | 3 --- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 00265f9839..33a3ccd044 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -296,8 +296,20 @@ File::open() Archive::connectInstance(it->second, mImpl->mNamedGrids); } } else { - // Read in just the grid descriptors. - readGridDescriptors(inputStream()); + gridDescriptors().clear(); + + for (int32_t i = 0, N = readGridCount(inputStream()); i < N; ++i) { + // Read the grid descriptor. + GridDescriptor gd; + gd.readHeader(inputStream()); + gd.readStreamPos(inputStream()); + + // Add the descriptor to the dictionary. + gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); + + // Skip forward to the next descriptor. + gd.seekToEnd(inputStream()); + } } mImpl->mIsOpen = true; @@ -571,31 +583,6 @@ File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta) const } -//////////////////////////////////////// - - -void -File::readGridDescriptors(std::istream& is) -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - gridDescriptors().clear(); - - for (int32_t i = 0, N = readGridCount(is); i < N; ++i) { - // Read the grid descriptor. - GridDescriptor gd; - gd.readHeader(is); - gd.readStreamPos(is); - - // Add the descriptor to the dictionary. - gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); - - // Skip forward to the next descriptor. - gd.seekToEnd(is); - } -} - //////////////////////////////////////// diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index be8ab89968..3b64058c0b 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -146,9 +146,6 @@ class OPENVDB_API File: public Archive NameIterator endName() const; private: - /// Read in all grid descriptors that are stored in the given stream. - void readGridDescriptors(std::istream&); - /// @brief Return an iterator to the descriptor for the grid with the given name. /// If the name is non-unique, return an iterator to the first matching descriptor. NameMapCIter findDescriptor(const Name&) const; From f5495e29dee65444f4cc0e3061cbaa5b8b260d1c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 19 Mar 2026 22:27:50 -0700 Subject: [PATCH 18/80] Remove File::readGridByName() Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 9 +-------- openvdb/openvdb/io/File.h | 4 ---- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 33a3ccd044..d399c09586 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -491,19 +491,12 @@ File::readGridMetadata(const Name& name) GridBase::Ptr File::readGrid(const Name& name) { - return readGridByName(name, BBoxd()); + return readGrid(name, BBoxd()); } GridBase::Ptr File::readGrid(const Name& name, const BBoxd& bbox) -{ - return readGridByName(name, bbox); -} - - -GridBase::Ptr -File::readGridByName(const Name& name, const BBoxd& bbox) { if (!isOpen()) { OPENVDB_THROW(IoError, filename() << " is not open for reading."); diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 3b64058c0b..f9e11b342c 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -153,10 +153,6 @@ class OPENVDB_API File: public Archive /// Return a newly created, empty grid of the type specified by the given grid descriptor. GridBase::Ptr createGrid(const GridDescriptor&) const; - /// @brief Read a grid, including its data blocks, but only where it - /// intersects the given world-space bounding box. - GridBase::Ptr readGridByName(const Name&, const BBoxd&); - /// Read in and return the partially-populated grid specified by the given grid descriptor. GridBase::ConstPtr readGridPartial(const GridDescriptor&, bool readTopology) const; From 2fc931221c0dfa6652b1fa0165deeac050f50a1f Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 19 Mar 2026 22:28:33 -0700 Subject: [PATCH 19/80] Simplify readGridPartial() and remove unused topology read option Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 48 ++++++++++++++------------------------ openvdb/openvdb/io/File.h | 6 +---- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index d399c09586..ff5aa6345b 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -442,7 +442,7 @@ File::readAllGridMetadata() // Read just the metadata and transforms for all grids. for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { const GridDescriptor& gd = i->second; - GridBase::ConstPtr grid = readGridPartial(gd, /*readTopology=*/false); + GridBase::ConstPtr grid = readGridPartial(gd); // Return copies of the grids, but with empty trees. // (As of 0.98.0, at least, it would suffice to just const cast // the grid pointers returned by readGridPartial(), but shallow @@ -479,7 +479,7 @@ File::readGridMetadata(const Name& name) // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; - ret = readGridPartial(gd, /*readTopology=*/false); + ret = readGridPartial(gd); } return ret->copyGridWithNewTree(); } @@ -644,18 +644,30 @@ File::createGrid(const GridDescriptor& gd) const GridBase::ConstPtr -File::readGridPartial(const GridDescriptor& gd, bool readTopology) const +File::readGridPartial(const GridDescriptor& gd) const { // This method should not be called for files that don't contain grid offsets. OPENVDB_ASSERT(inputHasGridOffsets()); + std::istream& is = inputStream(); + GridBase::Ptr grid = createGrid(gd); // Seek to grid. - gd.seekToGrid(inputStream()); + gd.seekToGrid(is); + + // This code needs to stay in sync with io::Archive::readGrid(), in terms of + // the order of operations. + readGridCompression(is); + grid->readMeta(is); + + // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists + if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { + grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); + } - // Read the grid partially. - readGridPartial(grid, inputStream(), gd.isInstance(), readTopology); + // Read the transform. + grid->readTransform(is); // Promote to a const grid. GridBase::ConstPtr constGrid = grid; @@ -685,30 +697,6 @@ File::readGrid(const GridDescriptor& gd, const CoordBBox& bbox) const } -void -File::readGridPartial(GridBase::Ptr grid, std::istream& is, - bool isInstance, bool readTopology) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - // This code needs to stay in sync with io::Archive::readGrid(), in terms of - // the order of operations. - readGridCompression(is); - grid->readMeta(is); - - // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } - - grid->readTransform(is); - if (!isInstance && readTopology) { - grid->readTopology(is); - } -} - - //////////////////////////////////////// diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index f9e11b342c..49ce98eb6d 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -154,7 +154,7 @@ class OPENVDB_API File: public Archive GridBase::Ptr createGrid(const GridDescriptor&) const; /// Read in and return the partially-populated grid specified by the given grid descriptor. - GridBase::ConstPtr readGridPartial(const GridDescriptor&, bool readTopology) const; + GridBase::ConstPtr readGridPartial(const GridDescriptor&) const; /// Read in and return the grid specified by the given grid descriptor. GridBase::Ptr readGrid(const GridDescriptor&) const; @@ -165,10 +165,6 @@ class OPENVDB_API File: public Archive /// that intersects the given index-space bounding box. GridBase::Ptr readGrid(const GridDescriptor&, const CoordBBox&) const; - /// @brief Partially populate the given grid by reading its metadata and transform and, - /// if the grid is not an instance, its tree structure, but not the tree's leaf nodes. - void readGridPartial(GridBase::Ptr, std::istream&, bool isInstance, bool readTopology) const; - /// @brief Retrieve a grid from @c mNamedGrids. Return a null pointer /// if @c mNamedGrids was not populated (because this file is random-access). /// @throw KeyError if no grid with the given name exists in this file. From 5eb40b118171389041eb752c5d5b4102f6f8d276 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 19 Mar 2026 23:15:22 -0700 Subject: [PATCH 20/80] Eliminate File::Impl Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 286 ++++++++++++--------------- openvdb/openvdb/io/File.h | 20 +- openvdb/openvdb/unittest/TestFile.cc | 56 ++++-- 3 files changed, 183 insertions(+), 179 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index ff5aa6345b..0424c89da4 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -25,83 +25,24 @@ OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { namespace io { -// Implementation details of the File class -struct File::Impl -{ - struct NoBBox {}; - - // Common implementation of the various File::readGrid() overloads, - // with and without bounding box clipping - template - static GridBase::Ptr readGrid(const File& file, const GridDescriptor& gd, const BoxType& bbox) - { - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(file.inputHasGridOffsets()); - - GridBase::Ptr grid = file.createGrid(gd); - gd.seekToGrid(file.inputStream()); - unarchive(file, grid, gd, bbox); - return grid; - } - - static void unarchive(const File& file, GridBase::Ptr& grid, - const GridDescriptor& gd, NoBBox) - { - file.Archive::readGrid(grid, gd, file.inputStream()); - } - - static void unarchive(const File& file, GridBase::Ptr& grid, - const GridDescriptor& gd, const CoordBBox& indexBBox) - { - file.Archive::readGrid(grid, gd, file.inputStream(), indexBBox); - } - - static void unarchive(const File& file, GridBase::Ptr& grid, - const GridDescriptor& gd, const BBoxd& worldBBox) - { - file.Archive::readGrid(grid, gd, file.inputStream(), worldBBox); - } - std::string mFilename; - // The file-level metadata - MetaMap::Ptr mMeta; - // The file stream that is open for reading - std::unique_ptr mInStream; - // File-level stream metadata (file format, compression, etc.) - StreamMetadata::Ptr mStreamMetadata; - // Flag indicating if we have read in the global information (header, - // metadata, and grid descriptors) for this VDB file - bool mIsOpen; - // Grid descriptors for all grids stored in the file, indexed by grid name - NameMap mGridDescriptors; - // All grids, indexed by unique name (used only when mHasGridOffsets is false) - Archive::NamedGridMap mNamedGrids; - // All grids stored in the file (used only when mHasGridOffsets is false) - GridPtrVecPtr mGrids; -}; // class File::Impl - - -//////////////////////////////////////// - - -File::File(const std::string& filename): mImpl(new Impl) +File::File(const std::string& filename) + : Archive() + , mFilename(filename) { - mImpl->mFilename = filename; - mImpl->mIsOpen = false; setInputHasGridOffsets(true); } -File::~File() -{ -} - - File::File(const File& other) : Archive(other) - , mImpl(new Impl) + , mFilename(other.mFilename) + , mMeta(other.mMeta) + , mIsOpen(false) + , mGridDescriptors(other.mGridDescriptors) + , mNamedGrids(other.mNamedGrids) + , mGrids(other.mGrids) { - *this = other; } @@ -110,13 +51,12 @@ File::operator=(const File& other) { if (&other != this) { Archive::operator=(other); - const Impl& otherImpl = *other.mImpl; - mImpl->mFilename = otherImpl.mFilename; - mImpl->mMeta = otherImpl.mMeta; - mImpl->mIsOpen = false; // don't want two file objects reading from the same stream - mImpl->mGridDescriptors = otherImpl.mGridDescriptors; - mImpl->mNamedGrids = otherImpl.mNamedGrids; - mImpl->mGrids = otherImpl.mGrids; + mFilename = other.mFilename; + mMeta = other.mMeta; + mIsOpen = false; // don't want two file objects reading from the same stream + mGridDescriptors = other.mGridDescriptors; + mNamedGrids = other.mNamedGrids; + mGrids = other.mGrids; } return *this; } @@ -135,43 +75,43 @@ File::copy() const const std::string& File::filename() const { - return mImpl->mFilename; + return mFilename; } MetaMap::Ptr File::fileMetadata() { - return mImpl->mMeta; + return mMeta; } MetaMap::ConstPtr File::fileMetadata() const { - return mImpl->mMeta; + return mMeta; } const File::NameMap& File::gridDescriptors() const { - return mImpl->mGridDescriptors; + return mGridDescriptors; } File::NameMap& File::gridDescriptors() { - return mImpl->mGridDescriptors; + return mGridDescriptors; } std::istream& File::inputStream() const { - if (!mImpl->mInStream) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mInStream) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } - return *mImpl->mInStream; + return *mInStream; } @@ -187,11 +127,11 @@ File::getSize() const Index64 result = std::numeric_limits::max(); - std::string mesg = "could not get size of file " + filename(); + std::string mesg = "could not get size of file " + mFilename; #ifdef _WIN32 // Get the file size by seeking to the end of the file. - std::ifstream fstrm(filename()); + std::ifstream fstrm(mFilename); if (fstrm) { fstrm.seekg(0, fstrm.end); result = static_cast(fstrm.tellg()); @@ -201,7 +141,7 @@ File::getSize() const #else // Get the file size using the stat() system call. struct stat info; - if (0 != ::stat(filename().c_str(), &info)) { + if (0 != ::stat(mFilename.c_str(), &info)) { std::string s = getErrorString(); if (!s.empty()) mesg += " (" + s + ")"; OPENVDB_THROW(IoError, mesg); @@ -223,25 +163,25 @@ File::getSize() const bool File::isOpen() const { - return mImpl->mIsOpen; + return mIsOpen; } bool File::open() { - if (isOpen()) { - OPENVDB_THROW(IoError, filename() << " is already open"); + if (mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is already open"); } - mImpl->mInStream.reset(); + mInStream.reset(); // Open the file using standard I/O (delayed loading has been removed) std::unique_ptr newStream; newStream.reset(new std::ifstream( - filename().c_str(), std::ios_base::in | std::ios_base::binary)); + mFilename.c_str(), std::ios_base::in | std::ios_base::binary)); if (newStream->fail()) { - OPENVDB_THROW(IoError, "could not open file " << filename()); + OPENVDB_THROW(IoError, "could not open file " << mFilename); } // Read in the file header. @@ -251,31 +191,31 @@ File::open() } catch (IoError& e) { if (e.what() && std::string("not a VDB file") == e.what()) { // Rethrow, adding the filename. - OPENVDB_THROW(IoError, filename() << " is not a VDB file"); + OPENVDB_THROW(IoError, mFilename << " is not a VDB file"); } throw; } - mImpl->mInStream.swap(newStream); + mInStream.swap(newStream); // Tag the input stream with the file format and library version numbers // and other metadata. - mImpl->mStreamMetadata.reset(new StreamMetadata); - mImpl->mStreamMetadata->setSeekable(true); - io::setStreamMetadataPtr(inputStream(), mImpl->mStreamMetadata, /*transfer=*/false); + mStreamMetadata.reset(new StreamMetadata); + mStreamMetadata->setSeekable(true); + io::setStreamMetadataPtr(inputStream(), mStreamMetadata, /*transfer=*/false); Archive::setFormatVersion(inputStream()); Archive::setLibraryVersion(inputStream()); Archive::setDataCompression(inputStream()); // Read in the VDB metadata. - mImpl->mMeta = MetaMap::Ptr(new MetaMap); - mImpl->mMeta->readMeta(inputStream()); + mMeta = MetaMap::Ptr(new MetaMap); + mMeta->readMeta(inputStream()); if (!inputHasGridOffsets()) { - OPENVDB_LOG_DEBUG_RUNTIME("file " << filename() << " does not support partial reading"); + OPENVDB_LOG_DEBUG_RUNTIME("file " << mFilename << " does not support partial reading"); - mImpl->mGrids.reset(new GridPtrVec); - mImpl->mNamedGrids.clear(); + mGrids.reset(new GridPtrVec); + mNamedGrids.clear(); // Stream in the entire contents of the file and append all grids to mGrids. const int32_t gridCount = readGridCount(inputStream()); @@ -287,16 +227,16 @@ File::open() GridBase::Ptr grid = createGrid(gd); Archive::readGrid(grid, gd, inputStream()); - gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); - mImpl->mGrids->push_back(grid); - mImpl->mNamedGrids[gd.uniqueName()] = grid; + mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); + mGrids->push_back(grid); + mNamedGrids[gd.uniqueName()] = grid; } // Connect instances (grids that share trees with other grids). - for (NameMapCIter it = gridDescriptors().begin(); it != gridDescriptors().end(); ++it) { - Archive::connectInstance(it->second, mImpl->mNamedGrids); + for (NameMapCIter it = mGridDescriptors.begin(); it != mGridDescriptors.end(); ++it) { + Archive::connectInstance(it->second, mNamedGrids); } } else { - gridDescriptors().clear(); + mGridDescriptors.clear(); for (int32_t i = 0, N = readGridCount(inputStream()); i < N; ++i) { // Read the grid descriptor. @@ -305,14 +245,14 @@ File::open() gd.readStreamPos(inputStream()); // Add the descriptor to the dictionary. - gridDescriptors().insert(std::make_pair(gd.gridName(), gd)); + mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); // Skip forward to the next descriptor. gd.seekToEnd(inputStream()); } } - mImpl->mIsOpen = true; + mIsOpen = true; return newFile; // true if file is not identical to opened file } @@ -321,14 +261,14 @@ void File::close() { // Reset all data. - mImpl->mMeta.reset(); - mImpl->mGridDescriptors.clear(); - mImpl->mGrids.reset(); - mImpl->mNamedGrids.clear(); - mImpl->mInStream.reset(); - mImpl->mStreamMetadata.reset(); - - mImpl->mIsOpen = false; + mMeta.reset(); + mGridDescriptors.clear(); + mGrids.reset(); + mNamedGrids.clear(); + mInStream.reset(); + mStreamMetadata.reset(); + + mIsOpen = false; setInputHasGridOffsets(true); } @@ -339,44 +279,44 @@ File::close() bool File::hasGrid(const Name& name) const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } - return (findDescriptor(name) != gridDescriptors().end()); + return (findDescriptor(name) != mGridDescriptors.end()); } MetaMap::Ptr File::getMetadata() const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } // Return a deep copy of the file-level metadata, which was read // when the file was opened. - return MetaMap::Ptr(new MetaMap(*mImpl->mMeta)); + return MetaMap::Ptr(new MetaMap(*mMeta)); } GridPtrVecPtr File::getGrids() const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } GridPtrVecPtr ret; if (!inputHasGridOffsets()) { // If the input file doesn't have grid offsets, then all of the grids // have already been streamed in and stored in mGrids. - ret = mImpl->mGrids; + ret = mGrids; } else { ret.reset(new GridPtrVec); Archive::NamedGridMap namedGrids; // Read all grids represented by the GridDescriptors. - for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { + for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; GridBase::Ptr grid = readGrid(gd); ret->push_back(grid); @@ -384,7 +324,7 @@ File::getGrids() const } // Connect instances (grids that share trees with other grids). - for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { + for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { Archive::connectInstance(i->second, namedGrids); } } @@ -404,11 +344,11 @@ File::retrieveCachedGrid(const Name& name) const // Search by unique name. Archive::NamedGridMap::const_iterator it = - mImpl->mNamedGrids.find(GridDescriptor::stringAsUniqueName(name)); + mNamedGrids.find(GridDescriptor::stringAsUniqueName(name)); // If not found, search by grid name. - if (it == mImpl->mNamedGrids.end()) it = mImpl->mNamedGrids.find(name); - if (it == mImpl->mNamedGrids.end()) { - OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\""); + if (it == mNamedGrids.end()) it = mNamedGrids.find(name); + if (it == mNamedGrids.end()) { + OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\""); } return it->second; } @@ -420,8 +360,8 @@ File::retrieveCachedGrid(const Name& name) const GridPtrVecPtr File::readAllGridMetadata() { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } if (fileVersion() < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) { @@ -434,13 +374,13 @@ File::readAllGridMetadata() if (!inputHasGridOffsets()) { // If the input file doesn't have grid offsets, then all of the grids // have already been streamed in and stored in mGrids. - for (size_t i = 0, N = mImpl->mGrids->size(); i < N; ++i) { + for (size_t i = 0, N = mGrids->size(); i < N; ++i) { // Return copies of the grids, but with empty trees. - ret->push_back((*mImpl->mGrids)[i]->copyGridWithNewTree()); + ret->push_back((*mGrids)[i]->copyGridWithNewTree()); } } else { // Read just the metadata and transforms for all grids. - for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) { + for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; GridBase::ConstPtr grid = readGridPartial(gd); // Return copies of the grids, but with empty trees. @@ -457,8 +397,8 @@ File::readAllGridMetadata() GridBase::Ptr File::readGridMetadata(const Name& name) { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading."); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading."); } if (fileVersion() < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) { @@ -473,8 +413,8 @@ File::readGridMetadata(const Name& name) ret = readGrid(name); } else { NameMapCIter it = findDescriptor(name); - if (it == gridDescriptors().end()) { - OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\""); + if (it == mGridDescriptors.end()) { + OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\""); } // Seek to and read in the grid from the file. @@ -498,8 +438,8 @@ File::readGrid(const Name& name) GridBase::Ptr File::readGrid(const Name& name, const BBoxd& bbox) { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading."); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading."); } const bool clip = bbox.isSorted(); @@ -517,8 +457,8 @@ File::readGrid(const Name& name, const BBoxd& bbox) } NameMapCIter it = findDescriptor(name); - if (it == gridDescriptors().end()) { - OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\""); + if (it == mGridDescriptors.end()) { + OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\""); } // Seek to and read in the grid from the file. @@ -529,11 +469,11 @@ File::readGrid(const Name& name, const BBoxd& bbox) /// @todo Refactor to share code with Archive::connectInstance()? NameMapCIter parentIt = findDescriptor(GridDescriptor::nameAsString(gd.instanceParentName())); - if (parentIt == gridDescriptors().end()) { + if (parentIt == mGridDescriptors.end()) { OPENVDB_THROW(KeyError, "missing instance parent \"" << GridDescriptor::nameAsString(gd.instanceParentName()) << "\" for grid " << GridDescriptor::nameAsString(gd.uniqueName()) - << " in file " << filename()); + << " in file " << mFilename); } GridBase::Ptr parent; @@ -555,18 +495,18 @@ File::readGrid(const Name& name, const BBoxd& bbox) void File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta) const { - if (isOpen()) { + if (mIsOpen) { OPENVDB_THROW(IoError, - filename() << " cannot be written because it is open for reading"); + mFilename << " cannot be written because it is open for reading"); } // Create a file stream and write it out. std::ofstream file; - file.open(filename().c_str(), + file.open(mFilename.c_str(), std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); if (file.fail()) { - OPENVDB_THROW(IoError, "could not open " << filename() << " for writing"); + OPENVDB_THROW(IoError, "could not open " << mFilename << " for writing"); } // Write out the vdb. @@ -586,20 +526,20 @@ File::findDescriptor(const Name& name) const const Name uniqueName = GridDescriptor::stringAsUniqueName(name); // Find all descriptors with the given grid name. - std::pair range = gridDescriptors().equal_range(name); + std::pair range = mGridDescriptors.equal_range(name); if (range.first == range.second) { // If no descriptors were found with the given grid name, the name might have // a suffix ("name[N]"). In that case, remove the "[N]" suffix and search again. - range = gridDescriptors().equal_range(GridDescriptor::stripSuffix(uniqueName)); + range = mGridDescriptors.equal_range(GridDescriptor::stripSuffix(uniqueName)); } const size_t count = size_t(std::distance(range.first, range.second)); if (count > 1 && name == uniqueName) { - OPENVDB_LOG_WARN(filename() << " has more than one grid named \"" << name << "\""); + OPENVDB_LOG_WARN(mFilename << " has more than one grid named \"" << name << "\""); } - NameMapCIter ret = gridDescriptors().end(); + NameMapCIter ret = mGridDescriptors.end(); if (count > 0) { if (name == uniqueName) { @@ -632,7 +572,7 @@ File::createGrid(const GridDescriptor& gd) const if (!GridBase::isRegistered(gd.gridType())) { OPENVDB_THROW(KeyError, "Cannot read grid " << GridDescriptor::nameAsString(gd.uniqueName()) - << " from " << filename() << ": grid type " + << " from " << mFilename << ": grid type " << gd.gridType() << " is not registered"); } @@ -679,21 +619,39 @@ File::readGridPartial(const GridDescriptor& gd) const GridBase::Ptr File::readGrid(const GridDescriptor& gd) const { - return Impl::readGrid(*this, gd, Impl::NoBBox()); + // This method should not be called for files that don't contain grid offsets. + OPENVDB_ASSERT(inputHasGridOffsets()); + + GridBase::Ptr grid = createGrid(gd); + gd.seekToGrid(inputStream()); + Archive::readGrid(grid, gd, inputStream()); + return grid; } GridBase::Ptr File::readGrid(const GridDescriptor& gd, const BBoxd& bbox) const { - return Impl::readGrid(*this, gd, bbox); + // This method should not be called for files that don't contain grid offsets. + OPENVDB_ASSERT(inputHasGridOffsets()); + + GridBase::Ptr grid = createGrid(gd); + gd.seekToGrid(inputStream()); + Archive::readGrid(grid, gd, inputStream(), bbox); + return grid; } GridBase::Ptr File::readGrid(const GridDescriptor& gd, const CoordBBox& bbox) const { - return Impl::readGrid(*this, gd, bbox); + // This method should not be called for files that don't contain grid offsets. + OPENVDB_ASSERT(inputHasGridOffsets()); + + GridBase::Ptr grid = createGrid(gd); + gd.seekToGrid(inputStream()); + Archive::readGrid(grid, gd, inputStream(), bbox); + return grid; } @@ -703,17 +661,17 @@ File::readGrid(const GridDescriptor& gd, const CoordBBox& bbox) const File::NameIterator File::beginName() const { - if (!isOpen()) { - OPENVDB_THROW(IoError, filename() << " is not open for reading"); + if (!mIsOpen) { + OPENVDB_THROW(IoError, mFilename << " is not open for reading"); } - return File::NameIterator(gridDescriptors().begin()); + return File::NameIterator(mGridDescriptors.begin()); } File::NameIterator File::endName() const { - return File::NameIterator(gridDescriptors().end()); + return File::NameIterator(mGridDescriptors.end()); } } // namespace io diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 49ce98eb6d..a572b3f841 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -34,7 +34,7 @@ class OPENVDB_API File: public Archive using NameMapCIter = NameMap::const_iterator; explicit File(const std::string& filename); - ~File() override; + ~File() override { } /// @brief Copy constructor /// @details The copy will be closed and will not reference the same @@ -183,8 +183,22 @@ class OPENVDB_API File: public Archive friend class ::TestFile; friend class ::TestStream; - struct Impl; - std::unique_ptr mImpl; + std::string mFilename; + // The file-level metadata + MetaMap::Ptr mMeta; + // The file stream that is open for reading + std::unique_ptr mInStream; + // File-level stream metadata (file format, compression, etc.) + StreamMetadata::Ptr mStreamMetadata; + // Flag indicating if we have read in the global information (header, + // metadata, and grid descriptors) for this VDB file + bool mIsOpen = false; + // Grid descriptors for all grids stored in the file, indexed by grid name + NameMap mGridDescriptors; + // All grids, indexed by unique name (used only when mHasGridOffsets is false) + Archive::NamedGridMap mNamedGrids; + // All grids stored in the file (used only when mHasGridOffsets is false) + GridPtrVecPtr mGrids; }; diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 9120bed409..189fed9589 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -636,11 +636,27 @@ TestFile::testReadGridDescriptors() File file2("something.vdb2"); std::istringstream istr(ostr.str(), std::ios_base::binary); io::setCurrentVersion(istr); - file2.readGridDescriptors(istr); + // file2.readGridDescriptors(istr); + //////////////////////////// + file2.mGridDescriptors.clear(); + + for (int32_t i = 0, N = file2.readGridCount(istr); i < N; ++i) { + // Read the grid descriptor. + GridDescriptor gd; + gd.readHeader(istr); + gd.readStreamPos(istr); + + // Add the descriptor to the dictionary. + file2.mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); + + // Skip forward to the next descriptor. + gd.seekToEnd(istr); + } + //////////////////////////// // Compare with the initial grid descriptors. File::NameMapCIter it = file2.findDescriptor("temperature"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); GridDescriptor file2gd = it->second; EXPECT_EQ(gd.gridName(), file2gd.gridName()); EXPECT_EQ(gd.getGridPos(), file2gd.getGridPos()); @@ -648,7 +664,7 @@ TestFile::testReadGridDescriptors() EXPECT_EQ(gd.getEndPos(), file2gd.getEndPos()); it = file2.findDescriptor("density"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); file2gd = it->second; EXPECT_EQ(gd2.gridName(), file2gd.gridName()); EXPECT_EQ(gd2.getGridPos(), file2gd.getGridPos()); @@ -883,11 +899,27 @@ TestFile::testEmptyGridIO() File file2(filename); std::istringstream istr(ostr.str(), std::ios_base::binary); io::setCurrentVersion(istr); - file2.readGridDescriptors(istr); + // file2.readGridDescriptors(istr); + //////////////////////////// + file2.mGridDescriptors.clear(); + + for (int32_t i = 0, N = file2.readGridCount(istr); i < N; ++i) { + // Read the grid descriptor. + GridDescriptor gd; + gd.readHeader(istr); + gd.readStreamPos(istr); + + // Add the descriptor to the dictionary. + file2.mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); + + // Skip forward to the next descriptor. + gd.seekToEnd(istr); + } + //////////////////////////// // Compare with the initial grid descriptors. File::NameMapCIter it = file2.findDescriptor("temperature"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); GridDescriptor file2gd = it->second; file2gd.seekToGrid(istr); GridBase::Ptr gd_grid = GridBase::createGrid(file2gd.gridType()); @@ -905,7 +937,7 @@ TestFile::testEmptyGridIO() EXPECT_EQ(gd.getEndPos(), file2gd.getEndPos()); it = file2.findDescriptor("density"); - EXPECT_TRUE(it != file2.gridDescriptors().end()); + EXPECT_TRUE(it != file2.mGridDescriptors.end()); file2gd = it->second; file2gd.seekToGrid(istr); gd_grid = GridBase::createGrid(file2gd.gridType()); @@ -1000,16 +1032,16 @@ void TestFile::testOpen() EXPECT_EQ(2009, vdbfile.getMetadata()->metaValue("year")); // Ensure we got the grid descriptors. - EXPECT_EQ(1, int(vdbfile.gridDescriptors().count("density"))); - EXPECT_EQ(1, int(vdbfile.gridDescriptors().count("temperature"))); + EXPECT_EQ(1, int(vdbfile.mGridDescriptors.count("density"))); + EXPECT_EQ(1, int(vdbfile.mGridDescriptors.count("temperature"))); io::File::NameMapCIter it = vdbfile.findDescriptor("density"); - EXPECT_TRUE(it != vdbfile.gridDescriptors().end()); + EXPECT_TRUE(it != vdbfile.mGridDescriptors.end()); io::GridDescriptor gd = it->second; EXPECT_EQ(IntTree::treeType(), gd.gridType()); it = vdbfile.findDescriptor("temperature"); - EXPECT_TRUE(it != vdbfile.gridDescriptors().end()); + EXPECT_TRUE(it != vdbfile.mGridDescriptors.end()); gd = it->second; EXPECT_EQ(FloatTree::treeType(), gd.gridType()); @@ -1021,8 +1053,8 @@ void TestFile::testOpen() // Test closing the file. vdbfile.close(); EXPECT_TRUE(vdbfile.isOpen() == false); - EXPECT_TRUE(vdbfile.fileMetadata().get() == nullptr); - EXPECT_EQ(0, int(vdbfile.gridDescriptors().size())); + EXPECT_TRUE(vdbfile.mMeta.get() == nullptr); + EXPECT_EQ(0, int(vdbfile.mGridDescriptors.size())); EXPECT_THROW(vdbfile.inputStream(), openvdb::IoError); remove("something.vdb2"); From c388ae6431f39c52b69a92070b1197ba74047b1b Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 20 Mar 2026 11:25:38 -0700 Subject: [PATCH 21/80] Eliminate Stream::Impl Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Stream.cc | 69 +++++++++++------------------------- openvdb/openvdb/io/Stream.h | 10 +++--- 2 files changed, 26 insertions(+), 53 deletions(-) diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 80326af693..9d52a8a1c4 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -19,32 +19,8 @@ OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { namespace io { -struct Stream::Impl -{ - Impl(): mOutputStream{nullptr} {} - Impl(const Impl& other) { *this = other; } - Impl& operator=(const Impl& other) - { - if (&other != this) { - mMeta = other.mMeta; ///< @todo deep copy? - mGrids = other.mGrids; ///< @todo deep copy? - mOutputStream = other.mOutputStream; - mFile.reset(); - } - return *this; - } - - MetaMap::Ptr mMeta; - GridPtrVecPtr mGrids; - std::ostream* mOutputStream; - std::unique_ptr mFile; -}; - -//////////////////////////////////////// - - -Stream::Stream(std::istream& is): mImpl(new Impl) +Stream::Stream(std::istream& is) { if (!is) return; @@ -59,14 +35,14 @@ Stream::Stream(std::istream& is): mImpl(new Impl) io::setDataCompression(is, compression()); // Read in the VDB metadata. - mImpl->mMeta.reset(new MetaMap); - mImpl->mMeta->readMeta(is); + mMeta.reset(new MetaMap); + mMeta->readMeta(is); // Read in the number of grids. const int32_t gridCount = readGridCount(is); // Read in all grids and insert them into mGrids. - mImpl->mGrids.reset(new GridPtrVec); + mGrids.reset(new GridPtrVec); std::vector descriptors; descriptors.reserve(gridCount); Archive::NamedGridMap namedGrids; @@ -76,7 +52,7 @@ Stream::Stream(std::istream& is): mImpl(new Impl) gd.readStreamPos(is); descriptors.push_back(gd); GridBase::Ptr grid = readGrid(gd, is); - mImpl->mGrids->push_back(grid); + mGrids->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -87,23 +63,18 @@ Stream::Stream(std::istream& is): mImpl(new Impl) } -Stream::Stream(): mImpl(new Impl) -{ -} - - -Stream::Stream(std::ostream& os): mImpl(new Impl) -{ - mImpl->mOutputStream = &os; -} - - -Stream::~Stream() +Stream::Stream(std::ostream& os) + : Archive() + , mOutputStream(&os) { } -Stream::Stream(const Stream& other): Archive(other), mImpl(new Impl(*other.mImpl)) +Stream::Stream(const Stream& other) + : Archive(other) + , mMeta(other.mMeta) + , mGrids(other.mGrids) + , mOutputStream(other.mOutputStream) { } @@ -112,7 +83,9 @@ Stream& Stream::operator=(const Stream& other) { if (&other != this) { - mImpl.reset(new Impl(*other.mImpl)); + mMeta = other.mMeta; + mGrids = other.mGrids; + mOutputStream = other.mOutputStream; } return *this; } @@ -150,10 +123,10 @@ Stream::readGrid(const GridDescriptor& gd, std::istream& is) const void Stream::write(const GridCPtrVec& grids, const MetaMap& metadata) const { - if (mImpl->mOutputStream == nullptr) { + if (mOutputStream == nullptr) { OPENVDB_THROW(ValueError, "no output stream was specified"); } - this->writeGrids(*mImpl->mOutputStream, grids, metadata); + this->writeGrids(*mOutputStream, grids, metadata); } @@ -171,10 +144,10 @@ MetaMap::Ptr Stream::getMetadata() const { MetaMap::Ptr result; - if (mImpl->mMeta) { + if (mMeta) { // Return a deep copy of the file-level metadata // that was read when this object was constructed. - result.reset(new MetaMap(*mImpl->mMeta)); + result.reset(new MetaMap(*mMeta)); } return result; } @@ -183,7 +156,7 @@ Stream::getMetadata() const GridPtrVecPtr Stream::getGrids() { - return mImpl->mGrids; + return mGrids; } } // namespace io diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index 480f5bf074..d6644953b8 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -29,14 +29,14 @@ class OPENVDB_API Stream: public Archive Stream(std::istream& is, bool /*delayLoad*/) : Stream(is) { } /// Construct an archive for stream output. - Stream(); + Stream() = default; /// Construct an archive for output to the given stream. explicit Stream(std::ostream&); Stream(const Stream&); Stream& operator=(const Stream&); - ~Stream() override; + ~Stream() override { } /// @brief Return a copy of this archive. Archive::Ptr copy() const override; @@ -64,9 +64,9 @@ class OPENVDB_API Stream: public Archive void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&) const; - - struct Impl; - std::unique_ptr mImpl; + MetaMap::Ptr mMeta; + GridPtrVecPtr mGrids; + std::ostream* mOutputStream = nullptr; }; From aaa82b537e2d7fff7d7aa3df1001b03eecaf1ba0 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 20 Mar 2026 10:43:50 -0700 Subject: [PATCH 22/80] Refactor Archive::readGrid() to always take a BBox Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Archive.cc | 57 ++++++++--------------------------- openvdb/openvdb/io/Archive.h | 5 --- openvdb/openvdb/io/File.cc | 39 +++--------------------- openvdb/openvdb/io/File.h | 5 --- openvdb/openvdb/io/Stream.cc | 2 +- 5 files changed, 17 insertions(+), 91 deletions(-) diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 7ddd4f48b8..3d298390c4 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -886,23 +886,13 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co //////////////////////////////////////// -namespace { - -struct NoBBox {}; - -template void -doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const BoxType& bbox) +Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, + std::istream& is, const BBoxd& worldBBox) { - struct Local { - static void readBuffers(GridBase& g, std::istream& istrm, NoBBox) { g.readBuffers(istrm); } - static void readBuffers(GridBase& g, std::istream& istrm, const CoordBBox& indexBBox) { - g.readBuffers(istrm, indexBBox); - } - static void readBuffers(GridBase& g, std::istream& istrm, const BBoxd& worldBBox) { - g.readBuffers(istrm, g.constTransform().worldToIndexNodeCentered(worldBBox)); - } - }; + // Read the compression settings for this grid and tag the stream with them + // so that downstream functions can reference them. + readGridCompression(is); // Restore the file-level stream metadata on exit. struct OnExit { @@ -942,39 +932,16 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const grid->readTransform(is); if (!gd.isInstance()) { grid->readTopology(is); - Local::readBuffers(*grid, is, bbox); + const bool clip = worldBBox.isSorted(); + if (clip) { + const auto indexBBox = grid->constTransform().worldToIndexNodeCentered(worldBBox); + grid->readBuffers(is, indexBBox); + } else { + grid->readBuffers(is); + } } } -} // unnamed namespace - - -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is) -{ - // Read the compression settings for this grid and tag the stream with them - // so that downstream functions can reference them. - readGridCompression(is); - - doReadGrid(grid, gd, is, NoBBox()); -} - -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, - std::istream& is, const BBoxd& worldBBox) -{ - readGridCompression(is); - doReadGrid(grid, gd, is, worldBBox); -} - -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, - std::istream& is, const CoordBBox& indexBBox) -{ - readGridCompression(is); - doReadGrid(grid, gd, is, indexBBox); -} - //////////////////////////////////////// diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index 87f31b14eb..32b011ec26 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -127,14 +127,9 @@ class OPENVDB_API Archive /// Read in and return the number of grids on the input stream. static int32_t readGridCount(std::istream&); - /// Populate the given grid from the input stream. - static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&); /// @brief Populate the given grid from the input stream, but only where it /// intersects the given world-space bounding box. static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&, const BBoxd&); - /// @brief Populate the given grid from the input stream, but only where it - /// intersects the given index-space bounding box. - static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&, const CoordBBox&); using NamedGridMap = std::map; diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 0424c89da4..2273cdc34f 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -225,7 +225,7 @@ File::open() gd.readStreamPos(inputStream()); GridBase::Ptr grid = createGrid(gd); - Archive::readGrid(grid, gd, inputStream()); + Archive::readGrid(grid, gd, inputStream(), BBoxd()); mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); mGrids->push_back(grid); @@ -318,7 +318,7 @@ File::getGrids() const // Read all grids represented by the GridDescriptors. for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; - GridBase::Ptr grid = readGrid(gd); + GridBase::Ptr grid = readGrid(gd, BBoxd()); ret->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -463,7 +463,7 @@ File::readGrid(const Name& name, const BBoxd& bbox) // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; - grid = (clip ? readGrid(gd, bbox) : readGrid(gd)); + grid = readGrid(gd, bbox); if (gd.isInstance()) { /// @todo Refactor to share code with Archive::connectInstance()? @@ -477,12 +477,7 @@ File::readGrid(const Name& name, const BBoxd& bbox) } GridBase::Ptr parent; - if (clip) { - const CoordBBox indexBBox = grid->constTransform().worldToIndexNodeCentered(bbox); - parent = readGrid(parentIt->second, indexBBox); - } else { - parent = readGrid(parentIt->second); - } + parent = readGrid(parentIt->second, bbox); if (parent) grid->setTree(parent->baseTreePtr()); } return grid; @@ -616,19 +611,6 @@ File::readGridPartial(const GridDescriptor& gd) const } -GridBase::Ptr -File::readGrid(const GridDescriptor& gd) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - GridBase::Ptr grid = createGrid(gd); - gd.seekToGrid(inputStream()); - Archive::readGrid(grid, gd, inputStream()); - return grid; -} - - GridBase::Ptr File::readGrid(const GridDescriptor& gd, const BBoxd& bbox) const { @@ -642,19 +624,6 @@ File::readGrid(const GridDescriptor& gd, const BBoxd& bbox) const } -GridBase::Ptr -File::readGrid(const GridDescriptor& gd, const CoordBBox& bbox) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - GridBase::Ptr grid = createGrid(gd); - gd.seekToGrid(inputStream()); - Archive::readGrid(grid, gd, inputStream(), bbox); - return grid; -} - - //////////////////////////////////////// diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index a572b3f841..b837e2d5e4 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -156,14 +156,9 @@ class OPENVDB_API File: public Archive /// Read in and return the partially-populated grid specified by the given grid descriptor. GridBase::ConstPtr readGridPartial(const GridDescriptor&) const; - /// Read in and return the grid specified by the given grid descriptor. - GridBase::Ptr readGrid(const GridDescriptor&) const; /// Read in and return the region of the grid specified by the given grid descriptor /// that intersects the given world-space bounding box. GridBase::Ptr readGrid(const GridDescriptor&, const BBoxd&) const; - /// Read in and return the region of the grid specified by the given grid descriptor - /// that intersects the given index-space bounding box. - GridBase::Ptr readGrid(const GridDescriptor&, const CoordBBox&) const; /// @brief Retrieve a grid from @c mNamedGrids. Return a null pointer /// if @c mNamedGrids was not populated (because this file is random-access). diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 9d52a8a1c4..b37c393688 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -114,7 +114,7 @@ Stream::readGrid(const GridDescriptor& gd, std::istream& is) const grid = GridBase::createGrid(gd.gridType()); if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); - Archive::readGrid(grid, gd, is); + Archive::readGrid(grid, gd, is, BBoxd()); } return grid; } From 3b2ea9c636f05b9767915efc3ebe1e6c39be9a72 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 20 Mar 2026 11:26:25 -0700 Subject: [PATCH 23/80] Archive::readGrid() now creates the grid, remove File::readGridPartial() Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Archive.cc | 19 +++++-- openvdb/openvdb/io/Archive.h | 7 +-- openvdb/openvdb/io/File.cc | 93 +++++++---------------------------- openvdb/openvdb/io/File.h | 10 ---- openvdb/openvdb/io/Stream.cc | 21 +------- openvdb/openvdb/io/Stream.h | 5 -- 6 files changed, 38 insertions(+), 117 deletions(-) diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 3d298390c4..e0f37f5535 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -886,9 +886,8 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co //////////////////////////////////////// -void -Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, - std::istream& is, const BBoxd& worldBBox) +GridBase::Ptr +Archive::readGrid(const GridDescriptor& gd, std::istream& is, const BBoxd& worldBBox, bool partial/* = false*/) { // Read the compression settings for this grid and tag the stream with them // so that downstream functions can reference them. @@ -903,6 +902,16 @@ Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, }; OnExit restore(is); + // Create the grid. + if (!GridBase::isRegistered(gd.gridType())) { + OPENVDB_THROW(KeyError, "Cannot read grid " + << GridDescriptor::nameAsString(gd.uniqueName()) + << ": grid type " << gd.gridType() << " is not registered"); + } + + GridBase::Ptr grid = GridBase::createGrid(gd.gridType()); + if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); + // Stream metadata varies per grid, and it needs to persist // in case delayed load is in effect. io::StreamMetadata::Ptr streamMetadata; @@ -930,7 +939,7 @@ Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, io::setGridClass(is, gridClass); grid->readTransform(is); - if (!gd.isInstance()) { + if (!partial && !gd.isInstance()) { grid->readTopology(is); const bool clip = worldBBox.isSorted(); if (clip) { @@ -940,6 +949,8 @@ Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, grid->readBuffers(is); } } + + return grid; } diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index 32b011ec26..e61a0c8735 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -127,9 +127,10 @@ class OPENVDB_API Archive /// Read in and return the number of grids on the input stream. static int32_t readGridCount(std::istream&); - /// @brief Populate the given grid from the input stream, but only where it - /// intersects the given world-space bounding box. - static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&, const BBoxd&); + /// @brief Read in and create the grid represented by the given grid descriptor using the + /// given input stream, but only where it intersects the given world-space bounding box. + /// @param partial If true, only read the metadata and transform of the grid. + static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, const BBoxd&, bool partial = false); using NamedGridMap = std::map; diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 2273cdc34f..1e0e6acd35 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -224,8 +224,7 @@ File::open() gd.readHeader(inputStream()); gd.readStreamPos(inputStream()); - GridBase::Ptr grid = createGrid(gd); - Archive::readGrid(grid, gd, inputStream(), BBoxd()); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), BBoxd()); mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); mGrids->push_back(grid); @@ -318,7 +317,9 @@ File::getGrids() const // Read all grids represented by the GridDescriptors. for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; - GridBase::Ptr grid = readGrid(gd, BBoxd()); + // Seek to the grid in the file. + gd.seekToGrid(inputStream()); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), BBoxd()); ret->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -382,10 +383,12 @@ File::readAllGridMetadata() // Read just the metadata and transforms for all grids. for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) { const GridDescriptor& gd = i->second; - GridBase::ConstPtr grid = readGridPartial(gd); + // Seek to the grid in the file. + gd.seekToGrid(inputStream()); + GridBase::ConstPtr grid = Archive::readGrid(gd, inputStream(), BBoxd(), /*partial=*/true); // Return copies of the grids, but with empty trees. // (As of 0.98.0, at least, it would suffice to just const cast - // the grid pointers returned by readGridPartial(), but shallow + // the grid pointers returned by readGrid(partial=true), but shallow // copying the grids helps to ensure future compatibility.) ret->push_back(grid->copyGridWithNewTree()); } @@ -419,7 +422,8 @@ File::readGridMetadata(const Name& name) // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; - ret = readGridPartial(gd); + gd.seekToGrid(inputStream()); + ret = Archive::readGrid(gd, inputStream(), BBoxd(), /*partial=*/true); } return ret->copyGridWithNewTree(); } @@ -463,7 +467,11 @@ File::readGrid(const Name& name, const BBoxd& bbox) // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; - grid = readGrid(gd, bbox); + // This method should not be called for files that don't contain grid offsets. + OPENVDB_ASSERT(inputHasGridOffsets()); + // Seek to the grid in the file. + gd.seekToGrid(inputStream()); + grid = Archive::readGrid(gd, inputStream(), bbox); if (gd.isInstance()) { /// @todo Refactor to share code with Archive::connectInstance()? @@ -477,7 +485,9 @@ File::readGrid(const Name& name, const BBoxd& bbox) } GridBase::Ptr parent; - parent = readGrid(parentIt->second, bbox); + OPENVDB_ASSERT(inputHasGridOffsets()); + parentIt->second.seekToGrid(inputStream()); + parent = Archive::readGrid(parentIt->second, inputStream(), bbox); if (parent) grid->setTree(parent->baseTreePtr()); } return grid; @@ -560,73 +570,6 @@ File::findDescriptor(const Name& name) const //////////////////////////////////////// -GridBase::Ptr -File::createGrid(const GridDescriptor& gd) const -{ - // Create the grid. - if (!GridBase::isRegistered(gd.gridType())) { - OPENVDB_THROW(KeyError, "Cannot read grid " - << GridDescriptor::nameAsString(gd.uniqueName()) - << " from " << mFilename << ": grid type " - << gd.gridType() << " is not registered"); - } - - GridBase::Ptr grid = GridBase::createGrid(gd.gridType()); - if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); - - return grid; -} - - -GridBase::ConstPtr -File::readGridPartial(const GridDescriptor& gd) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - std::istream& is = inputStream(); - - GridBase::Ptr grid = createGrid(gd); - - // Seek to grid. - gd.seekToGrid(is); - - // This code needs to stay in sync with io::Archive::readGrid(), in terms of - // the order of operations. - readGridCompression(is); - grid->readMeta(is); - - // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists - if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) { - grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD); - } - - // Read the transform. - grid->readTransform(is); - - // Promote to a const grid. - GridBase::ConstPtr constGrid = grid; - - return constGrid; -} - - -GridBase::Ptr -File::readGrid(const GridDescriptor& gd, const BBoxd& bbox) const -{ - // This method should not be called for files that don't contain grid offsets. - OPENVDB_ASSERT(inputHasGridOffsets()); - - GridBase::Ptr grid = createGrid(gd); - gd.seekToGrid(inputStream()); - Archive::readGrid(grid, gd, inputStream(), bbox); - return grid; -} - - -//////////////////////////////////////// - - File::NameIterator File::beginName() const { diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index b837e2d5e4..8222263cf1 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -150,16 +150,6 @@ class OPENVDB_API File: public Archive /// If the name is non-unique, return an iterator to the first matching descriptor. NameMapCIter findDescriptor(const Name&) const; - /// Return a newly created, empty grid of the type specified by the given grid descriptor. - GridBase::Ptr createGrid(const GridDescriptor&) const; - - /// Read in and return the partially-populated grid specified by the given grid descriptor. - GridBase::ConstPtr readGridPartial(const GridDescriptor&) const; - - /// Read in and return the region of the grid specified by the given grid descriptor - /// that intersects the given world-space bounding box. - GridBase::Ptr readGrid(const GridDescriptor&, const BBoxd&) const; - /// @brief Retrieve a grid from @c mNamedGrids. Return a null pointer /// if @c mNamedGrids was not populated (because this file is random-access). /// @throw KeyError if no grid with the given name exists in this file. diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index b37c393688..9021fd7706 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -51,7 +51,7 @@ Stream::Stream(std::istream& is) gd.readHeader(is); gd.readStreamPos(is); descriptors.push_back(gd); - GridBase::Ptr grid = readGrid(gd, is); + GridBase::Ptr grid = Archive::readGrid(gd, is, BBoxd()); mGrids->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -101,25 +101,6 @@ Stream::copy() const //////////////////////////////////////// -GridBase::Ptr -Stream::readGrid(const GridDescriptor& gd, std::istream& is) const -{ - GridBase::Ptr grid; - - if (!GridBase::isRegistered(gd.gridType())) { - OPENVDB_THROW(TypeError, "can't read grid \"" - << GridDescriptor::nameAsString(gd.uniqueName()) << - "\" from input stream because grid type " << gd.gridType() << " is unknown"); - } else { - grid = GridBase::createGrid(gd.gridType()); - if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); - - Archive::readGrid(grid, gd, is, BBoxd()); - } - return grid; -} - - void Stream::write(const GridCPtrVec& grids, const MetaMap& metadata) const { diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index d6644953b8..4bdc1662ee 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -57,11 +57,6 @@ class OPENVDB_API Stream: public Archive void write(const GridPtrContainerT&, const MetaMap& = MetaMap()) const; private: - /// Create a new grid of the type specified by the given descriptor, - /// then populate the grid from the given input stream. - /// @return the newly created grid. - GridBase::Ptr readGrid(const GridDescriptor&, std::istream&) const; - void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&) const; MetaMap::Ptr mMeta; From f32548c6c2f001b6f5b2a29b0796bcf6a39dc764 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 16 Apr 2026 13:29:50 -0700 Subject: [PATCH 24/80] Update TestFile to use Archive::readGrid Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestFile.cc | 55 +++------------------------- 1 file changed, 6 insertions(+), 49 deletions(-) diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 189fed9589..47a26cbaf9 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -158,21 +158,7 @@ TestFile::testWriteGrid() gd2.readHeader(istr); gd2.readStreamPos(istr); - // manually create the grid - GridBase::Ptr gd2_grid = GridBase::createGrid(gd2.gridType()); - - EXPECT_EQ(gd.gridName(), gd2.gridName()); - EXPECT_EQ(GridType::gridType(), gd2_grid->type()); - EXPECT_EQ(gd.getGridPos(), gd2.getGridPos()); - EXPECT_EQ(gd.getBlockPos(), gd2.getBlockPos()); - EXPECT_EQ(gd.getEndPos(), gd2.getEndPos()); - - // Position the stream to beginning of the grid storage and read the grid. - gd2.seekToGrid(istr); - Archive::readGridCompression(istr); - gd2_grid->readMeta(istr); - gd2_grid->readTransform(istr); - gd2_grid->readTopology(istr); + GridBase::Ptr gd2_grid = Archive::readGrid(gd2, istr, BBoxd()); // Delay load metadata should not exist. ASSERT_FALSE(bool((*gd2_grid)["file_delayed_load"])); @@ -192,9 +178,6 @@ TestFile::testWriteGrid() EXPECT_EQ( grid->baseTree().treeDepth(), gd2_grid->baseTree().treeDepth()); - // Read in the data blocks. - gd2.seekToBlocks(istr); - gd2_grid->readBuffers(istr); TreeType::Ptr tree2 = DynamicPtrCast(gd2_grid->baseTreePtr()); EXPECT_TRUE(tree2.get() != nullptr); EXPECT_EQ(10, tree2->getValue(Coord(10, 1, 2))); @@ -260,7 +243,8 @@ TestFile::testWriteMultipleGrids() gd_in.readHeader(istr); gd_in.readStreamPos(istr); - GridBase::Ptr gd_in_grid = GridBase::createGrid(gd_in.gridType()); + + GridBase::Ptr gd_in_grid = Archive::readGrid(gd_in, istr, BBoxd()); // Ensure read in the right values. EXPECT_EQ(gd.gridName(), gd_in.gridName()); @@ -269,13 +253,6 @@ TestFile::testWriteMultipleGrids() EXPECT_EQ(gd.getBlockPos(), gd_in.getBlockPos()); EXPECT_EQ(gd.getEndPos(), gd_in.getEndPos()); - // Position the stream to beginning of the grid storage and read the grid. - gd_in.seekToGrid(istr); - Archive::readGridCompression(istr); - gd_in_grid->readMeta(istr); - gd_in_grid->readTransform(istr); - gd_in_grid->readTopology(istr); - // Ensure that we have the same topology and transform. EXPECT_EQ( grid->baseTree().leafCount(), gd_in_grid->baseTree().leafCount()); @@ -289,8 +266,6 @@ TestFile::testWriteMultipleGrids() // EXPECT_EQ(0.1, gd_in_grid->getTransform()->getVoxelSizeZ()); // Read in the data blocks. - gd_in.seekToBlocks(istr); - gd_in_grid->readBuffers(istr); TreeType::Ptr grid_in = DynamicPtrCast(gd_in_grid->baseTreePtr()); EXPECT_TRUE(grid_in.get() != nullptr); EXPECT_EQ(10, grid_in->getValue(Coord(10, 1, 2))); @@ -306,7 +281,7 @@ TestFile::testWriteMultipleGrids() GridDescriptor gd2_in; gd2_in.readHeader(istr); gd2_in.readStreamPos(istr); - GridBase::Ptr gd2_in_grid = GridBase::createGrid(gd2_in.gridType()); + GridBase::Ptr gd2_in_grid = Archive::readGrid(gd2_in, istr, BBoxd()); // Ensure that we read in the right values. EXPECT_EQ(gd2.gridName(), gd2_in.gridName()); @@ -315,13 +290,6 @@ TestFile::testWriteMultipleGrids() EXPECT_EQ(gd2.getBlockPos(), gd2_in.getBlockPos()); EXPECT_EQ(gd2.getEndPos(), gd2_in.getEndPos()); - // Position the stream to beginning of the grid storage and read the grid. - gd2_in.seekToGrid(istr); - Archive::readGridCompression(istr); - gd2_in_grid->readMeta(istr); - gd2_in_grid->readTransform(istr); - gd2_in_grid->readTopology(istr); - // Ensure that we have the same topology and transform. EXPECT_EQ( grid2->baseTree().leafCount(), gd2_in_grid->baseTree().leafCount()); @@ -333,9 +301,6 @@ TestFile::testWriteMultipleGrids() // EXPECT_EQ(0.2, gd2_in_grid->getTransform()->getVoxelSizeY()); // EXPECT_EQ(0.2, gd2_in_grid->getTransform()->getVoxelSizeZ()); - // Read in the data blocks. - gd2_in.seekToBlocks(istr); - gd2_in_grid->readBuffers(istr); TreeType::Ptr grid2_in = DynamicPtrCast(gd2_in_grid->baseTreePtr()); EXPECT_TRUE(grid2_in.get() != nullptr); EXPECT_EQ(50, grid2_in->getValue(Coord(1000, 1000, 1000))); @@ -922,11 +887,7 @@ TestFile::testEmptyGridIO() EXPECT_TRUE(it != file2.mGridDescriptors.end()); GridDescriptor file2gd = it->second; file2gd.seekToGrid(istr); - GridBase::Ptr gd_grid = GridBase::createGrid(file2gd.gridType()); - Archive::readGridCompression(istr); - gd_grid->readMeta(istr); - gd_grid->readTransform(istr); - gd_grid->readTopology(istr); + GridBase::Ptr gd_grid = Archive::readGrid(file2gd, istr, BBoxd()); EXPECT_EQ(gd.gridName(), file2gd.gridName()); EXPECT_TRUE(gd_grid.get() != nullptr); EXPECT_EQ(0, int(gd_grid->baseTree().leafCount())); @@ -940,11 +901,7 @@ TestFile::testEmptyGridIO() EXPECT_TRUE(it != file2.mGridDescriptors.end()); file2gd = it->second; file2gd.seekToGrid(istr); - gd_grid = GridBase::createGrid(file2gd.gridType()); - Archive::readGridCompression(istr); - gd_grid->readMeta(istr); - gd_grid->readTransform(istr); - gd_grid->readTopology(istr); + gd_grid = Archive::readGrid(file2gd, istr, BBoxd()); EXPECT_EQ(gd2.gridName(), file2gd.gridName()); EXPECT_TRUE(gd_grid.get() != nullptr); EXPECT_EQ(0, int(gd_grid->baseTree().leafCount())); From 518d4bda27fc0ae5013aeae62309aa2d256fd5fc Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 23 Mar 2026 13:50:14 -0700 Subject: [PATCH 25/80] Add io/Codec.h with ReadOptions and WriteOptions Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 2 ++ openvdb/openvdb/io/Codec.h | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 openvdb/openvdb/io/Codec.h diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 91fc5fcfd7..6a613f0326 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -366,6 +366,7 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/Archive.h + io/Codec.h io/Compression.h io/File.h io/GridDescriptor.h @@ -734,6 +735,7 @@ endif() install(FILES ${OPENVDB_LIBRARY_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/openvdb/version.h DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb) install(FILES ${OPENVDB_LIBRARY_IO_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/io) +install(FILES ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/codecs) install(FILES ${OPENVDB_LIBRARY_MATH_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/math) install(FILES ${OPENVDB_LIBRARY_POINTS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points) install(FILES ${OPENVDB_LIBRARY_POINTS_IMPL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points/impl) diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h new file mode 100644 index 0000000000..a96b076b41 --- /dev/null +++ b/openvdb/openvdb/io/Codec.h @@ -0,0 +1,51 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED + +#include + +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace io { + + +enum class ReadMode { + Original, // Read data as-is (default) + TopologyOnly // Read topology only, no buffer data +}; + +/// Global read configuration that applies to all grids +/// Contains common options shared across all codec types +struct OPENVDB_API ReadOptions +{ + using Ptr = std::shared_ptr; + + virtual ~ReadOptions() = default; + + // BBox Clip Data + BBoxd clipBBox = BBoxd(); + + // Grid Conversion Options + ReadMode readMode = ReadMode::Original; +}; // struct ReadOptions + +/// Global write configuration that applies to all grids +/// Contains common options shared across all codec types +struct OPENVDB_API WriteOptions +{ + using Ptr = std::shared_ptr; + + virtual ~WriteOptions() = default; +}; // struct WriteOptions + + +} // namespace io +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED From 7405a77417f6edd3df06f83a92a525aeaff8167b Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 23 Mar 2026 13:51:26 -0700 Subject: [PATCH 26/80] Pass writeOptions as an argument to all the I/O write routines Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Archive.cc | 14 +++++++------- openvdb/openvdb/io/Archive.h | 19 +++++++++++++------ openvdb/openvdb/io/File.cc | 4 ++-- openvdb/openvdb/io/File.h | 18 +++++++++++------- openvdb/openvdb/io/Stream.cc | 10 ++++++---- openvdb/openvdb/io/Stream.h | 14 +++++++++----- 6 files changed, 48 insertions(+), 31 deletions(-) diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index e0f37f5535..cf89a5aba1 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -959,15 +959,15 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const BBoxd& world void Archive::write(std::ostream& os, const GridPtrVec& grids, bool seekable, - const MetaMap& metadata) const + const MetaMap& metadata, const io::WriteOptions& writeOptions) const { - this->write(os, GridCPtrVec(grids.begin(), grids.end()), seekable, metadata); + this->write(os, GridCPtrVec(grids.begin(), grids.end()), seekable, metadata, writeOptions); } void Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, - const MetaMap& metadata) const + const MetaMap& metadata, const io::WriteOptions& writeOptions) const { // Set stream flags so that downstream functions can reference them. io::StreamMetadata::Ptr streamMetadata = io::getStreamMetadataPtr(os); @@ -1039,7 +1039,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, // Get the name of the other grid. gd.setInstanceParentName(mapIter->second.uniqueName()); // Write out this grid's descriptor and metadata, but not its tree. - writeGridInstance(gd, grid, os, seekable); + writeGridInstance(gd, grid, os, seekable, writeOptions); OPENVDB_LOG_DEBUG_RUNTIME("io::Archive::write(): " << GridDescriptor::nameAsString(gd.uniqueName()) @@ -1048,7 +1048,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, << GridDescriptor::nameAsString(gd.instanceParentName())); } else { // Write out the grid descriptor and its associated grid. - writeGrid(gd, grid, os, seekable); + writeGrid(gd, grid, os, seekable, writeOptions); // Record the grid's tree pointer so that the tree doesn't get written // more than once. treeMap[treePtr] = gd; @@ -1064,7 +1064,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, void Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable) const + std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const { // Restore file-level stream metadata on exit. struct OnExit { @@ -1143,7 +1143,7 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, void Archive::writeGridInstance(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable) const + std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const { // Write out the Descriptor's header information (grid name, type // and instance parent name). diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index e61a0c8735..8065a3b0b5 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -5,11 +5,14 @@ #define OPENVDB_IO_ARCHIVE_HAS_BEEN_INCLUDED #include -#include "Compression.h" // for COMPRESS_ZIP, etc. #include #include #include #include // for VersionId + +#include "Codec.h" +#include "Compression.h" // for COMPRESS_ZIP, etc. + #include #include #include @@ -89,7 +92,8 @@ class OPENVDB_API Archive void setGridStatsMetadataEnabled(bool b) { mEnableGridStats = b; } /// @brief Write the grids in the given container to this archive's output stream. - virtual void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const {} + virtual void write(const GridCPtrVec&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const {} /// @brief Return @c false (delayed loading has been removed). static bool isDelayedLoadingEnabled() { return false; } @@ -141,13 +145,14 @@ class OPENVDB_API Archive /// Write the given grid descriptor and grid to an output stream /// and update the GridDescriptor offsets. /// @param seekable if true, the output stream supports seek operations - void writeGrid(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable) const; + void writeGrid(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable, + const io::WriteOptions& writeOptions = io::WriteOptions{}) const; /// Write the given grid descriptor and grid metadata to an output stream /// and update the GridDescriptor offsets, but don't write the grid's tree, /// since it is shared with another grid. /// @param seekable if true, the output stream supports seek operations void writeGridInstance(GridDescriptor&, GridBase::ConstPtr, - std::ostream&, bool seekable) const; + std::ostream&, bool seekable, const io::WriteOptions& writeOptions = io::WriteOptions{}) const; /// @brief Read the magic number, version numbers, UUID, etc. from the given input stream. /// @return @c true if the input UUID differs from the previously-read UUID. @@ -159,8 +164,10 @@ class OPENVDB_API Archive //@{ /// Write the given grids to an output stream. - void write(std::ostream&, const GridPtrVec&, bool seekable, const MetaMap& = MetaMap()) const; - void write(std::ostream&, const GridCPtrVec&, bool seekable, const MetaMap& = MetaMap()) const; + void write(std::ostream&, const GridPtrVec&, bool seekable, const MetaMap&, + const io::WriteOptions& writeOptions = io::WriteOptions{}) const; + void write(std::ostream&, const GridCPtrVec&, bool seekable, const MetaMap&, + const io::WriteOptions& writeOptions = io::WriteOptions{}) const; //@} private: diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 1e0e6acd35..551716ee73 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -498,7 +498,7 @@ File::readGrid(const Name& name, const BBoxd& bbox) void -File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta) const +File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta, const io::WriteOptions& writeOptions) const { if (mIsOpen) { OPENVDB_THROW(IoError, @@ -515,7 +515,7 @@ File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta) const } // Write out the vdb. - Archive::write(file, grids, /*seekable=*/true, meta); + Archive::write(file, grids, /*seekable=*/true, meta, writeOptions); file.close(); } diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 8222263cf1..8752e45f5c 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -110,12 +110,14 @@ class OPENVDB_API File: public Archive /// @brief Write the grids in the given container to the file whose name /// was given in the constructor. - void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const override; + void write(const GridCPtrVec&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const override; /// @brief Write the grids in the given container to the file whose name /// was given in the constructor. template - void write(const GridPtrContainerT&, const MetaMap& = MetaMap()) const; + void write(const GridPtrContainerT&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const; /// A const iterator that iterates over all names in the file. This is only /// valid once the file has been opened. @@ -155,7 +157,7 @@ class OPENVDB_API File: public Archive /// @throw KeyError if no grid with the given name exists in this file. GridBase::Ptr retrieveCachedGrid(const Name&) const; - void writeGrids(const GridCPtrVec&, const MetaMap&) const; + void writeGrids(const GridCPtrVec&, const MetaMap&, const io::WriteOptions&) const; MetaMap::Ptr fileMetadata(); MetaMap::ConstPtr fileMetadata() const; @@ -191,19 +193,21 @@ class OPENVDB_API File: public Archive inline void -File::write(const GridCPtrVec& grids, const MetaMap& meta) const +File::write(const GridCPtrVec& grids, const MetaMap& meta, + const io::WriteOptions& writeOptions) const { - this->writeGrids(grids, meta); + this->writeGrids(grids, meta, writeOptions); } template inline void -File::write(const GridPtrContainerT& container, const MetaMap& meta) const +File::write(const GridPtrContainerT& container, const MetaMap& meta, + const io::WriteOptions& writeOptions) const { GridCPtrVec grids; std::copy(container.begin(), container.end(), std::back_inserter(grids)); - this->writeGrids(grids, meta); + this->writeGrids(grids, meta, writeOptions); } } // namespace io diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 9021fd7706..b65e32703a 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -102,19 +102,21 @@ Stream::copy() const void -Stream::write(const GridCPtrVec& grids, const MetaMap& metadata) const +Stream::write(const GridCPtrVec& grids, const MetaMap& metadata, + const io::WriteOptions& writeOptions) const { if (mOutputStream == nullptr) { OPENVDB_THROW(ValueError, "no output stream was specified"); } - this->writeGrids(*mOutputStream, grids, metadata); + this->writeGrids(*mOutputStream, grids, metadata, writeOptions); } void -Stream::writeGrids(std::ostream& os, const GridCPtrVec& grids, const MetaMap& metadata) const +Stream::writeGrids(std::ostream& os, const GridCPtrVec& grids, const MetaMap& metadata, + const io::WriteOptions& writeOptions) const { - Archive::write(os, grids, /*seekable=*/false, metadata); + Archive::write(os, grids, /*seekable=*/false, metadata, writeOptions); } diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index 4bdc1662ee..418a06b142 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -49,15 +49,18 @@ class OPENVDB_API Stream: public Archive /// @brief Write the grids in the given container to this archive's output stream. /// @throw ValueError if this archive was constructed without specifying an output stream. - void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const override; + void write(const GridCPtrVec&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const override; /// @brief Write the grids in the given container to this archive's output stream. /// @throw ValueError if this archive was constructed without specifying an output stream. template - void write(const GridPtrContainerT&, const MetaMap& = MetaMap()) const; + void write(const GridPtrContainerT&, const MetaMap& = MetaMap(), + const io::WriteOptions& = io::WriteOptions{}) const; private: - void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&) const; + void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&, + const io::WriteOptions&) const; MetaMap::Ptr mMeta; GridPtrVecPtr mGrids; @@ -70,11 +73,12 @@ class OPENVDB_API Stream: public Archive template inline void -Stream::write(const GridPtrContainerT& container, const MetaMap& metadata) const +Stream::write(const GridPtrContainerT& container, const MetaMap& metadata, + const io::WriteOptions& writeOptions) const { GridCPtrVec grids; std::copy(container.begin(), container.end(), std::back_inserter(grids)); - this->write(grids, metadata); + this->write(grids, metadata, writeOptions); } } // namespace io From de3a3e0813e868b5402fce87dbf4bf2e31ab5a17 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 23 Mar 2026 17:44:57 -0700 Subject: [PATCH 27/80] Pass readOptions as an argument to all the I/O read routines Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Archive.cc | 9 +++++---- openvdb/openvdb/io/Archive.h | 6 +++--- openvdb/openvdb/io/File.cc | 30 +++++++++++++++++----------- openvdb/openvdb/io/File.h | 6 +++--- openvdb/openvdb/io/Stream.cc | 2 +- openvdb/openvdb/unittest/TestFile.cc | 10 +++++----- 6 files changed, 35 insertions(+), 28 deletions(-) diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index cf89a5aba1..4eb9d0fe74 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -887,7 +887,7 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co GridBase::Ptr -Archive::readGrid(const GridDescriptor& gd, std::istream& is, const BBoxd& worldBBox, bool partial/* = false*/) +Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions) { // Read the compression settings for this grid and tag the stream with them // so that downstream functions can reference them. @@ -939,8 +939,9 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const BBoxd& world io::setGridClass(is, gridClass); grid->readTransform(is); - if (!partial && !gd.isInstance()) { + if (readOptions.readMode != io::ReadMode::TopologyOnly && !gd.isInstance()) { grid->readTopology(is); + const auto& worldBBox = readOptions.clipBBox; const bool clip = worldBBox.isSorted(); if (clip) { const auto indexBBox = grid->constTransform().worldToIndexNodeCentered(worldBBox); @@ -1064,7 +1065,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, void Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const + std::ostream& os, bool seekable, const io::WriteOptions&) const { // Restore file-level stream metadata on exit. struct OnExit { @@ -1143,7 +1144,7 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, void Archive::writeGridInstance(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const + std::ostream& os, bool seekable, const io::WriteOptions&) const { // Write out the Descriptor's header information (grid name, type // and instance parent name). diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index 8065a3b0b5..a3b0893ea6 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -132,9 +132,9 @@ class OPENVDB_API Archive static int32_t readGridCount(std::istream&); /// @brief Read in and create the grid represented by the given grid descriptor using the - /// given input stream, but only where it intersects the given world-space bounding box. - /// @param partial If true, only read the metadata and transform of the grid. - static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, const BBoxd&, bool partial = false); + /// given input stream, using the provided options if given. + static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, + const io::ReadOptions& readOptions = io::ReadOptions{}); using NamedGridMap = std::map; diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 551716ee73..bd4ebdd4f5 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -224,7 +224,7 @@ File::open() gd.readHeader(inputStream()); gd.readStreamPos(inputStream()); - GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), BBoxd()); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), io::ReadOptions{}); mGridDescriptors.insert(std::make_pair(gd.gridName(), gd)); mGrids->push_back(grid); @@ -298,7 +298,7 @@ File::getMetadata() const GridPtrVecPtr -File::getGrids() const +File::getGrids(const io::ReadOptions& readOptions) const { if (!mIsOpen) { OPENVDB_THROW(IoError, mFilename << " is not open for reading"); @@ -319,7 +319,7 @@ File::getGrids() const const GridDescriptor& gd = i->second; // Seek to the grid in the file. gd.seekToGrid(inputStream()); - GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), BBoxd()); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), readOptions); ret->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -385,7 +385,9 @@ File::readAllGridMetadata() const GridDescriptor& gd = i->second; // Seek to the grid in the file. gd.seekToGrid(inputStream()); - GridBase::ConstPtr grid = Archive::readGrid(gd, inputStream(), BBoxd(), /*partial=*/true); + io::ReadOptions readOptions; + readOptions.readMode = io::ReadMode::TopologyOnly; + GridBase::ConstPtr grid = Archive::readGrid(gd, inputStream(), readOptions); // Return copies of the grids, but with empty trees. // (As of 0.98.0, at least, it would suffice to just const cast // the grid pointers returned by readGrid(partial=true), but shallow @@ -423,7 +425,9 @@ File::readGridMetadata(const Name& name) // Seek to and read in the grid from the file. const GridDescriptor& gd = it->second; gd.seekToGrid(inputStream()); - ret = Archive::readGrid(gd, inputStream(), BBoxd(), /*partial=*/true); + io::ReadOptions readOptions; + readOptions.readMode = io::ReadMode::TopologyOnly; + ret = Archive::readGrid(gd, inputStream(), readOptions); } return ret->copyGridWithNewTree(); } @@ -433,26 +437,28 @@ File::readGridMetadata(const Name& name) GridBase::Ptr -File::readGrid(const Name& name) +File::readGrid(const Name& name, const BBoxd& bbox) { - return readGrid(name, BBoxd()); + io::ReadOptions readOptions; + readOptions.clipBBox = bbox; + return readGrid(name, readOptions); } GridBase::Ptr -File::readGrid(const Name& name, const BBoxd& bbox) +File::readGrid(const Name& name, const io::ReadOptions& readOptions) { if (!mIsOpen) { OPENVDB_THROW(IoError, mFilename << " is not open for reading."); } - const bool clip = bbox.isSorted(); - // If a grid with the given name was already read and cached // (along with the entire contents of the file, because the file // doesn't support random access), retrieve and return it. GridBase::Ptr grid = retrieveCachedGrid(name); if (grid) { + const auto& bbox = readOptions.clipBBox; + const bool clip = bbox.isSorted(); if (clip) { grid = grid->deepCopyGrid(); grid->clipGrid(bbox); @@ -471,7 +477,7 @@ File::readGrid(const Name& name, const BBoxd& bbox) OPENVDB_ASSERT(inputHasGridOffsets()); // Seek to the grid in the file. gd.seekToGrid(inputStream()); - grid = Archive::readGrid(gd, inputStream(), bbox); + grid = Archive::readGrid(gd, inputStream(), readOptions); if (gd.isInstance()) { /// @todo Refactor to share code with Archive::connectInstance()? @@ -487,7 +493,7 @@ File::readGrid(const Name& name, const BBoxd& bbox) GridBase::Ptr parent; OPENVDB_ASSERT(inputHasGridOffsets()); parentIt->second.seekToGrid(inputStream()); - parent = Archive::readGrid(parentIt->second, inputStream(), bbox); + parent = Archive::readGrid(parentIt->second, inputStream(), readOptions); if (parent) grid->setTree(parent->baseTreePtr()); } return grid; diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 8752e45f5c..920b86a50f 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -87,7 +87,7 @@ class OPENVDB_API File: public Archive MetaMap::Ptr getMetadata() const; /// Read the entire contents of the file and return a list of grid pointers. - GridPtrVecPtr getGrids() const; + GridPtrVecPtr getGrids(const io::ReadOptions& readOptions = io::ReadOptions{}) const; /// @brief Read just the grid metadata and transforms from the file and return a list /// of pointers to grids that are empty except for their metadata and transforms. @@ -100,8 +100,8 @@ class OPENVDB_API File: public Archive /// @throw KeyError if no grid with the given name exists in this file. GridBase::Ptr readGridMetadata(const Name&); - /// Read an entire grid, including all of its data blocks. - GridBase::Ptr readGrid(const Name&); + /// Read an entire grid, including all of its data blocks, using the provided options if given. + GridBase::Ptr readGrid(const Name&, const io::ReadOptions& readOptions = io::ReadOptions{}); /// @brief Read a grid, including its data blocks, but only where it /// intersects the given world-space bounding box. GridBase::Ptr readGrid(const Name&, const BBoxd&); diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index b65e32703a..938cc321b0 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -51,7 +51,7 @@ Stream::Stream(std::istream& is) gd.readHeader(is); gd.readStreamPos(is); descriptors.push_back(gd); - GridBase::Ptr grid = Archive::readGrid(gd, is, BBoxd()); + GridBase::Ptr grid = Archive::readGrid(gd, is, io::ReadOptions{}); mGrids->push_back(grid); namedGrids[gd.uniqueName()] = grid; } diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 47a26cbaf9..fad51e9567 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -158,7 +158,7 @@ TestFile::testWriteGrid() gd2.readHeader(istr); gd2.readStreamPos(istr); - GridBase::Ptr gd2_grid = Archive::readGrid(gd2, istr, BBoxd()); + GridBase::Ptr gd2_grid = Archive::readGrid(gd2, istr); // Delay load metadata should not exist. ASSERT_FALSE(bool((*gd2_grid)["file_delayed_load"])); @@ -244,7 +244,7 @@ TestFile::testWriteMultipleGrids() gd_in.readHeader(istr); gd_in.readStreamPos(istr); - GridBase::Ptr gd_in_grid = Archive::readGrid(gd_in, istr, BBoxd()); + GridBase::Ptr gd_in_grid = Archive::readGrid(gd_in, istr); // Ensure read in the right values. EXPECT_EQ(gd.gridName(), gd_in.gridName()); @@ -281,7 +281,7 @@ TestFile::testWriteMultipleGrids() GridDescriptor gd2_in; gd2_in.readHeader(istr); gd2_in.readStreamPos(istr); - GridBase::Ptr gd2_in_grid = Archive::readGrid(gd2_in, istr, BBoxd()); + GridBase::Ptr gd2_in_grid = Archive::readGrid(gd2_in, istr); // Ensure that we read in the right values. EXPECT_EQ(gd2.gridName(), gd2_in.gridName()); @@ -887,7 +887,7 @@ TestFile::testEmptyGridIO() EXPECT_TRUE(it != file2.mGridDescriptors.end()); GridDescriptor file2gd = it->second; file2gd.seekToGrid(istr); - GridBase::Ptr gd_grid = Archive::readGrid(file2gd, istr, BBoxd()); + GridBase::Ptr gd_grid = Archive::readGrid(file2gd, istr); EXPECT_EQ(gd.gridName(), file2gd.gridName()); EXPECT_TRUE(gd_grid.get() != nullptr); EXPECT_EQ(0, int(gd_grid->baseTree().leafCount())); @@ -901,7 +901,7 @@ TestFile::testEmptyGridIO() EXPECT_TRUE(it != file2.mGridDescriptors.end()); file2gd = it->second; file2gd.seekToGrid(istr); - gd_grid = Archive::readGrid(file2gd, istr, BBoxd()); + gd_grid = Archive::readGrid(file2gd, istr); EXPECT_EQ(gd2.gridName(), file2gd.gridName()); EXPECT_TRUE(gd_grid.get() != nullptr); EXPECT_EQ(0, int(gd_grid->baseTree().leafCount())); From a6eaed5b390c5abc1e2e38617dd4cbb2fc9e2e54 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 27 Mar 2026 17:25:41 -0700 Subject: [PATCH 28/80] Make ReadOptions and WriteOptions non-virtual and remove Ptr typdefs Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Codec.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index a96b076b41..bd257056a1 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -23,10 +23,6 @@ enum class ReadMode { /// Contains common options shared across all codec types struct OPENVDB_API ReadOptions { - using Ptr = std::shared_ptr; - - virtual ~ReadOptions() = default; - // BBox Clip Data BBoxd clipBBox = BBoxd(); @@ -38,9 +34,6 @@ struct OPENVDB_API ReadOptions /// Contains common options shared across all codec types struct OPENVDB_API WriteOptions { - using Ptr = std::shared_ptr; - - virtual ~WriteOptions() = default; }; // struct WriteOptions From 2ee82e57ccd7a0c7c7de2211793f9818f6f13182 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 23 Mar 2026 18:57:12 -0700 Subject: [PATCH 29/80] Add BoolCodec, Codec Registry and Archive integration Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 7 + openvdb/openvdb/codecs/BoolCodec.h | 154 +++++++++++ openvdb/openvdb/codecs/TopologyCodec.h | 326 ++++++++++++++++++++++++ openvdb/openvdb/io/Archive.cc | 77 ++++-- openvdb/openvdb/io/Archive.h | 3 + openvdb/openvdb/io/Codec.cc | 106 ++++++++ openvdb/openvdb/io/Codec.h | 92 ++++++- openvdb/openvdb/openvdb.cc | 4 + openvdb/openvdb/unittest/CMakeLists.txt | 1 + openvdb/openvdb/unittest/TestCodec.cc | 185 ++++++++++++++ 10 files changed, 937 insertions(+), 18 deletions(-) create mode 100644 openvdb/openvdb/codecs/BoolCodec.h create mode 100644 openvdb/openvdb/codecs/TopologyCodec.h create mode 100644 openvdb/openvdb/io/Codec.cc create mode 100644 openvdb/openvdb/unittest/TestCodec.cc diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 6a613f0326..f0fd0fed5a 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -329,6 +329,7 @@ configure_file(version.h.in openvdb/version.h) set(OPENVDB_LIBRARY_SOURCE_FILES Grid.cc io/Archive.cc + io/Codec.cc io/Compression.cc io/File.cc io/GridDescriptor.cc @@ -364,6 +365,11 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES TypeList.h ) +set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES + codecs/BoolCodec.h + codecs/TopologyCodec.h +) + set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/Archive.h io/Codec.h @@ -559,6 +565,7 @@ if(USE_EXPLICIT_INSTANTIATION) # inexpensive to generate in this case. set(OPENVDB_LIBRARY_INSTANTIATION_HEADERS ${OPENVDB_LIBRARY_TOOLS_INCLUDE_FILES} + ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} ) # A new source file is generated by CMake for every header diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h new file mode 100644 index 0000000000..2aafb86aaa --- /dev/null +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -0,0 +1,154 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct ReadBoolBuffersOp +{ + using TreeT = typename GridT::TreeType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + ReadBoolBuffersOp(std::istream& _is, bool _saveFloatAsHalf, + const ValueT& _background, const CoordBBox* _clipBBox) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) + , clipBBox(_clipBBox) { } + + void operator()(RootT& root, size_t) + { + // Clip root-level tiles and prune children that were clipped. + if (clipBBox) { + root.clip(*clipBBox); + } + } + + template + void operator()(NodeT& node, size_t) + { + // Clip internal node tiles and prune children that were clipped. + if (clipBBox) { + node.clip(*clipBBox, background); + } + } + + void operator()(LeafT& leaf, size_t) + { + // Read in the value mask. + leaf.getValueMask().load(is); + // Read in the origin. + Coord origin; + is.read(reinterpret_cast(&origin), sizeof(Coord::ValueType) * 3); + leaf.setOrigin(origin); + + // Read in the mask for the voxel values. + typename LeafT::Buffer::NodeMaskType nodeMask; + nodeMask.load(is); + typename LeafT::Buffer temp(nodeMask); + leaf.swap(temp); + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; + const CoordBBox* clipBBox = nullptr; +}; // struct ReadBoolBuffersOp + +template +struct WriteBoolBuffersOp +{ + using TreeT = typename GridT::TreeType; + using LeafT = typename TreeT::LeafNodeType; + + WriteBoolBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + // Write out the value mask. + leaf.getValueMask().save(os); + + // Write out the origin. + os.write(reinterpret_cast(&leaf.origin()), sizeof(Coord::ValueType) * 3); + + // Write out the voxel values. + leaf.buffer().storage().save(os); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WriteBoolBuffersOp + +} // namespace internal + +template +struct BoolCodec : public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~BoolCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options) final + { + GridT& grid = static_cast(*data.grid); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in BoolCodec"); + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + std::unique_ptr clipIndexBBox; + if (options.clipBBox.isSorted()) { + clipIndexBBox = std::make_unique(grid.constTransform().worldToIndexNodeCentered(options.clipBBox)); + } + + internal::ReadBoolBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), clipIndexBBox.get()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + const GridT& grid = static_cast(gridBase); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in BoolCodec"); + } + + internal::WriteBoolBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); + } +}; // struct BoolCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h new file mode 100644 index 0000000000..0f411598c0 --- /dev/null +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -0,0 +1,326 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED + +#include +#include +#include +#include +#include +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct WriteTopologyOp +{ + using ValueT = typename TreeT::ValueType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + + WriteTopologyOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + void operator()(const RootT& root, size_t) + { + int32_t bufferCount = 1; + os.write(reinterpret_cast(&bufferCount), sizeof(int32_t)); + + background = &root.background(); + if (!saveFloatAsHalf) { + os.write(reinterpret_cast(background), sizeof(ValueT)); + } else { + ValueT truncatedVal = io::truncateRealToHalf(*background); + os.write(reinterpret_cast(&truncatedVal), sizeof(ValueT)); + } + io::setGridBackgroundValuePtr(os, background); + + const Index numTiles = root.tileCount(), numChildren = root.childCount(); + os.write(reinterpret_cast(&numTiles), sizeof(Index)); + os.write(reinterpret_cast(&numChildren), sizeof(Index)); + + // Write tiles. + for (auto iter = root.cbeginValueAll(); iter; ++iter) { + const auto& ijk = iter.getCoord(); + os.write(reinterpret_cast(ijk.asPointer()), 3 * sizeof(Int32)); + ValueT value; + bool active = root.getTileValueUnsafe(ijk, value); + os.write(reinterpret_cast(&value), sizeof(ValueT)); + os.write(reinterpret_cast(&active), sizeof(bool)); + } + + rootChildLevel = RootT::LEVEL-1; + } + + template + void operator()(const NodeT& node, size_t) + { + // Write origin for RootNode children only + if (rootChildLevel == NodeT::LEVEL) { + const auto& ijk = node.origin(); + os.write(reinterpret_cast(ijk.asPointer()), 3 * sizeof(Int32)); + } + + const auto& childMask = node.getChildMask(); + const auto& valueMask = node.getValueMask(); + + childMask.save(os); + valueMask.save(os); + + { + // Copy all of this node's values into an array. + std::unique_ptr valuePtr(new ValueT[NodeT::NUM_VALUES]); + ValueT* values = valuePtr.get(); + const ValueT zero = zeroVal(); + for (Index i = 0; i < NodeT::NUM_VALUES; ++i) { + values[i] = (node.isChildMaskOff(i) ? node.getValueUnsafe(i) : zero); + } + // Compress (optionally) and write out the contents of the array. + io::writeCompressedValues(os, values, NodeT::NUM_VALUES, valueMask, childMask, saveFloatAsHalf); + } + } + + void operator()(const LeafT& leaf, size_t) + { + leaf.getValueMask().save(os); + } + + std::ostream& os; + const bool saveFloatAsHalf; + const ValueT* background = nullptr; + Index rootChildLevel = std::numeric_limits::max(); +}; // struct WriteTopologyOp + + +template +struct ReadTopologyOp +{ + using ValueT = typename TreeT::ValueType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using StorageValueT = typename StorageTreeT::ValueType; + + ReadTopologyOp(std::istream& _is, bool _saveFloatAsHalf) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + void operator()(RootT& root) + { + using ChildT = typename RootT::ChildNodeType; + + int32_t bufferCount; + is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); + if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported"); + + // Delete the existing tree. + root.clear(); + + // Read a RootNode that was stored in the current format. + + if constexpr (std::is_same_v) { + is.read(reinterpret_cast(&background), sizeof(ValueT)); + } else { + StorageValueT _background; + is.read(reinterpret_cast(&_background), sizeof(StorageValueT)); + background = static_cast(_background); + } + + root.setBackground(background, false); + io::setGridBackgroundValuePtr(is, &root.background()); + + Index numTiles = 0, numChildren = 0; + is.read(reinterpret_cast(&numTiles), sizeof(Index)); + is.read(reinterpret_cast(&numChildren), sizeof(Index)); + + Int32 vec[3]; + StorageValueT value; + bool active; + + // Read tiles. + for (Index n = 0; n < numTiles; ++n) { + is.read(reinterpret_cast(vec), 3 * sizeof(Int32)); + is.read(reinterpret_cast(&value), sizeof(StorageValueT)); + is.read(reinterpret_cast(&active), sizeof(bool)); + Coord origin(vec); + if constexpr (std::is_same_v) { + root.addTile(origin, value, active); + } else { + root.addTile(origin, static_cast(value), active); + } + } + + // Read child nodes. + for (Index n = 0; n < numChildren; ++n) { + is.read(reinterpret_cast(vec), 3 * sizeof(Int32)); + Coord origin(vec); + ChildT* child = new ChildT(PartialCreate(), origin, background); + (*this)(*child); + root.addChild(child); + } + } + + template + void operator()(NodeT& node) + { + using ChildT = typename NodeT::ChildNodeType; + using NodeMaskT = typename NodeT::NodeMaskType; + + NodeMaskT childMask, valueMask; + childMask.load(is); + valueMask.load(is); + node.setValueMaskUnsafe(valueMask); + + const Index numValues = NodeT::NUM_VALUES; + { + // Read in (and uncompress, if necessary) all of this node's values + // into a contiguous array. + std::unique_ptr valuePtr(new StorageValueT[numValues]); + StorageValueT* values = valuePtr.get(); + io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf); + + // Copy values from the array into this node's table. + if constexpr (std::is_same_v) { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), values[iter.pos()]); + } + } else { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); + } + } + } + + // Read in all child nodes and insert them into the table at their proper locations. + for (auto iter = childMask.beginOn(); iter; ++iter) { + Coord origin = node.offsetToGlobalCoord(iter.pos()); + auto* child = new ChildT(PartialCreate(), origin, background); + (*this)(*child); + node.setChildUnsafe(iter.pos(), child); + } + } + + void operator()(LeafT& leaf) + { + typename LeafT::NodeMaskType valueMask; + valueMask.load(is); + leaf.setValueMask(valueMask); + } + + std::istream& is; + bool saveFloatAsHalf; + ValueT background; +}; // struct ReadTopologyOp + +template +struct SetTilesToBackgroundOp +{ + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + explicit SetTilesToBackgroundOp(const ValueT& background) + : mBackground(background) { } + + bool operator()(RootT& root, size_t) const + { + for (auto it = root.beginValueAll(); it; ++it) { + it.setValue(mBackground); + } + return true; + } + + template + bool operator()(NodeT& node, size_t) const + { + for (Index i = 0; i < NodeT::NUM_VALUES; ++i) { + if (node.isChildMaskOff(i)) { + node.setValueOnlyUnsafe(i, mBackground); + } + } + return true; + } + + bool operator()(LeafT&, size_t) const { return true; } + +private: + const ValueT mBackground; +}; // struct SetTilesToBackgroundOp + +template +void setTilesToBackground(TreeT& tree) +{ + const typename TreeT::ValueType& background = tree.root().background(); + SetTilesToBackgroundOp op(background); + tree::DynamicNodeManager nodeManager(tree); + nodeManager.foreachTopDown(op); +} + +// Free-standing function for read case (supports type conversion via StorageGridT) +template +void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options) +{ + io::checkFormatVersion(is); + + GridT& grid = static_cast(gridBase); + grid.tree().clearAllAccessors(); + + internal::ReadTopologyOp readTopologyOp(is, grid.saveFloatAsHalf()); + readTopologyOp(grid.tree().root()); + + if (options.readMode == io::ReadMode::TopologyOnly) { + internal::setTilesToBackground(grid.tree()); + return; + } +} + +// Free-standing function for write case (no StorageGridT needed) +template +void topologyCodecWriteTopology(const GridBase& gridBase, std::ostream& os) +{ + const GridT& grid = static_cast(gridBase); + using TreeType = typename GridT::TreeType; + + internal::WriteTopologyOp writeTopologyOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeTopologyOp); +} + +} // namespace internal + +template +struct TopologyCodec : public io::Codec +{ + using Ptr = std::unique_ptr>; + + ~TopologyCodec() noexcept = default; + + io::CodecData::Ptr createData() override + { + auto data = std::make_unique(); + data->grid = GridT::create(); + return data; + } + + void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options) final + { + internal::topologyCodecReadTopology(*data.grid, is, options); + } + + void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + internal::topologyCodecWriteTopology(gridBase, os); + } +}; // struct TopologyCodec + + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 4eb9d0fe74..712d3d2b99 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -855,6 +855,17 @@ Archive::readGridCount(std::istream& is) //////////////////////////////////////// +io::Codec* +Archive::findCodec(const std::string& gridType, const io::ReadOptions&) +{ + // Determine the I/O codec to use to read this grid + return io::CodecRegistry::get(gridType); +} + + +//////////////////////////////////////// + + void Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) const { @@ -902,15 +913,26 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti }; OnExit restore(is); + // Find the codec for the grid type and options. + io::Codec* codec = findCodec(gd.gridType(), readOptions); + + GridBase::Ptr grid; + io::CodecData::Ptr codecData; + // Create the grid. - if (!GridBase::isRegistered(gd.gridType())) { - OPENVDB_THROW(KeyError, "Cannot read grid " - << GridDescriptor::nameAsString(gd.uniqueName()) - << ": grid type " << gd.gridType() << " is not registered"); - } + if (codec) { + codecData = codec->createData(); + if (codecData->grid) grid = codecData->grid; + } else { + if (!GridBase::isRegistered(gd.gridType())) { + OPENVDB_THROW(KeyError, "Cannot read grid " + << GridDescriptor::nameAsString(gd.uniqueName()) + << ": grid type " << gd.gridType() << " is not registered"); + } - GridBase::Ptr grid = GridBase::createGrid(gd.gridType()); - if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); + grid = GridBase::createGrid(gd.gridType()); + } + grid->setSaveFloatAsHalf(gd.saveFloatAsHalf()); // Stream metadata varies per grid, and it needs to persist // in case delayed load is in effect. @@ -940,14 +962,24 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti grid->readTransform(is); if (readOptions.readMode != io::ReadMode::TopologyOnly && !gd.isInstance()) { - grid->readTopology(is); - const auto& worldBBox = readOptions.clipBBox; - const bool clip = worldBBox.isSorted(); - if (clip) { - const auto indexBBox = grid->constTransform().worldToIndexNodeCentered(worldBBox); - grid->readBuffers(is, indexBBox); + // read topology + if (codec) { + codec->readTopology(is, *codecData, readOptions); + } else { + grid->readTopology(is); + } + // read buffers + if (codec) { + codec->readBuffers(is, *codecData, readOptions); } else { - grid->readBuffers(is); + const auto& worldBBox = readOptions.clipBBox; + const bool clip = worldBBox.isSorted(); + if (clip) { + const auto indexBBox = grid->constTransform().worldToIndexNodeCentered(worldBBox); + grid->readBuffers(is, indexBBox); + } else { + grid->readBuffers(is); + } } } @@ -1065,7 +1097,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable, void Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, - std::ostream& os, bool seekable, const io::WriteOptions&) const + std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const { // Restore file-level stream metadata on exit. struct OnExit { @@ -1076,6 +1108,9 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, }; OnExit restore(os); + // Find the codec for the grid type and options. + io::Codec* codec = findCodec(gd.gridType()); + // Stream metadata varies per grid, so make a copy of the file-level stream metadata. io::StreamMetadata::Ptr streamMetadata; if (io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(os)) { @@ -1119,13 +1154,21 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid, grid->writeTransform(os); // Save the grid's structure. - grid->writeTopology(os); + if (codec) { + codec->writeTopology(os, *grid, writeOptions); + } else { + grid->writeTopology(os); + } // Now we know the grid block storage position. if (seekable) gd.setBlockPos(os.tellp()); // Save out the data blocks of the grid. - grid->writeBuffers(os); + if (codec) { + codec->writeBuffers(os, *grid, writeOptions); + } else { + grid->writeBuffers(os); + } // Now we know the end position of this grid. if (seekable) gd.setEndPos(os.tellp()); diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index a3b0893ea6..dad4cb8816 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -131,6 +131,9 @@ class OPENVDB_API Archive /// Read in and return the number of grids on the input stream. static int32_t readGridCount(std::istream&); + /// @brief Find the codec for the given grid type and options. + static io::Codec* findCodec(const std::string& gridType, const io::ReadOptions& options = io::ReadOptions{}); + /// @brief Read in and create the grid represented by the given grid descriptor using the /// given input stream, using the provided options if given. static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc new file mode 100644 index 0000000000..6c4661bdae --- /dev/null +++ b/openvdb/openvdb/io/Codec.cc @@ -0,0 +1,106 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#include "Codec.h" + +#include + +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace io { + +namespace { + +using CodecFactoryMap = std::map; +using CodecFactoryMapCIter = CodecFactoryMap::const_iterator; + +struct LockedCodecRegistry { + LockedCodecRegistry() {} + ~LockedCodecRegistry() {} + std::mutex mMutex; + CodecFactoryMap mMap; +}; + +// Global function for accessing the registry +static LockedCodecRegistry* +getCodecRegistry() +{ + static LockedCodecRegistry registry; + return ®istry; +} + +} // unnamed namespace + + +namespace internal { + +void initialize() +{ + CodecRegistry::registerCodec>(); +} + +void uninitialize() +{ + CodecRegistry::clear(); +} + +} // namespace internal + + +//////////////////////////////////////// + + +bool +CodecRegistry::isRegistered(const std::string& name) +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + return (registry->mMap.find(name) != registry->mMap.end()); +} + + +void +CodecRegistry::registerCodecByName(const std::string& name, Codec::Ptr&& codec) +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + if (registry->mMap.find(name) != registry->mMap.end()) { + OPENVDB_THROW(KeyError, + "Cannot register codec " << name << ". Codec is already registered"); + } + + registry->mMap[name] = std::move(codec); +} + + +Codec* +CodecRegistry::get(const std::string& name) +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + CodecFactoryMapCIter iter = registry->mMap.find(name); + + return (iter != registry->mMap.end()) ? iter->second.get() : nullptr; +} + + +void +CodecRegistry::clear() +{ + LockedCodecRegistry* registry = getCodecRegistry(); + std::lock_guard lock(registry->mMutex); + + registry->mMap.clear(); +} + + +} // namespace io +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index bd257056a1..4923ee7f83 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -4,21 +4,53 @@ #ifndef OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED #define OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED +#include +#include #include +#include -#include +#include namespace openvdb { OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { + +// Forward declaration +class GridBase; + namespace io { +namespace internal { + +OPENVDB_API void initialize(); +OPENVDB_API void uninitialize(); + +} // namespace internal + +//////////////////////////////////////// enum class ReadMode { Original, // Read data as-is (default) TopologyOnly // Read topology only, no buffer data }; +/// Base class for per-grid-type codec-specific options +/// Derive from this to add codec-specific read configuration +struct OPENVDB_API ReadTypedOptions +{ + using Ptr = std::shared_ptr; + + virtual ~ReadTypedOptions() = default; + + /// Type-safe casting helper for derived ReadTypedOptions types + /// Uses OPENVDB_ASSERT to verify type safety when assertions are enabled + template + static T& cast(const Ptr& data) { + OPENVDB_ASSERT(dynamic_cast(data.get()) != nullptr); + return *static_cast(data.get()); + } +}; // struct ReadTypedOptions + /// Global read configuration that applies to all grids /// Contains common options shared across all codec types struct OPENVDB_API ReadOptions @@ -28,6 +60,9 @@ struct OPENVDB_API ReadOptions // Grid Conversion Options ReadMode readMode = ReadMode::Original; + + // Per-grid-type codec-specific options, keyed by grid type string + std::unordered_map typeData; }; // struct ReadOptions /// Global write configuration that applies to all grids @@ -36,6 +71,61 @@ struct OPENVDB_API WriteOptions { }; // struct WriteOptions +/// Per-codec mutable data created by Codec::createData() +/// Contains the grid and any codec-specific state +struct OPENVDB_API CodecData +{ + using Ptr = std::unique_ptr; + + virtual ~CodecData() = default; + + SharedPtr grid; +}; // struct CodecData + +struct OPENVDB_API Codec +{ + using Ptr = std::unique_ptr; + + virtual ~Codec() = default; + + /// Create codec data including a new grid of the appropriate type + /// @return A new CodecData object containing the grid and any codec-specific state + virtual CodecData::Ptr createData() = 0; + + /// Read the grid topology + virtual void readTopology(std::istream&, CodecData&, const ReadOptions&) { } + + /// Read all data buffers for this grid + virtual void readBuffers(std::istream&, CodecData&, const ReadOptions&) { } + + /// Write the grid topology + virtual void writeTopology(std::ostream&, const GridBase&, const WriteOptions&) { } + + /// Write all data buffers for this grid + virtual void writeBuffers(std::ostream&, const GridBase&, const WriteOptions&) { } +}; // struct Codec + +struct OPENVDB_API CodecRegistry +{ + /// Return @c true if the given codec name is registered. + static bool isRegistered(const std::string& name); + + /// Register a codec by name with a factory function. + static void registerCodecByName(const std::string& name, Codec::Ptr&& codec); + + /// Register a codec by type. + template + static void registerCodec() + { + registerCodecByName(CodecT::name(), std::make_unique()); + } + + /// Return a pointer to the codec with the given name, or nullptr if not found. + static Codec* get(const std::string& name); + + /// Clear the codec registry. + static void clear(); +}; // struct CodecRegistry } // namespace io } // namespace OPENVDB_VERSION_NAME diff --git a/openvdb/openvdb/openvdb.cc b/openvdb/openvdb/openvdb.cc index f830054849..ff163071de 100644 --- a/openvdb/openvdb/openvdb.cc +++ b/openvdb/openvdb/openvdb.cc @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "openvdb.h" +#include "io/Codec.h" #include "points/PointDataGrid.h" #include "tools/PointIndexGrid.h" #include "util/logging.h" @@ -70,6 +71,8 @@ initialize() logging::initialize(); + io::internal::initialize(); + // Register metadata. Metadata::clearRegistry(); MetaTypes::foreach(); @@ -130,6 +133,7 @@ __pragma(warning(default:1711)) Metadata::clearRegistry(); GridBase::clearRegistry(); math::MapRegistry::clear(); + io::internal::uninitialize(); points::internal::uninitialize(); #ifdef OPENVDB_USE_BLOSC diff --git a/openvdb/openvdb/unittest/CMakeLists.txt b/openvdb/openvdb/unittest/CMakeLists.txt index 1a0168b6c7..18046cd8e0 100644 --- a/openvdb/openvdb/unittest/CMakeLists.txt +++ b/openvdb/openvdb/unittest/CMakeLists.txt @@ -82,6 +82,7 @@ else() TestAttributeSet.cc TestBBox.cc TestClip.cc + TestCodec.cc TestConjGradient.cc TestCoord.cc TestCount.cc diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc new file mode 100644 index 0000000000..65996a3a56 --- /dev/null +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -0,0 +1,185 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include + +class TestCodec: public ::testing::Test +{ +public: + void SetUp() override { openvdb::initialize(); } + void TearDown() override { openvdb::uninitialize(); } +}; + +struct MockCodec : public openvdb::io::Codec +{ + static std::string name() { return "mock"; } + + openvdb::io::CodecData::Ptr createData() final { return nullptr; } +}; + +TEST_F(TestCodec, testCodecRegistry) +{ + using namespace openvdb::io; + + // Start clean + CodecRegistry::clear(); + + // Test isRegistered on empty registry + EXPECT_FALSE(CodecRegistry::isRegistered("mock")); + + // Test registerCodecByName + EXPECT_NO_THROW( + CodecRegistry::registerCodecByName("mock", std::make_unique()) + ); + + EXPECT_TRUE(CodecRegistry::isRegistered("mock")); + EXPECT_FALSE(CodecRegistry::isRegistered("nonexistent")); + + // Test duplicate registration throws KeyError + EXPECT_THROW( + CodecRegistry::registerCodecByName("mock", std::make_unique()), + openvdb::KeyError + ); + + // Test registerCodec template form also throws on duplicate + EXPECT_THROW( + CodecRegistry::registerCodec(), + openvdb::KeyError + ); + + // Test get + EXPECT_NE(CodecRegistry::get("mock"), nullptr); + EXPECT_EQ(CodecRegistry::get("nonexistent"), nullptr); + + // Test clear + CodecRegistry::clear(); + EXPECT_FALSE(CodecRegistry::isRegistered("mock")); + EXPECT_NO_THROW(CodecRegistry::clear()); // Clear on empty registry + + // Test registerCodec template form on fresh registry + EXPECT_NO_THROW(CodecRegistry::registerCodec()); + EXPECT_TRUE(CodecRegistry::isRegistered("mock")); + + // Test io::initialize and io::uninitialize + CodecRegistry::clear(); + EXPECT_FALSE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); + + EXPECT_NO_THROW(internal::initialize()); + EXPECT_TRUE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); + + EXPECT_NO_THROW(internal::uninitialize()); + EXPECT_FALSE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); +} + + +template +void testIOImpl( + const std::string& gridName, + const typename GridT::ValueType& bgValue, + const typename GridT::ValueType& fillValue) +{ + using namespace openvdb; + using namespace openvdb::io; + + typename GridT::Ptr srcGrid = GridT::create(bgValue); + srcGrid->setName(gridName); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), fillValue, true); + + std::stringstream ss("test"); + if (CodecRegistry::isRegistered(GridT::gridType())) { + ss << "_codec"; + } else { + ss << "_tree"; + } + ss << "_" << GridT::gridType() << ".vdb"; + const std::string path = ss.str(); + { + io::File f(path); + f.write(GridPtrVec{srcGrid}); + } + + typename GridT::Ptr readGrid; + { + io::File f(path); + f.open(); + readGrid = gridPtrCast(f.readGrid(gridName)); + f.close(); + } + ASSERT_TRUE(readGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(readGrid->tree())); + { + auto readAcc = readGrid->getConstAccessor(); + for (typename GridT::ValueOnCIter it = srcGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, readAcc.getValue(it.getCoord())); + } + } + + // clip read + const BBoxd clipBBox(Vec3d(0.0), Vec3d(3.5)); + auto srcClipped = tools::clip(*srcGrid, clipBBox); + + typename GridT::Ptr readClipped; + { + io::File f(path); + f.open(); + readClipped = gridPtrCast(f.readGrid(gridName, clipBBox)); + f.close(); + } + ASSERT_TRUE(readClipped); + EXPECT_TRUE(srcClipped->tree().hasSameTopology(readClipped->tree())); + { + auto readAcc = readClipped->getConstAccessor(); + for (typename GridT::ValueOnCIter it = srcClipped->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, readAcc.getValue(it.getCoord())); + } + } + + // topology-only read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + typename GridT::Ptr readTopo; + { + io::File f(path); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid(gridName, topoOpts)); + readTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(readTopo); + EXPECT_EQ(readTopo->activeVoxelCount(), Index64(0)); + EXPECT_TRUE(readTopo->tree().leafCount() == 0); + EXPECT_EQ(readTopo->getName(), gridName); + + // Cleanup + std::remove(path.c_str()); +} + +template +void testCodecIOImpl( + const std::string& gridName, + const typename GridT::ValueType& bgValue, + const typename GridT::ValueType& fillValue) +{ + // initialize to register all the codecs + openvdb::io::CodecRegistry::clear(); + openvdb::io::internal::initialize(); + // ensure the codec is registered + ASSERT_TRUE(openvdb::io::CodecRegistry::isRegistered(GridT::gridType())); + // test the io implementation (codec) + testIOImpl(gridName, bgValue, fillValue); + // clear the codec registry (now read/write falls back to Tree I/O) + openvdb::io::CodecRegistry::clear(); + // ensure the codec is not registered + ASSERT_FALSE(openvdb::io::CodecRegistry::isRegistered(GridT::gridType())); + // test the io implementation (tree I/O) + testIOImpl(gridName, bgValue, fillValue); +} + +TEST_F(TestCodec, testBoolCodecIO) { testCodecIOImpl("bool_grid", false, true); } From fa84a408b2b97136a53dca50f3f9c64295c9ec63 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 31 Mar 2026 21:49:53 -0700 Subject: [PATCH 30/80] Add ReadDiagnostics to Codecs Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/BoolCodec.h | 2 +- openvdb/openvdb/codecs/TopologyCodec.h | 22 ++++++--- openvdb/openvdb/io/Archive.cc | 43 +++++++++++++++-- openvdb/openvdb/io/Archive.h | 17 +++++++ openvdb/openvdb/io/Codec.h | 49 ++++++++++++++++++- openvdb/openvdb/io/File.cc | 7 +-- openvdb/openvdb/unittest/TestCodec.cc | 65 ++++++++++++++++++++++++++ 7 files changed, 189 insertions(+), 16 deletions(-) diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h index 2aafb86aaa..68d7997a89 100644 --- a/openvdb/openvdb/codecs/BoolCodec.h +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -110,7 +110,7 @@ struct BoolCodec : public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options) final + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { GridT& grid = static_cast(*data.grid); diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 0f411598c0..7e2fc952b8 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -107,9 +107,12 @@ struct ReadTopologyOp using LeafT = typename TreeT::LeafNodeType; using StorageValueT = typename StorageTreeT::ValueType; - ReadTopologyOp(std::istream& _is, bool _saveFloatAsHalf) + ReadTopologyOp(std::istream& _is, bool _saveFloatAsHalf, io::ReadDiagnostics& _diagnostics, + const std::string& _gridName) : is(_is) - , saveFloatAsHalf(_saveFloatAsHalf) { } + , saveFloatAsHalf(_saveFloatAsHalf) + , diagnostics(_diagnostics) + , gridName(_gridName) { } void operator()(RootT& root) { @@ -117,7 +120,9 @@ struct ReadTopologyOp int32_t bufferCount; is.read(reinterpret_cast(&bufferCount), sizeof(int32_t)); - if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported"); + if (bufferCount != 1) { + diagnostics.addWarning(gridName, "multi-buffer trees are no longer supported"); + } // Delete the existing tree. root.clear(); @@ -216,6 +221,8 @@ struct ReadTopologyOp std::istream& is; bool saveFloatAsHalf; ValueT background; + io::ReadDiagnostics& diagnostics; + std::string gridName; }; // struct ReadTopologyOp template @@ -264,14 +271,14 @@ void setTilesToBackground(TreeT& tree) // Free-standing function for read case (supports type conversion via StorageGridT) template -void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options) +void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) { io::checkFormatVersion(is); GridT& grid = static_cast(gridBase); grid.tree().clearAllAccessors(); - internal::ReadTopologyOp readTopologyOp(is, grid.saveFloatAsHalf()); + internal::ReadTopologyOp readTopologyOp(is, grid.saveFloatAsHalf(), diagnostics, grid.getName()); readTopologyOp(grid.tree().root()); if (options.readMode == io::ReadMode::TopologyOnly) { @@ -307,9 +314,10 @@ struct TopologyCodec : public io::Codec return data; } - void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options) final + void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options, + io::ReadDiagnostics& diagnostics) final { - internal::topologyCodecReadTopology(*data.grid, is, options); + internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics); } void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 712d3d2b99..92e99f7c83 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -384,6 +384,35 @@ Archive::copy() const } +void +Archive::enableReadDiagnostics() +{ + mReadDiagnostics.enable(); +} + + +void +Archive::disableReadDiagnostics() +{ + mReadDiagnostics.disable(); + mReadDiagnostics.clear(); +} + + +const ReadDiagnostics& +Archive::readDiagnostics() const +{ + return mReadDiagnostics; +} + + +void +Archive::clearReadDiagnostics() +{ + mReadDiagnostics.clear(); +} + + //////////////////////////////////////// @@ -898,7 +927,7 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co GridBase::Ptr -Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions) +Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions, ReadDiagnostics& diagnostics) { // Read the compression settings for this grid and tag the stream with them // so that downstream functions can reference them. @@ -964,13 +993,13 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti if (readOptions.readMode != io::ReadMode::TopologyOnly && !gd.isInstance()) { // read topology if (codec) { - codec->readTopology(is, *codecData, readOptions); + codec->readTopology(is, *codecData, readOptions, diagnostics); } else { grid->readTopology(is); } // read buffers if (codec) { - codec->readBuffers(is, *codecData, readOptions); + codec->readBuffers(is, *codecData, readOptions, diagnostics); } else { const auto& worldBBox = readOptions.clipBBox; const bool clip = worldBBox.isSorted(); @@ -987,6 +1016,14 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti } +GridBase::Ptr +Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions) +{ + ReadDiagnostics nullDiagnostics; + return readGrid(gd, is, readOptions, nullDiagnostics); +} + + //////////////////////////////////////// diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index dad4cb8816..e0414f166a 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -98,6 +98,18 @@ class OPENVDB_API Archive /// @brief Return @c false (delayed loading has been removed). static bool isDelayedLoadingEnabled() { return false; } + /// @brief Enable collection of read diagnostics (warnings, etc.) during I/O operations. + void enableReadDiagnostics(); + + /// @brief Disable collection of read diagnostics. + void disableReadDiagnostics(); + + /// @brief Return a const reference to the diagnostics collector. + const ReadDiagnostics& readDiagnostics() const; + + /// @brief Clear the diagnostics list while keeping collection active. + void clearReadDiagnostics(); + protected: /// @brief Return @c true if the input stream contains grid offsets /// that allow for random access or partial reading. @@ -136,6 +148,8 @@ class OPENVDB_API Archive /// @brief Read in and create the grid represented by the given grid descriptor using the /// given input stream, using the provided options if given. + static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, + const io::ReadOptions& readOptions, ReadDiagnostics& diagnostics); static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&, const io::ReadOptions& readOptions = io::ReadOptions{}); @@ -173,6 +187,9 @@ class OPENVDB_API Archive const io::WriteOptions& writeOptions = io::WriteOptions{}) const; //@} + /// Diagnostics collector for read operations (always valid; enabled/disabled via flag) + mutable ReadDiagnostics mReadDiagnostics; + private: friend class ::TestFile; diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 4923ee7f83..304a5b7976 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include @@ -71,6 +73,49 @@ struct OPENVDB_API WriteOptions { }; // struct WriteOptions +enum class DiagnosticSeverity { Warning }; + +struct ReadDiagnostic { + DiagnosticSeverity severity; + std::string context; + std::string message; +}; + +struct OPENVDB_API ReadDiagnostics { + ReadDiagnostics() = default; + ReadDiagnostics(const ReadDiagnostics& other) { + std::lock_guard lock(other.mMutex); + mEnabled = other.mEnabled; + mDiagnostics = other.mDiagnostics; + } + ReadDiagnostics& operator=(const ReadDiagnostics& other) { + if (this != &other) { + std::scoped_lock lock(mMutex, other.mMutex); + mEnabled = other.mEnabled; + mDiagnostics = other.mDiagnostics; + } + return *this; + } + void addWarning(const std::string& context, const std::string& message) { + if (!mEnabled) return; + std::lock_guard lock(mMutex); + mDiagnostics.push_back({DiagnosticSeverity::Warning, context, message}); + } + void clear() { + std::lock_guard lock(mMutex); + mDiagnostics.clear(); + } + void enable() { mEnabled = true; } + void disable() { mEnabled = false; } + bool enabled() const { return mEnabled; } + const std::vector& diagnostics() const { return mDiagnostics; } + bool empty() const { return mDiagnostics.empty(); } +private: + bool mEnabled = false; + std::vector mDiagnostics; + mutable std::mutex mMutex; +}; // struct ReadDiagnostics + /// Per-codec mutable data created by Codec::createData() /// Contains the grid and any codec-specific state struct OPENVDB_API CodecData @@ -93,10 +138,10 @@ struct OPENVDB_API Codec virtual CodecData::Ptr createData() = 0; /// Read the grid topology - virtual void readTopology(std::istream&, CodecData&, const ReadOptions&) { } + virtual void readTopology(std::istream&, CodecData&, const ReadOptions&, ReadDiagnostics&) { } /// Read all data buffers for this grid - virtual void readBuffers(std::istream&, CodecData&, const ReadOptions&) { } + virtual void readBuffers(std::istream&, CodecData&, const ReadOptions&, ReadDiagnostics&) { } /// Write the grid topology virtual void writeTopology(std::ostream&, const GridBase&, const WriteOptions&) { } diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index bd4ebdd4f5..d1e3856d9d 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -319,7 +319,7 @@ File::getGrids(const io::ReadOptions& readOptions) const const GridDescriptor& gd = i->second; // Seek to the grid in the file. gd.seekToGrid(inputStream()); - GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), readOptions); + GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), readOptions, mReadDiagnostics); ret->push_back(grid); namedGrids[gd.uniqueName()] = grid; } @@ -477,7 +477,7 @@ File::readGrid(const Name& name, const io::ReadOptions& readOptions) OPENVDB_ASSERT(inputHasGridOffsets()); // Seek to the grid in the file. gd.seekToGrid(inputStream()); - grid = Archive::readGrid(gd, inputStream(), readOptions); + grid = Archive::readGrid(gd, inputStream(), readOptions, mReadDiagnostics); if (gd.isInstance()) { /// @todo Refactor to share code with Archive::connectInstance()? @@ -493,7 +493,7 @@ File::readGrid(const Name& name, const io::ReadOptions& readOptions) GridBase::Ptr parent; OPENVDB_ASSERT(inputHasGridOffsets()); parentIt->second.seekToGrid(inputStream()); - parent = Archive::readGrid(parentIt->second, inputStream(), readOptions); + parent = Archive::readGrid(parentIt->second, inputStream(), readOptions, mReadDiagnostics); if (parent) grid->setTree(parent->baseTreePtr()); } return grid; @@ -592,6 +592,7 @@ File::endName() const return File::NameIterator(mGridDescriptors.end()); } + } // namespace io } // namespace OPENVDB_VERSION_NAME } // namespace openvdb diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index 65996a3a56..384c52eaa2 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -77,6 +77,71 @@ TEST_F(TestCodec, testCodecRegistry) } +TEST_F(TestCodec, testReadDiagnostics) +{ + using namespace openvdb; + using namespace openvdb::io; + + // ReadDiagnostics struct: disabled by default, addWarning is a no-op until enabled + { + ReadDiagnostics diags; + EXPECT_FALSE(diags.enabled()); + diags.addWarning("grid_a", "something went wrong"); + EXPECT_TRUE(diags.diagnostics().empty()); + + diags.enable(); + diags.addWarning("grid_a", "something went wrong"); + ASSERT_EQ(diags.diagnostics().size(), size_t(1)); + EXPECT_EQ(diags.diagnostics()[0].severity, DiagnosticSeverity::Warning); + + diags.clear(); + EXPECT_TRUE(diags.diagnostics().empty()); + } + + CodecRegistry::clear(); + openvdb::io::internal::initialize(); + + // Archive API and getGrids() with diagnostics + + BoolGrid::Ptr srcGrid = BoolGrid::create(false); + srcGrid->setName("bool_grid"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), true, true); + + const std::string codecPath = "testReadDiagnostics.vdb"; + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + // Disabled by default; enabling produces no warnings on a clean read + { + io::File f(codecPath); + f.open(); + EXPECT_FALSE(f.readDiagnostics().enabled()); + f.enableReadDiagnostics(); + EXPECT_TRUE(f.readDiagnostics().enabled()); + f.readGrid("bool_grid"); + EXPECT_TRUE(f.readDiagnostics().diagnostics().empty()); + f.close(); + } + + // clearReadDiagnostics() resets entries but keeps diagnostics enabled + { + io::File f(codecPath); + f.open(); + f.enableReadDiagnostics(); + GridPtrVecPtr grids = f.getGrids(); + ASSERT_TRUE(grids && !grids->empty()); + f.clearReadDiagnostics(); + EXPECT_TRUE(f.readDiagnostics().enabled()); + EXPECT_TRUE(f.readDiagnostics().diagnostics().empty()); + f.close(); + } + + std::remove(codecPath.c_str()); +} + + template void testIOImpl( const std::string& gridName, From 906eef63afaf2f0c41c451a3f6ed4a562b909615 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 23 Apr 2026 11:28:15 -0700 Subject: [PATCH 31/80] Add Scalar and ValueMask codecs Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 2 + openvdb/openvdb/codecs/ScalarCodec.h | 185 +++++++++++++++++++++++ openvdb/openvdb/codecs/ScalarLeafCodec.h | 104 +++++++++++++ openvdb/openvdb/codecs/TopologyCodec.h | 7 +- openvdb/openvdb/codecs/ValueMaskCodec.h | 146 ++++++++++++++++++ openvdb/openvdb/io/Codec.cc | 9 ++ openvdb/openvdb/io/Codec.h | 5 + openvdb/openvdb/unittest/TestCodec.cc | 9 ++ openvdb/openvdb/unittest/TestGridIO.cc | 8 +- openvdb/openvdb/unittest/TestTree.cc | 5 + 10 files changed, 477 insertions(+), 3 deletions(-) create mode 100644 openvdb/openvdb/codecs/ScalarCodec.h create mode 100644 openvdb/openvdb/codecs/ScalarLeafCodec.h create mode 100644 openvdb/openvdb/codecs/ValueMaskCodec.h diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index f0fd0fed5a..9d8d819d49 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -367,7 +367,9 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES codecs/BoolCodec.h + codecs/ScalarCodec.h codecs/TopologyCodec.h + codecs/ValueMaskCodec.h ) set(OPENVDB_LIBRARY_IO_INCLUDE_FILES diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h new file mode 100644 index 0000000000..c9f8dbe49e --- /dev/null +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -0,0 +1,185 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED + +#include +#include +#include +#include +#include + +#include "ScalarLeafCodec.h" +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct WriteBuffersOp +{ + using LeafT = typename TreeT::LeafNodeType; + + WriteBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + writeScalarLeafBuffers(leaf, os, saveFloatAsHalf); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WriteBuffersOp + + +template +struct ReadBuffersOp +{ + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + using StorageLeafT = typename StorageTreeT::LeafNodeType; + + ReadBuffersOp(std::istream& _is, bool _saveFloatAsHalf, const ValueT& _background, + const CoordBBox* _clipBBox) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) + , clipBBox(_clipBBox) { } + + void operator()(RootT& root, size_t) + { + if (clipBBox) { + root.clip(*clipBBox); + } + } + + template + void operator()(NodeT& node, size_t) + { + if (clipBBox) { + node.clip(*clipBBox, background); + } + } + + void operator()(LeafT& leaf, size_t) + { + readScalarLeafBuffers(leaf, is, saveFloatAsHalf, background, /*skip=*/false, clipBBox); + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; + const CoordBBox* clipBBox = nullptr; +}; // struct ReadBuffersOp + + +// Free-standing function for both standard and conversion codec cases +// Uses StorageGridT = GridT by default, but allows different storage type for conversions +template +void scalarCodecReadBuffers(GridT& grid, std::istream& is, const io::ReadOptions& options) +{ + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec"); + } + + using TreeT = typename GridT::TreeType; + using StorageTreeT = typename StorageGridT::TreeType; + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + std::unique_ptr clipIndexBBox; + if (options.clipBBox.isSorted()) { + clipIndexBBox = std::make_unique(grid.constTransform().worldToIndexNodeCentered(options.clipBBox)); + } + + // Works for both standard (TreeT == StorageTreeT) and conversion cases + ReadBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), + clipIndexBBox.get()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); +} + +// Free-standing function for write case (no StorageGridT needed) +template +void scalarCodecWriteBuffers(const GridT& grid, std::ostream& os) +{ + using TreeType = typename GridT::TreeType; + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec"); + } + + WriteBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); +} + +} // namespace internal + +template +struct ScalarCodec : public TopologyCodec +{ + static_assert(GridT::TreeType::RootNodeType::template SameConfiguration< + typename StorageGridT::TreeType::RootNodeType>::value, + "GridT and StorageGridT must have the same configuration"); + + using Ptr = std::unique_ptr>; + + struct ScalarCodecData : public io::CodecData + { + std::vector leafCoords; + }; + + ~ScalarCodec() noexcept = default; + + io::CodecData::Ptr createData() final + { + auto data = std::make_unique(); + data->grid = GridT::create(); + return data; + } + + static inline std::string name() + { + if constexpr (std::is_same_v) { + return GridT::gridType(); + } else { + std::string buildType = typeNameAsString(); + return StorageGridT::gridType() + "_to_" + buildType; + } + } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + { + GridT& grid = static_cast(*data.grid); + internal::scalarCodecReadBuffers(grid, is, options); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + if constexpr (Mode == io::CodecMode::ReadOnly) return; + + const GridT& grid = static_cast(gridBase); + internal::scalarCodecWriteBuffers(grid, os); + } +}; // struct ScalarCodec + + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/ScalarLeafCodec.h b/openvdb/openvdb/codecs/ScalarLeafCodec.h new file mode 100644 index 0000000000..f72efc56d9 --- /dev/null +++ b/openvdb/openvdb/codecs/ScalarLeafCodec.h @@ -0,0 +1,104 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED + +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +void writeScalarLeafBuffers(const LeafT& leaf, std::ostream& os, bool saveFloatAsHalf) +{ + using NodeMaskT = typename LeafT::NodeMaskType; + + // Write out the value mask. + leaf.getValueMask().save(os); + + leaf.buffer().data(); // load values + + io::writeCompressedValues(os, leaf.buffer().data(), LeafT::SIZE, + leaf.getValueMask(), /*childMask=*/NodeMaskT(), saveFloatAsHalf); +} + + +template +void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, + const typename LeafT::ValueType& background, bool skip = false, + const math::CoordBBox* clipBBox = nullptr) +{ + using ValueT = typename LeafT::ValueType; + using NodeMaskT = typename LeafT::NodeMaskType; + using StorageBufferT = typename StorageLeafT::Buffer; + using StorageValueT = typename StorageLeafT::ValueType; + + constexpr Index SIZE = LeafT::SIZE; + + SharedPtr meta = io::getStreamMetadataPtr(is); + const bool seekable = meta && meta->seekable(); + + auto& valueMask = leaf.getValueMask(); + + // Load or seek the value mask + if (seekable) valueMask.seek(is); + else valueMask.load(is); + + if (skip) { + if (seekable) { + io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + } + // Clear the value mask so that the skipped leaf has no active + // voxels. Without this, the leaf retains its on-disk active + // topology even though no data was read into its buffer. + valueMask.setOff(); + return; + } + + if constexpr (std::is_same_v) { + // ValueMask leaf: value == active state, already captured in the value mask above. + // Seek/consume past the storage buffer without populating any separate leaf buffer. + if (seekable) { + io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + } + } else if constexpr (std::is_same_v) { + // Bool leaf: must read storage values regardless of seekability, then convert. + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + for (Index i = 0; i < SIZE; ++i) { + leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); + } + } else { + leaf.buffer().allocate(); + if constexpr (std::is_same_v) { + io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + for (Index i = 0; i < SIZE; ++i) { + leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); + } + } + } + + if (clipBBox) { + leaf.clip(*clipBBox, background); + } +} + +} // namespace internal +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 7e2fc952b8..c99b39277b 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -300,10 +300,10 @@ void topologyCodecWriteTopology(const GridBase& gridBase, std::ostream& os) } // namespace internal -template +template struct TopologyCodec : public io::Codec { - using Ptr = std::unique_ptr>; + using Ptr = std::unique_ptr>; ~TopologyCodec() noexcept = default; @@ -322,6 +322,9 @@ struct TopologyCodec : public io::Codec void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final { + // disable implementation when read only + if constexpr (Mode == io::CodecMode::ReadOnly) return; + internal::topologyCodecWriteTopology(gridBase, os); } }; // struct TopologyCodec diff --git a/openvdb/openvdb/codecs/ValueMaskCodec.h b/openvdb/openvdb/codecs/ValueMaskCodec.h new file mode 100644 index 0000000000..17171cebf7 --- /dev/null +++ b/openvdb/openvdb/codecs/ValueMaskCodec.h @@ -0,0 +1,146 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED + +#include + +#include +#include + +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct ReadValueMaskBuffersOp +{ + using TreeT = typename GridT::TreeType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + ReadValueMaskBuffersOp(std::istream& _is, bool _saveFloatAsHalf, + const ValueT& _background, const CoordBBox* _clipBBox) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) + , clipBBox(_clipBBox) { } + + void operator()(RootT& root, size_t) + { + // Clip root-level tiles and prune children that were clipped. + if (clipBBox) { + root.clip(*clipBBox); + } + } + + template + void operator()(NodeT& node, size_t) + { + // Clip internal node tiles and prune children that were clipped. + if (clipBBox) { + node.clip(*clipBBox, background); + } + } + + void operator()(LeafT& leaf, size_t) + { + // Read in the value mask. + leaf.getValueMask().load(is); + // Read in the origin. + Coord origin; + is.read(reinterpret_cast(&origin), sizeof(Coord::ValueType) * 3); + leaf.setOrigin(origin); + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; + const CoordBBox* clipBBox = nullptr; +}; // struct ReadValueMaskBuffersOp + +template +struct WriteValueMaskBuffersOp +{ + using TreeT = typename GridT::TreeType; + using LeafT = typename TreeT::LeafNodeType; + + WriteValueMaskBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + // Write out the value mask. + leaf.getValueMask().save(os); + + // Write out the origin. + os.write(reinterpret_cast(&leaf.origin()), sizeof(Coord::ValueType) * 3); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WriteValueMaskBuffersOp + +} // namespace internal + +template +struct ValueMaskCodec : public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~ValueMaskCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + { + GridT& grid = static_cast(*data.grid); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ValueMaskCodec"); + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + std::unique_ptr clipIndexBBox; + if (options.clipBBox.isSorted()) { + clipIndexBBox = std::make_unique(grid.constTransform().worldToIndexNodeCentered(options.clipBBox)); + } + + internal::ReadValueMaskBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), clipIndexBBox.get()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + const GridT& grid = static_cast(gridBase); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ValueMaskCodec"); + } + + internal::WriteValueMaskBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); + } +}; // struct ValueMaskCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index 6c4661bdae..f123f0ec0a 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -3,7 +3,9 @@ #include "Codec.h" +#include #include +#include #include @@ -37,9 +39,16 @@ getCodecRegistry() namespace internal { +template +struct RegisterCodec { inline void operator()() { CodecRegistry::registerCodec>(); } }; + void initialize() { + NumericGridTypes::foreach(); + Vec3GridTypes::foreach(); + CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); } void uninitialize() diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 304a5b7976..8f41b7d017 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -31,6 +31,11 @@ OPENVDB_API void uninitialize(); //////////////////////////////////////// +enum class CodecMode { + ReadWrite, // Both read and write enabled (default) + ReadOnly // Only read methods enabled +}; + enum class ReadMode { Original, // Read data as-is (default) TopologyOnly // Read topology only, no buffer data diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index 384c52eaa2..da68e4b321 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -247,4 +247,13 @@ void testCodecIOImpl( testIOImpl(gridName, bgValue, fillValue); } +TEST_F(TestCodec, testFloatCodecIO) { testCodecIOImpl("float_grid", 0.0f, 1.0f); } +TEST_F(TestCodec, testDoubleCodecIO) { testCodecIOImpl("double_grid", 0.0, 1.0); } +TEST_F(TestCodec, testInt32CodecIO) { testCodecIOImpl("int32_grid", 0, 1); } +TEST_F(TestCodec, testInt64CodecIO) { testCodecIOImpl("int64_grid", openvdb::Int64(0), openvdb::Int64(1)); } +TEST_F(TestCodec, testHalfCodecIO) { testCodecIOImpl("half_grid", openvdb::Half(0.0), openvdb::Half(1.5)); } +TEST_F(TestCodec, testVec3ICodecIO) { testCodecIOImpl("vec3i_grid", openvdb::Vec3i(0), openvdb::Vec3i(1, 2, 3)); } +TEST_F(TestCodec, testVec3SCodecIO) { testCodecIOImpl("vec3s_grid", openvdb::Vec3s(0.0f), openvdb::Vec3s(1.0f, 2.0f, 3.0f)); } +TEST_F(TestCodec, testVec3DCodecIO) { testCodecIOImpl("vec3d_grid", openvdb::Vec3d(0.0), openvdb::Vec3d(1.0, 2.0, 3.0)); } TEST_F(TestCodec, testBoolCodecIO) { testCodecIOImpl("bool_grid", false, true); } +TEST_F(TestCodec, testMaskCodecIO) { testCodecIOImpl("mask_grid", false, true); } diff --git a/openvdb/openvdb/unittest/TestGridIO.cc b/openvdb/openvdb/unittest/TestGridIO.cc index 184379a030..3f605b86dd 100644 --- a/openvdb/openvdb/unittest/TestGridIO.cc +++ b/openvdb/openvdb/unittest/TestGridIO.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include // for remove() @@ -278,5 +279,10 @@ TEST_F(TestGridIO, testReadAllBool) { readAllTest(); } TEST_F(TestGridIO, testReadAllFloat) { readAllTest(); } TEST_F(TestGridIO, testReadAllHalf) { readAllTest(); } TEST_F(TestGridIO, testReadAllVec3S) { readAllTest(); } -TEST_F(TestGridIO, testReadAllFloat5432) { Float5432Grid::registerGrid(); readAllTest(); } +TEST_F(TestGridIO, testReadAllFloat5432) +{ + Float5432Grid::registerGrid(); + openvdb::io::CodecRegistry::registerCodec>(); + readAllTest(); +} TEST_F(TestGridIO, testCreateWriteReadHalf) { testCreateWriteReadHalf(); } \ No newline at end of file diff --git a/openvdb/openvdb/unittest/TestTree.cc b/openvdb/openvdb/unittest/TestTree.cc index 90a3de5e45..624906b199 100644 --- a/openvdb/openvdb/unittest/TestTree.cc +++ b/openvdb/openvdb/unittest/TestTree.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include // for tools::setValueOnMin(), et al. #include @@ -109,6 +110,10 @@ TEST_F(TestTree, testHalf) openvdb::Grid::registerGrid(); openvdb::Grid::registerGrid(); + // Register scalar codecs for all vec2 tree types which are not registered by default + openvdb::io::CodecRegistry::registerCodec>>(); + openvdb::io::CodecRegistry::registerCodec>>(); + testWriteHalf(); testWriteHalf(); testWriteHalf(); From 42fbdf741c2864008316a366bd44d76e511cb003 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 20 Apr 2026 21:44:28 -0700 Subject: [PATCH 32/80] Add support for implicit conversion - float to half and scalar to bool/mask Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Archive.cc | 18 +++- openvdb/openvdb/io/Codec.cc | 15 +++ openvdb/openvdb/io/Codec.h | 3 + openvdb/openvdb/unittest/TestCodec.cc | 146 ++++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 2 deletions(-) diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 92e99f7c83..d11e7a1688 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -885,10 +885,24 @@ Archive::readGridCount(std::istream& is) io::Codec* -Archive::findCodec(const std::string& gridType, const io::ReadOptions&) +Archive::findCodec(const std::string& gridType, const io::ReadOptions& options) { // Determine the I/O codec to use to read this grid - return io::CodecRegistry::get(gridType); + auto codec = io::CodecRegistry::get(gridType); + + // if readMode is Half, then search for a codec that converts + // from the storage grid type to the grid type + if (options.readMode == ReadMode::Half) { + codec = io::CodecRegistry::get(gridType + "_to_half"); + } + if (options.readMode == ReadMode::Bool) { + codec = io::CodecRegistry::get(gridType + "_to_bool"); + } + if (options.readMode == ReadMode::Mask) { + codec = io::CodecRegistry::get(gridType + "_to_mask"); + } + + return codec; } diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index f123f0ec0a..cfb1489efc 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -42,6 +42,15 @@ namespace internal { template struct RegisterCodec { inline void operator()() { CodecRegistry::registerCodec>(); } }; +template +struct RegisterConvertCodec { + inline void operator()() + { + CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); + } +}; // struct RegisterConvertCodec + void initialize() { NumericGridTypes::foreach(); @@ -49,6 +58,12 @@ void initialize() CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); + + // register the plugin that converts from scalar to mask/bool + NumericGridTypes::foreach(); + + // register the plugin that converts from float to half + CodecRegistry::registerCodec>(); } void uninitialize() diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 8f41b7d017..28b00c4a91 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -38,6 +38,9 @@ enum class CodecMode { enum class ReadMode { Original, // Read data as-is (default) + Half, // Read data as half precision + Bool, // Read data as boolean + Mask, // Read data as mask TopologyOnly // Read topology only, no buffer data }; diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index da68e4b321..d8981ca3f7 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -257,3 +258,148 @@ TEST_F(TestCodec, testVec3SCodecIO) { testCodecIOImpl("vec3s TEST_F(TestCodec, testVec3DCodecIO) { testCodecIOImpl("vec3d_grid", openvdb::Vec3d(0.0), openvdb::Vec3d(1.0, 2.0, 3.0)); } TEST_F(TestCodec, testBoolCodecIO) { testCodecIOImpl("bool_grid", false, true); } TEST_F(TestCodec, testMaskCodecIO) { testCodecIOImpl("mask_grid", false, true); } + +TEST_F(TestCodec, testFloatToHalfCodecConversion) +{ + using namespace openvdb; + using namespace openvdb::io; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Verify the conversion codec name + const std::string expectedName = FloatGrid::gridType() + "_to_half"; + EXPECT_EQ((codecs::ScalarCodec::name()), + expectedName); + + // Verify the codec is registered after initialize() + io::internal::initialize(); + EXPECT_TRUE(CodecRegistry::isRegistered(expectedName)); + + // Write a FloatGrid with a known fill value (1.5f is exactly representable in half) + const std::string floatPath = "test_float_to_half.vdb"; + const std::string gridName = "float_grid"; + FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f); + srcGrid->setName(gridName); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + { + io::File f(floatPath); + f.write(GridPtrVec{srcGrid}); + } + + // Read back with ReadMode::Half — triggers the float-to-half conversion codec + ReadOptions halfOpts; + halfOpts.readMode = ReadMode::Half; + + GridBase::Ptr base; + { + io::File f(floatPath); + f.open(); + base = f.readGrid(gridName, halfOpts); + f.close(); + } + ASSERT_TRUE(base); + + // The returned grid must be a HalfGrid + EXPECT_TRUE(base->isType()); + HalfGrid::Ptr halfGrid = gridPtrCast(base); + ASSERT_TRUE(halfGrid); + + // Topology must match the source FloatGrid + EXPECT_TRUE(srcGrid->tree().hasSameTopology(halfGrid->tree())); + + // All active voxel values must equal Half(1.5f) + for (HalfGrid::ValueOnCIter it = halfGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, Half(1.5f)); + } + + // Background value must equal Half(0.0f) + EXPECT_EQ(halfGrid->background(), Half(0.0f)); + + // Cleanup + std::remove(floatPath.c_str()); +} + +template +void testConvertCodecImpl() +{ + using namespace openvdb; + using namespace openvdb::io; + + openvdb::io::CodecRegistry::clear(); + openvdb::io::internal::initialize(); + + // Verify the conversion codec name + const std::string expectedName = + SrcGridT::gridType() + "_to_" + typeNameAsString(); + EXPECT_EQ((codecs::ScalarCodec::name()), + expectedName); + + // Verify the codec is registered after initialize() + EXPECT_TRUE(CodecRegistry::isRegistered(expectedName)); + + // Write a SrcGridT with a non-zero fill value + const std::string testPath = "test_" + expectedName + ".vdb"; + const std::string gridName = SrcGridT::gridType(); + typename SrcGridT::Ptr srcGrid = + SrcGridT::create(typename SrcGridT::ValueType(0)); + srcGrid->setName(gridName); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), + typename SrcGridT::ValueType(1), true); + + { + io::File f(testPath); + f.write(GridPtrVec{srcGrid}); + } + + // Read back with the specified ReadMode — triggers the conversion codec + ReadOptions opts; + opts.readMode = mode; + + GridBase::Ptr base; + { + io::File f(testPath); + f.open(); + base = f.readGrid(gridName, opts); + f.close(); + } + ASSERT_TRUE(base); + + // The returned grid must be a DstGridT + EXPECT_TRUE(base->isType()); + typename DstGridT::Ptr dstGrid = gridPtrCast(base); + ASSERT_TRUE(dstGrid); + + // Topology must match the source grid + EXPECT_TRUE(srcGrid->tree().hasSameTopology(dstGrid->tree())); + + // All active voxel values must be true + for (typename DstGridT::ValueOnCIter it = dstGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, typename DstGridT::ValueType(true)); + } + + // Background value must be false + EXPECT_EQ(dstGrid->background(), typename DstGridT::ValueType(false)); + + // Cleanup + std::remove(testPath.c_str()); +} + +TEST_F(TestCodec, testNumericToBoolCodecConversion) +{ + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); +} + +TEST_F(TestCodec, testNumericToMaskCodecConversion) +{ + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); + testConvertCodecImpl(); +} From 94e20c0239f495843f04e3392e9426f6f1946e5a Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 17 Feb 2026 11:22:54 -0800 Subject: [PATCH 33/80] Update to Tree Iterator to work with Mask Grids Signed-off-by: Dan Bailey --- openvdb/openvdb/tree/Iterator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvdb/openvdb/tree/Iterator.h b/openvdb/openvdb/tree/Iterator.h index 1116c2b34f..e80ee1e6e5 100644 --- a/openvdb/openvdb/tree/Iterator.h +++ b/openvdb/openvdb/tree/Iterator.h @@ -83,7 +83,7 @@ class IteratorBase /// @details If this iterator is pointing to a child node, then the current item /// in the parent node's table is required to be inactive. In that case, /// this method has no effect. - void setValueOff() const { parent().mValueMask.setOff(this->pos()); } + void setValueOff() const { parent().setValueMask(this->pos(), false); } /// Return the coordinates of the item to which this iterator is pointing. Coord getCoord() const { return parent().offsetToGlobalCoord(this->pos()); } From c1eabdbbdf76b3e72a060de3550077aba49e28d6 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 6 May 2026 16:43:12 -0700 Subject: [PATCH 34/80] Add pending changes Signed-off-by: Dan Bailey --- pendingchanges/iorefactor.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 pendingchanges/iorefactor.txt diff --git a/pendingchanges/iorefactor.txt b/pendingchanges/iorefactor.txt new file mode 100644 index 0000000000..331fd2ba22 --- /dev/null +++ b/pendingchanges/iorefactor.txt @@ -0,0 +1,8 @@ +OpenVDB: + API changes: + - GridDescriptor::read() has been deprecated, use GridDescriptor::readHeader() + followed by GridDescriptor::readStreamPos() instead. + + Improvements: + - Large refactor of I/O classes, many private and protected member functions have + been modified. No change in behavior for io::File and io::Stream. From ea040c95543afa9103e86a9c8216df6c1c74a48d Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 6 May 2026 21:43:24 -0700 Subject: [PATCH 35/80] Add pending changes Signed-off-by: Dan Bailey --- pendingchanges/treebaseio.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 pendingchanges/treebaseio.txt diff --git a/pendingchanges/treebaseio.txt b/pendingchanges/treebaseio.txt new file mode 100644 index 0000000000..260862ee03 --- /dev/null +++ b/pendingchanges/treebaseio.txt @@ -0,0 +1,4 @@ + OpenVDB: + API changes: + - TreeBase::readTopology() and TreeBase::writeTopology() are now pure virtual. Derived + classes of TreeBase now need to read and write int32_t(1) to remain backwards-compatible. From c4f28dad40d7f53f2a78be767e8f112931010378 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 7 May 2026 11:39:23 -0700 Subject: [PATCH 36/80] Add pending changes Signed-off-by: Dan Bailey --- pendingchanges/iooptions.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 pendingchanges/iooptions.txt diff --git a/pendingchanges/iooptions.txt b/pendingchanges/iooptions.txt new file mode 100644 index 0000000000..ca67c833f5 --- /dev/null +++ b/pendingchanges/iooptions.txt @@ -0,0 +1,6 @@ +OpenVDB: + API changes: + - Added optional trailing WriteOptions parameter to io::File::write() and io::Stream::write(). + No changes in behavior. + - Added optional trailing ReadOptions parameter to io::File::getGrids() and io::File::readGrid(). + No changes in behavior. From 0e420878120e269524fc26c57f90091e93a1b80c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 12 May 2026 10:51:09 -0700 Subject: [PATCH 37/80] Improve and extend docs Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Codec.h | 329 +++++++++++++++++++++++++++++++++---- 1 file changed, 296 insertions(+), 33 deletions(-) diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 304a5b7976..6800fa240c 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -24,6 +24,13 @@ namespace io { namespace internal { +// Global registration of codecs. These initialization functions are NOT intended +// to be called directly by the user. +// While they are thread-safe, they lack the early-exit logic of openvdb::initialize() - +// this can result in exceptions being thrown when attempting to register a codec that is +// already registered. +// They are also used extensively in unit tests. + OPENVDB_API void initialize(); OPENVDB_API void uninitialize(); @@ -31,21 +38,58 @@ OPENVDB_API void uninitialize(); //////////////////////////////////////// +/// @brief Controls how grid data is read. +/// +/// Passed via @c ReadOptions::readMode to influence which portions of a grid +/// are deserialized and whether any on-the-fly conversion is performed during read. +/// Codecs are expected to honour this setting in their @c readTopology() and +/// @c readBuffers() implementations. enum class ReadMode { - Original, // Read data as-is (default) - TopologyOnly // Read topology only, no buffer data + /// Deserialize both topology and all value buffers. This is the default + /// behaviour and produces a fully populated grid identical to the one + /// that was written. + Original, + /// Deserialize topology only; value buffers are skipped. The resulting + /// grid has a valid tree structure (active/inactive state, node + /// hierarchy) but leaf buffer data is left at its default (background) + /// value. Useful when only the active-voxel mask is needed and + /// avoiding the cost of reading large value buffers is desirable. + TopologyOnly }; -/// Base class for per-grid-type codec-specific options -/// Derive from this to add codec-specific read configuration +/// @brief Base class for per-grid-type, codec-specific read options. +/// +/// Codecs that require type-specific read configuration should derive from +/// this class and store an instance in @c ReadOptions::typeData, keyed by +/// the grid type string (e.g. @c "Vec3SGrid"). +/// +/// @par Example +/// @code +/// struct MyCodecReadOptions : public ReadTypedOptions { +/// bool someFlag = false; +/// }; +/// +/// ReadOptions options; +/// auto data = std::make_shared(); +/// data->someFlag = true; +/// options.typeData["Vec3SGrid"] = data; +/// @endcode struct OPENVDB_API ReadTypedOptions { using Ptr = std::shared_ptr; virtual ~ReadTypedOptions() = default; - /// Type-safe casting helper for derived ReadTypedOptions types - /// Uses OPENVDB_ASSERT to verify type safety when assertions are enabled + /// @brief Downcast a @c ReadTypedOptions::Ptr to a concrete derived type. + /// + /// Performs a @c static_cast to @c T after verifying via @c dynamic_cast + /// (when @c OPENVDB_ASSERT is enabled) that @p data actually holds an + /// instance of @c T. Calling this with a mismatched type is undefined + /// behaviour in release builds. + /// + /// @tparam T The concrete @c ReadTypedOptions subclass to cast to. + /// @param data A shared pointer to the base @c ReadTypedOptions object. + /// @return A reference to the underlying @c T instance. template static T& cast(const Ptr& data) { OPENVDB_ASSERT(dynamic_cast(data.get()) != nullptr); @@ -53,34 +97,73 @@ struct OPENVDB_API ReadTypedOptions } }; // struct ReadTypedOptions -/// Global read configuration that applies to all grids -/// Contains common options shared across all codec types +/// @brief Global read configuration passed to every codec during deserialization. +/// +/// An instance of this struct is threaded through all codec @c readTopology() and +/// @c readBuffers() calls so that site-wide policy (clipping, read mode) and any +/// type-specific overrides are available in a single place. +/// +/// @par Clipping +/// When @c clipBBox is non-empty (i.e. @c !clipBBox.empty()), codecs should +/// restrict the voxel data they load to the region that intersects that world-space +/// bounding box. An empty bbox (the default) means no clipping is applied. +/// +/// @par Read mode +/// @c readMode controls the granularity of deserialization. The default, +/// @c ReadMode::Original, loads both topology and value buffers. +/// @c ReadMode::TopologyOnly skips value buffers entirely, which can be +/// significantly faster when only the active-voxel mask is needed. +/// +/// @par Per-type options +/// @c typeData allows callers to attach codec-specific configuration for +/// individual grid types. Entries are keyed by the grid type string +/// (e.g. @c "Vec3SGrid") and hold a @c ReadTypedOptions-derived object. +/// Codecs retrieve their entry via @c ReadTypedOptions::cast(). struct OPENVDB_API ReadOptions { - // BBox Clip Data + /// World-space bounding box used to spatially clip the read. + /// An empty bbox (the default) disables clipping. BBoxd clipBBox = BBoxd(); - // Grid Conversion Options + /// Controls which portions of each grid are deserialized. + /// Defaults to @c ReadMode::Original (full topology + value buffers). ReadMode readMode = ReadMode::Original; - // Per-grid-type codec-specific options, keyed by grid type string + /// Optional per-grid-type codec configuration, keyed by grid type string. + /// Values are @c ReadTypedOptions subclass instances; use + /// @c ReadTypedOptions::cast() to retrieve the concrete type. std::unordered_map typeData; }; // struct ReadOptions -/// Global write configuration that applies to all grids -/// Contains common options shared across all codec types +/// @brief Global write configuration passed to every codec during serialization. +/// +/// Currently carries no fields, but is provided for forward compatibility: +/// future write-time options (e.g. compression hints, metadata policies) can +/// be added here without changing the codec interface. struct OPENVDB_API WriteOptions { }; // struct WriteOptions +/// @brief Severity level for a read diagnostic. +/// @note Currently only @c Warning is defined. enum class DiagnosticSeverity { Warning }; +/// @brief A single read diagnostic message. +/// @note @c context is typically the grid name or another identifier that +/// locates the source of the message. struct ReadDiagnostic { DiagnosticSeverity severity; std::string context; std::string message; }; +/// @brief Thread-safe collection of read diagnostics accumulated during a codec read. +/// +/// @details Diagnostics report situations where a requested read option could +/// not be honoured by the codec — for example, clipping that was not applied +/// natively and had to fall back to a post-process, or an option that the codec +/// ignores entirely. The object must be explicitly enabled before any messages +/// are recorded; it is disabled by default. struct OPENVDB_API ReadDiagnostics { ReadDiagnostics() = default; ReadDiagnostics(const ReadDiagnostics& other) { @@ -96,19 +179,28 @@ struct OPENVDB_API ReadDiagnostics { } return *this; } + /// @brief Append a warning diagnostic. No-op if the object is disabled. + /// @note This method is thread-safe. void addWarning(const std::string& context, const std::string& message) { if (!mEnabled) return; std::lock_guard lock(mMutex); mDiagnostics.push_back({DiagnosticSeverity::Warning, context, message}); } + /// @brief Remove all recorded diagnostics. + /// @note This method is thread-safe. void clear() { std::lock_guard lock(mMutex); mDiagnostics.clear(); } + /// @brief Enable diagnostic recording. void enable() { mEnabled = true; } + /// @brief Disable diagnostic recording. void disable() { mEnabled = false; } + /// @brief Return @c true if diagnostic recording is enabled. bool enabled() const { return mEnabled; } + /// @brief Return the list of recorded diagnostics. const std::vector& diagnostics() const { return mDiagnostics; } + /// @brief Return @c true if no diagnostics have been recorded. bool empty() const { return mDiagnostics.empty(); } private: bool mEnabled = false; @@ -116,59 +208,230 @@ struct OPENVDB_API ReadDiagnostics { mutable std::mutex mMutex; }; // struct ReadDiagnostics -/// Per-codec mutable data created by Codec::createData() -/// Contains the grid and any codec-specific state +/// @brief Mutable per-operation state created by @c Codec::createData() and +/// passed into every read and write call on a given codec. +/// +/// @details The main purpose of this class is to provide a mechanism for passing +/// additional state between the different methods in the codec. @c ReadOptions is +/// immutable and cannot carry per-operation state; the codec itself is stateless +/// by design (to allow sharing across threads) and therefore also cannot be used +/// for this purpose. +/// +/// The base class provides the single field that every codec requires: +/// a shared pointer to the @c GridBase being populated (on read) or inspected +/// (on write). Derived codecs that need additional transient state should +/// subclass @c CodecData and add those fields. +/// +/// A @c CodecData instance is created once per grid per read or write operation +/// by @c Codec::createData(), which also allocates the concrete @c GridBase +/// subclass and stores it in @c grid. +/// +/// @par Deriving from CodecData +/// @code +/// struct MyCodecData : public io::CodecData { +/// // Additional per-operation state goes here +/// SomeCache intermediateBuffer; +/// }; +/// @endcode +/// +/// Inside @c Codec::createData(), the derived type is allocated and returned: +/// @code +/// io::CodecData::Ptr createData() override { +/// auto data = std::make_unique(); +/// data->grid = MyGridType::create(); +/// return data; +/// } +/// @endcode +/// +/// In the read and write methods, downcast the reference back to the concrete +/// type to access the extra fields: +/// @code +/// void readBuffers(std::istream& is, io::CodecData& data, ...) override { +/// auto& myData = static_cast(data); +/// MyGridType& grid = static_cast(*myData.grid); +/// // Use myData.intermediateBuffer, grid, etc. +/// } +/// @endcode struct OPENVDB_API CodecData { using Ptr = std::unique_ptr; virtual ~CodecData() = default; + /// The grid being populated on read, or being serialized on write. SharedPtr grid; }; // struct CodecData +/// @brief Abstract base class for grid I/O codecs. +/// +/// @details A codec encapsulates the serialization and deserialization logic +/// for a specific grid type (or family of grid types). The I/O subsystem +/// selects a codec by name at runtime, calls @c createData() to allocate +/// per-operation state, and then dispatches to the appropriate read or write +/// methods. +/// +/// Codecs must be stateless: all mutable per-operation state is stored in a +/// @c CodecData object (see above) that is created fresh for each grid read +/// or write. This ensures that a single registered codec instance can safely +/// handle concurrent I/O operations. +/// +/// @par Implementing a new codec +/// +/// 1. **Declare the codec struct**, deriving publicly from @c Codec (or from a +/// convenience intermediate that already handles topology, if one exists for +/// the grid family in question): +/// @code +/// struct MyCodec : public io::Codec { ... }; +/// @endcode +/// +/// 2. **Provide a static @c name() method** that returns the unique string +/// identifier under which the codec will be registered. Any globally +/// unique ASCII string is acceptable: +/// @code +/// static std::string name() { return "mycodec"; } +/// @endcode +/// +/// 3. **Override @c createData()** (the only pure-virtual method). Allocate a +/// @c CodecData (or a derived subclass if extra per-operation state is +/// needed), create the appropriate @c GridBase subclass, assign it to +/// @c CodecData::grid, and return the object: +/// @code +/// io::CodecData::Ptr createData() override { +/// auto data = std::make_unique(); +/// data->grid = MyGridType::create(); +/// return data; +/// } +/// @endcode +/// +/// 4. **Override the read and/or write methods** as required. All four +/// methods have default no-op implementations, so override only those that +/// the codec actually uses. The I/O layer separates topology (the tree +/// structure and active/inactive voxel mask) from value buffers (the actual +/// voxel data), allowing callers to request topology-only reads via +/// @c ReadOptions::readMode. Implementations should honour that setting: +/// @code +/// void readTopology(std::istream& is, io::CodecData& data, +/// const io::ReadOptions& options, +/// io::ReadDiagnostics& diagnostics) override +/// { +/// MyGridType& grid = static_cast(*data.grid); +/// // Deserialize the tree structure into grid... +/// } +/// +/// void readBuffers(std::istream& is, io::CodecData& data, +/// const io::ReadOptions& options, +/// io::ReadDiagnostics& diagnostics) override +/// { +/// if (options.readMode == io::ReadMode::TopologyOnly) return; +/// MyGridType& grid = static_cast(*data.grid); +/// // Deserialize voxel values into grid... +/// } +/// @endcode +/// If a requested option (e.g. spatial clipping) cannot be honoured natively, +/// record a warning via @c diagnostics.addWarning() rather than silently ignoring it. +/// +/// 5. **Register the codec** once at start-up, typically from the library's +/// @c initialize() function, using @c CodecRegistry::registerCodec(): +/// @code +/// io::CodecRegistry::registerCodec(); +/// @endcode +/// +/// @note The codec struct itself is never copied; the registry takes ownership +/// of a single heap-allocated instance via @c Codec::Ptr +/// (@c std::unique_ptr). struct OPENVDB_API Codec { using Ptr = std::unique_ptr; virtual ~Codec() = default; - /// Create codec data including a new grid of the appropriate type - /// @return A new CodecData object containing the grid and any codec-specific state + /// @brief Allocate per-operation codec state, including the target grid. + /// + /// This is the only pure-virtual method. Implementations must create a + /// concrete @c GridBase subclass appropriate for the codec, store it in + /// @c CodecData::grid, and return the @c CodecData (or a derived subclass + /// carrying additional state). The returned object is passed by reference + /// to every subsequent read or write call for the same grid. + /// + /// @return A fully initialized @c CodecData whose @c grid field is non-null. virtual CodecData::Ptr createData() = 0; - /// Read the grid topology - virtual void readTopology(std::istream&, CodecData&, const ReadOptions&, ReadDiagnostics&) { } - - /// Read all data buffers for this grid - virtual void readBuffers(std::istream&, CodecData&, const ReadOptions&, ReadDiagnostics&) { } - - /// Write the grid topology - virtual void writeTopology(std::ostream&, const GridBase&, const WriteOptions&) { } - - /// Write all data buffers for this grid - virtual void writeBuffers(std::ostream&, const GridBase&, const WriteOptions&) { } + /// @brief Deserialize the grid topology (tree structure and active-voxel + /// mask) from @a is into the grid held by @a data. + virtual void readTopology(std::istream& /*is*/, CodecData& /*data*/, + const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { } + + /// @brief Deserialize all voxel-value buffers from @a is into the grid + /// held by @a data. + /// + /// The default implementation is a no-op. Override to populate leaf-node + /// value buffers after topology has been established. When + /// @c options.readMode is @c ReadMode::TopologyOnly this method will not + /// be called by the I/O layer, so implementations may also guard against + /// that mode internally for safety. If @c options.clipBBox is non-empty, + /// restrict the loaded data to the region that intersects it; if the codec + /// cannot honour clipping natively, fall back to a post-process and record + /// a warning via @a diagnostics. + virtual void readBuffers(std::istream& /*is*/, CodecData& /*data*/, + const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { } + + /// @brief Serialize the grid topology (tree structure and active-voxel + /// mask) from @a grid to @a os. + /// + /// The default implementation is a no-op. Override when the codec stores + /// topology as a distinct section that precedes the value buffers in the + /// stream. + virtual void writeTopology(std::ostream& /*os*/, const GridBase& /*grid*/, + const WriteOptions& /*options*/) { } + + /// @brief Serialize all voxel-value buffers from @a grid to @a os. + /// + /// The default implementation is a no-op. Override to write the leaf-node + /// value buffers that follow the topology section. + virtual void writeBuffers(std::ostream& /*os*/, const GridBase& /*grid*/, + const WriteOptions& /*options*/) { } }; // struct Codec +/// @brief A thread-safe, process-global registry that maps codec names to +/// @c Codec instances. +/// +/// Codecs are identified by a unique string name (typically provided by @c Codec::name()) +/// and stored as @c Codec::Ptr (i.e. @c std::unique_ptr). +/// +/// @par Typical usage +/// @code +/// // Register a custom codec once at start-up (e.g. from initialize()): +/// CodecRegistry::registerCodec(); +/// @endcode +/// +/// @note Attempting to register a codec whose name is already present throws +/// @c openvdb::KeyError. Call @c isRegistered() first when the caller +/// cannot guarantee uniqueness. struct OPENVDB_API CodecRegistry { - /// Return @c true if the given codec name is registered. + /// Return @c true if a codec with the given @a name has been registered. static bool isRegistered(const std::string& name); - /// Register a codec by name with a factory function. + /// Register a codec under the explicit string @a name, transferring + /// ownership of @a codec to the registry. + /// @throw KeyError if @a name is already registered. static void registerCodecByName(const std::string& name, Codec::Ptr&& codec); - /// Register a codec by type. + /// Convenience wrapper that registers @c CodecT using the name returned + /// by @c CodecT::name(), constructing the instance internally. + /// @throw KeyError if the codec is already registered. template static void registerCodec() { registerCodecByName(CodecT::name(), std::make_unique()); } - /// Return a pointer to the codec with the given name, or nullptr if not found. + /// Return a raw (non-owning) pointer to the codec registered under + /// @a name, or @c nullptr if no such codec exists. + /// The returned pointer remains valid for the lifetime of the registry. static Codec* get(const std::string& name); - /// Clear the codec registry. + /// Deregister all codecs and reset the registry to an empty state. static void clear(); }; // struct CodecRegistry From 2a54d6d86850ea0349a71d6fa955b6cb09aacd38 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 12 May 2026 13:50:14 -0700 Subject: [PATCH 38/80] Update docs Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Codec.h | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index bc26233a6d..4fbf3412d1 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -38,9 +38,18 @@ OPENVDB_API void uninitialize(); //////////////////////////////////////// +/// @brief Controls which operations a codec exposes. +/// +/// Passed to codec registration to restrict how the codec may be used at +/// runtime. A @c ReadOnly codec can still be used to read existing files but +/// will not be selected as the write codec for new grids. enum class CodecMode { - ReadWrite, // Both read and write enabled (default) - ReadOnly // Only read methods enabled + /// Both @c readTopology()/@c readBuffers() and + /// @c writeTopology()/@c writeBuffers() are enabled. This is the default. + ReadWrite, + /// Only @c readTopology() and @c readBuffers() are enabled; the codec + /// will not be offered as a write target. + ReadOnly }; /// @brief Controls how grid data is read. @@ -54,9 +63,20 @@ enum class ReadMode { /// behaviour and produces a fully populated grid identical to the one /// that was written. Original, - Half, // Read data as half precision - Bool, // Read data as boolean - Mask, // Read data as mask + /// Deserialize topology and value buffers, converting each voxel value + /// to half-precision floating point on the fly. Codecs that do not + /// support native half conversion should fall back to @c Original and + /// record a @c ReadDiagnostic warning. + Half, + /// Deserialize topology and value buffers, converting each voxel value + /// to @c bool (non-zero to @c true, zero to @c false). Produces a + /// @c BoolGrid whose active set matches the source grid's active set. + Bool, + /// Deserialize topology only and promote the result to a mask grid, + /// discarding all value data. Equivalent to reading @c TopologyOnly and + /// then constructing a @c MaskGrid from the active-voxel set, but may + /// be performed more efficiently inside the codec. + Mask, /// Deserialize topology only; value buffers are skipped. The resulting /// grid has a valid tree structure (active/inactive state, node /// hierarchy) but leaf buffer data is left at its default (background) @@ -117,8 +137,11 @@ struct OPENVDB_API ReadTypedOptions /// bounding box. An empty bbox (the default) means no clipping is applied. /// /// @par Read mode -/// @c readMode controls the granularity of deserialization. The default, -/// @c ReadMode::Original, loads both topology and value buffers. +/// @c readMode controls the granularity and on-the-fly conversion applied +/// during deserialization. The default, @c ReadMode::Original, loads both +/// topology and value buffers without modification. +/// @c ReadMode::Half, @c ReadMode::Bool, and @c ReadMode::Mask request +/// in-place type conversion as data is read. /// @c ReadMode::TopologyOnly skips value buffers entirely, which can be /// significantly faster when only the active-voxel mask is needed. /// From f4682d14678101d2fe5153fa94dbfbc5908a2757 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 13 May 2026 16:36:41 -0700 Subject: [PATCH 39/80] Add pending notes Signed-off-by: Dan Bailey --- pendingchanges/houdinivulkan.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pendingchanges/houdinivulkan.txt diff --git a/pendingchanges/houdinivulkan.txt b/pendingchanges/houdinivulkan.txt new file mode 100644 index 0000000000..b64d663f0a --- /dev/null +++ b/pendingchanges/houdinivulkan.txt @@ -0,0 +1,3 @@ +OpenVDB: + Houdini: + - Add new Vulkan viewport support for VDB Points primitives and remove support for delayed loading. From 27f6e19edd8142598fb830d9c7e75ddcba0b8bbf Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 13 May 2026 16:57:35 -0700 Subject: [PATCH 40/80] Add back support for 221 VDB files Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/ScalarLeafCodec.h | 15 +++++++++++++ openvdb/openvdb/codecs/TopologyCodec.h | 28 +++++++++++++++++++----- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/openvdb/openvdb/codecs/ScalarLeafCodec.h b/openvdb/openvdb/codecs/ScalarLeafCodec.h index f72efc56d9..df13bc8f7f 100644 --- a/openvdb/openvdb/codecs/ScalarLeafCodec.h +++ b/openvdb/openvdb/codecs/ScalarLeafCodec.h @@ -48,6 +48,21 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, if (seekable) valueMask.seek(is); else valueMask.load(is); + // Pre-node-mask-compression format stored the origin and buffer count + // inline after the value mask. + int8_t numBuffers = 1; + if (io::getFormatVersion(is) < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) { + Coord origin; + is.read(reinterpret_cast(&origin), sizeof(Coord::ValueType) * 3); + leaf.setOrigin(origin); + + is.read(reinterpret_cast(&numBuffers), sizeof(int8_t)); + + if (numBuffers > 1) { + OPENVDB_THROW(IoError, "Old file format 221 (FLOAT_FRUSTUM_BBOX) with multiple buffers is not supported"); + } + } + if (skip) { if (seekable) { io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf); diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index c99b39277b..e2a09b8ddc 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -182,7 +182,9 @@ struct ReadTopologyOp valueMask.load(is); node.setValueMaskUnsafe(valueMask); - const Index numValues = NodeT::NUM_VALUES; + const bool oldVersion = + (io::getFormatVersion(is) < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION); + const Index numValues = (oldVersion ? childMask.countOff() : NodeT::NUM_VALUES); { // Read in (and uncompress, if necessary) all of this node's values // into a contiguous array. @@ -191,13 +193,27 @@ struct ReadTopologyOp io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf); // Copy values from the array into this node's table. - if constexpr (std::is_same_v) { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), values[iter.pos()]); + if (oldVersion) { + Index n = 0; + if constexpr (std::is_same_v) { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), values[n++]); + } + } else { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n++])); + } } + OPENVDB_ASSERT(n == numValues); } else { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); + if constexpr (std::is_same_v) { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), values[iter.pos()]); + } + } else { + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); + } } } } From ff40f237791a56ee32226b4b3e8a9f17a9ea27df Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 14 May 2026 14:21:29 -0700 Subject: [PATCH 41/80] Address feedback Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 6 ++++++ openvdb/openvdb/codecs/BoolCodec.h | 2 +- openvdb/openvdb/codecs/ScalarCodec.h | 16 ++------------- openvdb/openvdb/codecs/TopologyCodec.h | 20 ++++--------------- openvdb/openvdb/codecs/ValueMaskCodec.h | 6 +++--- .../codecs/{ => impl}/ScalarLeafCodec.h | 18 ++++++++++------- openvdb/openvdb/io/Archive.cc | 18 +++++++---------- 7 files changed, 34 insertions(+), 52 deletions(-) rename openvdb/openvdb/codecs/{ => impl}/ScalarLeafCodec.h (86%) diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 9d8d819d49..3572583e20 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -372,6 +372,10 @@ set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES codecs/ValueMaskCodec.h ) +set(OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES + codecs/impl/ScalarLeafCodec.h +) + set(OPENVDB_LIBRARY_IO_INCLUDE_FILES io/Archive.h io/Codec.h @@ -568,6 +572,7 @@ if(USE_EXPLICIT_INSTANTIATION) set(OPENVDB_LIBRARY_INSTANTIATION_HEADERS ${OPENVDB_LIBRARY_TOOLS_INCLUDE_FILES} ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} + ${OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES} ) # A new source file is generated by CMake for every header @@ -745,6 +750,7 @@ install(FILES ${OPENVDB_LIBRARY_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INC install(FILES ${CMAKE_CURRENT_BINARY_DIR}/openvdb/version.h DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb) install(FILES ${OPENVDB_LIBRARY_IO_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/io) install(FILES ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/codecs) +install(FILES ${OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/codecs/impl) install(FILES ${OPENVDB_LIBRARY_MATH_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/math) install(FILES ${OPENVDB_LIBRARY_POINTS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points) install(FILES ${OPENVDB_LIBRARY_POINTS_IMPL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points/impl) diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h index 68d7997a89..e8537fa662 100644 --- a/openvdb/openvdb/codecs/BoolCodec.h +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -102,7 +102,7 @@ struct WriteBoolBuffersOp } // namespace internal template -struct BoolCodec : public TopologyCodec +struct BoolCodec final: public TopologyCodec { using Ptr = std::unique_ptr>; diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index c9f8dbe49e..114c575bcf 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -10,7 +10,7 @@ #include #include -#include "ScalarLeafCodec.h" +#include "impl/ScalarLeafCodec.h" #include "TopologyCodec.h" namespace openvdb { @@ -130,7 +130,7 @@ void scalarCodecWriteBuffers(const GridT& grid, std::ostream& os) } // namespace internal template -struct ScalarCodec : public TopologyCodec +struct ScalarCodec final: public TopologyCodec { static_assert(GridT::TreeType::RootNodeType::template SameConfiguration< typename StorageGridT::TreeType::RootNodeType>::value, @@ -138,20 +138,8 @@ struct ScalarCodec : public TopologyCodec using Ptr = std::unique_ptr>; - struct ScalarCodecData : public io::CodecData - { - std::vector leafCoords; - }; - ~ScalarCodec() noexcept = default; - io::CodecData::Ptr createData() final - { - auto data = std::make_unique(); - data->grid = GridT::create(); - return data; - } - static inline std::string name() { if constexpr (std::is_same_v) { diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index e2a09b8ddc..e602f176ce 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -195,25 +195,13 @@ struct ReadTopologyOp // Copy values from the array into this node's table. if (oldVersion) { Index n = 0; - if constexpr (std::is_same_v) { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), values[n++]); - } - } else { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n++])); - } + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n++])); } OPENVDB_ASSERT(n == numValues); } else { - if constexpr (std::is_same_v) { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), values[iter.pos()]); - } - } else { - for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); - } + for (auto iter = node.beginValueAll(); iter; ++iter) { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); } } } diff --git a/openvdb/openvdb/codecs/ValueMaskCodec.h b/openvdb/openvdb/codecs/ValueMaskCodec.h index 17171cebf7..27f6289cb9 100644 --- a/openvdb/openvdb/codecs/ValueMaskCodec.h +++ b/openvdb/openvdb/codecs/ValueMaskCodec.h @@ -55,7 +55,7 @@ struct ReadValueMaskBuffersOp leaf.getValueMask().load(is); // Read in the origin. Coord origin; - is.read(reinterpret_cast(&origin), sizeof(Coord::ValueType) * 3); + origin.read(is); leaf.setOrigin(origin); } @@ -84,7 +84,7 @@ struct WriteValueMaskBuffersOp leaf.getValueMask().save(os); // Write out the origin. - os.write(reinterpret_cast(&leaf.origin()), sizeof(Coord::ValueType) * 3); + leaf.origin().write(os); } std::ostream& os; @@ -94,7 +94,7 @@ struct WriteValueMaskBuffersOp } // namespace internal template -struct ValueMaskCodec : public TopologyCodec +struct ValueMaskCodec final: public TopologyCodec { using Ptr = std::unique_ptr>; diff --git a/openvdb/openvdb/codecs/ScalarLeafCodec.h b/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h similarity index 86% rename from openvdb/openvdb/codecs/ScalarLeafCodec.h rename to openvdb/openvdb/codecs/impl/ScalarLeafCodec.h index df13bc8f7f..ad9e088604 100644 --- a/openvdb/openvdb/codecs/ScalarLeafCodec.h +++ b/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h @@ -1,8 +1,8 @@ // Copyright Contributors to the OpenVDB Project // SPDX-License-Identifier: Apache-2.0 -#ifndef OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED -#define OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED +#ifndef OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED #include @@ -88,10 +88,14 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, } } else if constexpr (std::is_same_v) { // Bool leaf: must read storage values regardless of seekability, then convert. - StorageBufferT storageTemp; - io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); - for (Index i = 0; i < SIZE; ++i) { - leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); + if constexpr (std::is_same_v) { + io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf); + } else { + StorageBufferT storageTemp; + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + for (Index i = 0; i < SIZE; ++i) { + leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); + } } } else { leaf.buffer().allocate(); @@ -116,4 +120,4 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, } // namespace OPENVDB_VERSION_NAME } // namespace openvdb -#endif // OPENVDB_IO_CODECS_SCALARLEAFCODEC_HAS_BEEN_INCLUDED +#endif // OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index b05d8eefb5..86dd2755cc 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -887,22 +887,18 @@ Archive::readGridCount(std::istream& is) io::Codec* Archive::findCodec(const std::string& gridType, const io::ReadOptions& options) { - // Determine the I/O codec to use to read this grid - auto codec = io::CodecRegistry::get(gridType); - // if readMode is Half, then search for a codec that converts // from the storage grid type to the grid type if (options.readMode == ReadMode::Half) { - codec = io::CodecRegistry::get(gridType + "_to_half"); - } - if (options.readMode == ReadMode::Bool) { - codec = io::CodecRegistry::get(gridType + "_to_bool"); - } - if (options.readMode == ReadMode::Mask) { - codec = io::CodecRegistry::get(gridType + "_to_mask"); + return io::CodecRegistry::get(gridType + "_to_half"); + } else if (options.readMode == ReadMode::Bool) { + return io::CodecRegistry::get(gridType + "_to_bool"); + } else if (options.readMode == ReadMode::Mask) { + return io::CodecRegistry::get(gridType + "_to_mask"); } - return codec; + // Determine the I/O codec to use to read this grid + return io::CodecRegistry::get(gridType); } From 96964f2c3aba8bab887283d493fa1f359a706681 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 13 Apr 2026 16:28:10 -0700 Subject: [PATCH 42/80] Add PointIndexCodec Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 1 + openvdb/openvdb/codecs/PointIndexCodec.h | 157 +++++++++++++++++++++ openvdb/openvdb/io/Codec.cc | 2 + openvdb/openvdb/unittest/CMakeLists.txt | 1 + openvdb/openvdb/unittest/TestPointCodec.cc | 147 +++++++++++++++++++ 5 files changed, 308 insertions(+) create mode 100644 openvdb/openvdb/codecs/PointIndexCodec.h create mode 100644 openvdb/openvdb/unittest/TestPointCodec.cc diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 3572583e20..e9437b86de 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -367,6 +367,7 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES codecs/BoolCodec.h + codecs/PointIndexCodec.h codecs/ScalarCodec.h codecs/TopologyCodec.h codecs/ValueMaskCodec.h diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h new file mode 100644 index 0000000000..d8d9f1ee32 --- /dev/null +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -0,0 +1,157 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include "ScalarLeafCodec.h" +#include "TopologyCodec.h" + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +template +struct ReadPointIndexBuffersOp +{ + using TreeT = typename GridT::TreeType; + using RootT = typename TreeT::RootNodeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + ReadPointIndexBuffersOp(std::istream& _is, bool _saveFloatAsHalf, + const ValueT& _background) + : is(_is) + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) { } + + template + void operator()(NodeT&, size_t) { } + + void operator()(LeafT& leaf, size_t) + { + using BaseLeaf = typename LeafT::BaseLeaf; + + // Read the value mask and voxel data via base class + BaseLeaf& baseLeaf = static_cast(leaf); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr); + + // Read the number of indices. + Index64 numIndices = Index64(0); + is.read(reinterpret_cast(&numIndices), sizeof(Index64)); + + // Read the indices data. + leaf.indices().resize(size_t(numIndices)); + is.read(reinterpret_cast(leaf.indices().data()), numIndices * sizeof(ValueT)); + + // Reserved for future use. + Index64 auxDataBytes = Index64(0); + is.read(reinterpret_cast(&auxDataBytes), sizeof(Index64)); + if (auxDataBytes > 0) { + // For now, read and discard any auxiliary data. + std::unique_ptr auxData{new char[auxDataBytes]}; + is.read(auxData.get(), auxDataBytes); + } + } + + std::istream& is; + const bool saveFloatAsHalf; + const ValueT& background; +}; // struct ReadPointIndexBuffersOp + +template +struct WritePointIndexBuffersOp +{ + using TreeT = typename GridT::TreeType; + using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; + + WritePointIndexBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + : os(_os) + , saveFloatAsHalf(_saveFloatAsHalf) { } + + template + void operator()(const NodeT&, size_t) { } + + void operator()(const LeafT& leaf, size_t) + { + using BaseLeaf = typename LeafT::BaseLeaf; + + // Write out the value mask and voxel values via base class + const BaseLeaf& baseLeaf = static_cast(leaf); + writeScalarLeafBuffers(baseLeaf, os, saveFloatAsHalf); + + // Write the number of indices. + Index64 numIndices = Index64(leaf.indices().size()); + os.write(reinterpret_cast(&numIndices), sizeof(Index64)); + + // Write the indices data. + os.write(reinterpret_cast(leaf.indices().data()), numIndices * sizeof(ValueT)); + + // Reserved for future use. + const Index64 auxDataBytes = Index64(0); + os.write(reinterpret_cast(&auxDataBytes), sizeof(Index64)); + } + + std::ostream& os; + const bool saveFloatAsHalf; +}; // struct WritePointIndexBuffersOp + +} // namespace internal + +template +struct PointIndexCodec : public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~PointIndexCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final + { + GridT& grid = static_cast(*data.grid); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in PointIndexCodec"); + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + if (options.clipBBox.isSorted()) { + diagnostics.addWarning(grid.getName(), "bounding box clipping is not supported for PointIndexGrids"); + } + + internal::ReadPointIndexBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background()); + tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + const GridT& grid = static_cast(gridBase); + + if (grid.hasMultiPassIO()) { + OPENVDB_THROW(IoError, "Multi-pass IO is not supported in PointIndexCodec"); + } + + internal::WritePointIndexBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); + } +}; // struct PointIndexCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index cfb1489efc..3e437fab17 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -58,6 +59,7 @@ void initialize() CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); // register the plugin that converts from scalar to mask/bool NumericGridTypes::foreach(); diff --git a/openvdb/openvdb/unittest/CMakeLists.txt b/openvdb/openvdb/unittest/CMakeLists.txt index 18046cd8e0..446d118027 100644 --- a/openvdb/openvdb/unittest/CMakeLists.txt +++ b/openvdb/openvdb/unittest/CMakeLists.txt @@ -146,6 +146,7 @@ else() TestParticlesToLevelSet.cc TestPointAdvect.cc TestPointAttribute.cc + TestPointCodec.cc TestPointConversion.cc TestPointCount.cc TestPointDataLeaf.cc diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc new file mode 100644 index 0000000000..2b8911419e --- /dev/null +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -0,0 +1,147 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include "util.h" // for unittest_util::genPoints + +namespace { + +class PointList { +public: + using PosType = openvdb::Vec3R; + PointList(const std::vector& points) : mPoints(&points) {} + size_t size() const { return mPoints->size(); } + void getPos(size_t n, PosType& xyz) const { xyz = (*mPoints)[n]; } +private: + std::vector const * const mPoints; +}; + +} // namespace + +class TestPointCodec: public ::testing::Test +{ +}; + +TEST_F(TestPointCodec, testPointIndexCodecIO) +{ + using namespace openvdb; + using namespace openvdb::io; + using PointIndexGrid = tools::PointIndexGrid; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Generate points on a unit sphere and build a PointIndexGrid + std::vector points; + unittest_util::genPoints(100, points); + PointList pointList(points); + + const double voxelSize = 0.1; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointIndexGrid::Ptr srcGrid = + tools::createPointIndexGrid(pointList, *transform); + srcGrid->setName("point_index_grid"); + + const std::string rawPath = "testPointIndexCodec_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointIndexGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("point_index_grid")); + f.close(); + } + ASSERT_TRUE(rawGrid); + + PointIndexGrid::Ptr rawTopo; + { + io::File f(rawPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("point_index_grid")); + rawTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(rawTopo); + EXPECT_EQ(rawTopo->activeVoxelCount(), Index64(97)); + EXPECT_EQ(rawTopo->getName(), std::string("point_index_grid")); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPointIndexCodec_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointIndexGrid::gridType())); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointIndexGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("point_index_grid")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + // Phase 3: full read comparison + EXPECT_TRUE(srcGrid->tree().hasSameTopology(srcGrid->tree())); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + { + auto codecAcc = codecGrid->getConstAccessor(); + for (PointIndexGrid::ValueOnCIter it = srcGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, codecAcc.getValue(it.getCoord())); + } + } + + // Compare leaf indices arrays + { + auto srcLeafIt = srcGrid->tree().cbeginLeaf(); + auto codecLeafIt = codecGrid->tree().cbeginLeaf(); + for (; srcLeafIt; ++srcLeafIt, ++codecLeafIt) { + ASSERT_TRUE(codecLeafIt); + EXPECT_EQ(srcLeafIt->indices().size(), codecLeafIt->indices().size()); + for (size_t i = 0; i < srcLeafIt->indices().size(); ++i) { + EXPECT_EQ(srcLeafIt->indices()[i], codecLeafIt->indices()[i]); + } + } + EXPECT_TRUE(!codecLeafIt); + } + + // Phase 4: TopologyOnly read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + PointIndexGrid::Ptr codecTopo; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("point_index_grid", topoOpts)); + codecTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecTopo); + EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0)); + EXPECT_TRUE(codecTopo->tree().leafCount() == 0); + EXPECT_EQ(codecTopo->getName(), std::string("point_index_grid")); + + // Cleanup + CodecRegistry::clear(); + std::remove(codecPath.c_str()); +} From 000fb2473fe385ebfd40de143c4c4fea4fb01a99 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 17 Feb 2026 11:24:26 -0800 Subject: [PATCH 43/80] Move PointData I/O overloads to a new header Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 1 + openvdb/openvdb/points/PointDataGrid.h | 130 +--------------------- openvdb/openvdb/points/PointDataIO.h | 148 +++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 128 deletions(-) create mode 100644 openvdb/openvdb/points/PointDataIO.h diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index e9437b86de..97317b1fb7 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -426,6 +426,7 @@ set(OPENVDB_LIBRARY_POINTS_INCLUDE_FILES points/PointConversion.h points/PointCount.h points/PointDataGrid.h + points/PointDataIO.h points/PointDelete.h points/PointGroup.h points/PointMask.h diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index fcb10bdb2a..946b5764c3 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -31,140 +31,14 @@ #include // std::pair, std::make_pair #include +#include // io::readCompressedValues(), io::writeCompressedValues(), io::writeCompressedValuesSize() + class TestPointDataLeaf; namespace openvdb { OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { -namespace io -{ - -/// @brief openvdb::io::readCompressedValues specialized on PointDataIndex32 arrays to -/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit -template<> -inline void -readCompressedValues( std::istream& is, PointDataIndex32* destBuf, Index destCount, - const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/) -{ - using compression::bloscDecompress; - - const bool seek = destBuf == nullptr; - - const size_t destBytes = destCount*sizeof(PointDataIndex32); - const size_t maximumBytes = std::numeric_limits::max(); - if (destBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot read more than " << - maximumBytes << " bytes in voxel values.") - } - - uint16_t bytes16; - - const io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is); - - if (seek && meta) { - // buffer size temporarily stored in the StreamMetadata pass - // to avoid having to perform an expensive disk read for 2-bytes - bytes16 = static_cast(meta->pass()); - // seek over size of the compressed buffer - is.seekg(sizeof(uint16_t), std::ios_base::cur); - } - else { - // otherwise read from disk - is.read(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } - - if (bytes16 == std::numeric_limits::max()) { - // read or seek uncompressed data - if (seek) { - is.seekg(destBytes, std::ios_base::cur); - } - else { - is.read(reinterpret_cast(destBuf), destBytes); - } - } - else { - // read or seek uncompressed data - if (seek) { - is.seekg(int(bytes16), std::ios_base::cur); - } - else { - // decompress into the destination buffer - std::unique_ptr bloscBuffer(new char[int(bytes16)]); - is.read(bloscBuffer.get(), bytes16); - std::unique_ptr buffer = bloscDecompress( bloscBuffer.get(), - destBytes, - /*resize=*/false); - std::memcpy(destBuf, buffer.get(), destBytes); - } - } -} - -/// @brief openvdb::io::writeCompressedValues specialized on PointDataIndex32 arrays to -/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit -template<> -inline void -writeCompressedValues( std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount, - const util::NodeMask<3>& /*valueMask*/, - const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/) -{ - using compression::bloscCompress; - - const size_t srcBytes = srcCount*sizeof(PointDataIndex32); - const size_t maximumBytes = std::numeric_limits::max(); - if (srcBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << - maximumBytes << " bytes in voxel values.") - } - - const char* charBuffer = reinterpret_cast(srcBuf); - - size_t compressedBytes; - std::unique_ptr buffer = bloscCompress( charBuffer, srcBytes, - compressedBytes, /*resize=*/false); - - if (compressedBytes > 0) { - auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - os.write(reinterpret_cast(buffer.get()), compressedBytes); - } - else { - auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - os.write(reinterpret_cast(srcBuf), srcBytes); - } -} - -template -inline void -writeCompressedValuesSize(std::ostream& os, const T* srcBuf, Index srcCount) -{ - using compression::bloscCompressedSize; - - const size_t srcBytes = srcCount*sizeof(T); - const size_t maximumBytes = std::numeric_limits::max(); - if (srcBytes >= maximumBytes) { - OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << - maximumBytes << " bytes in voxel values.") - } - - const char* charBuffer = reinterpret_cast(srcBuf); - - // calculate voxel buffer size after compression - size_t compressedBytes = bloscCompressedSize(charBuffer, srcBytes); - - if (compressedBytes > 0) { - auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } - else { - auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed - os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); - } -} - -} // namespace io - // forward declaration namespace tree { diff --git a/openvdb/openvdb/points/PointDataIO.h b/openvdb/openvdb/points/PointDataIO.h new file mode 100644 index 0000000000..fddbaef6c4 --- /dev/null +++ b/openvdb/openvdb/points/PointDataIO.h @@ -0,0 +1,148 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED +#define OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED + + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { + + +//////////////////////////////////////// + + +namespace io +{ + +/// @brief openvdb::io::readCompressedValues specialized on PointDataIndex32 arrays to +/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit +template<> +inline void +readCompressedValues( std::istream& is, PointDataIndex32* destBuf, Index destCount, + const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/) +{ + using compression::bloscDecompress; + + const bool seek = destBuf == nullptr; + + const size_t destBytes = destCount*sizeof(PointDataIndex32); + const size_t maximumBytes = std::numeric_limits::max(); + if (destBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot read more than " << + maximumBytes << " bytes in voxel values.") + } + + uint16_t bytes16; + + const io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is); + + if (seek && meta) { + // buffer size temporarily stored in the StreamMetadata pass + // to avoid having to perform an expensive disk read for 2-bytes + bytes16 = static_cast(meta->pass()); + // seek over size of the compressed buffer + is.seekg(sizeof(uint16_t), std::ios_base::cur); + } + else { + // otherwise read from disk + is.read(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } + + if (bytes16 == std::numeric_limits::max()) { + // read or seek uncompressed data + if (seek) { + is.seekg(destBytes, std::ios_base::cur); + } + else { + is.read(reinterpret_cast(destBuf), destBytes); + } + } + else { + // read or seek uncompressed data + if (seek) { + is.seekg(int(bytes16), std::ios_base::cur); + } + else { + // decompress into the destination buffer + std::unique_ptr bloscBuffer(new char[int(bytes16)]); + is.read(bloscBuffer.get(), bytes16); + std::unique_ptr buffer = bloscDecompress( bloscBuffer.get(), + destBytes, + /*resize=*/false); + std::memcpy(destBuf, buffer.get(), destBytes); + } + } +} + +/// @brief openvdb::io::writeCompressedValues specialized on PointDataIndex32 arrays to +/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit +template<> +inline void +writeCompressedValues( std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount, + const util::NodeMask<3>& /*valueMask*/, + const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/) +{ + using compression::bloscCompress; + + const size_t srcBytes = srcCount*sizeof(PointDataIndex32); + const size_t maximumBytes = std::numeric_limits::max(); + if (srcBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << + maximumBytes << " bytes in voxel values.") + } + + const char* charBuffer = reinterpret_cast(srcBuf); + + size_t compressedBytes; + std::unique_ptr buffer = bloscCompress( charBuffer, srcBytes, + compressedBytes, /*resize=*/false); + + if (compressedBytes > 0) { + auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + os.write(reinterpret_cast(buffer.get()), compressedBytes); + } + else { + auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + os.write(reinterpret_cast(srcBuf), srcBytes); + } +} + +template +inline void +writeCompressedValuesSize(std::ostream& os, const T* srcBuf, Index srcCount) +{ + using compression::bloscCompressedSize; + + const size_t srcBytes = srcCount*sizeof(T); + const size_t maximumBytes = std::numeric_limits::max(); + if (srcBytes >= maximumBytes) { + OPENVDB_THROW(openvdb::IoError, "Cannot write more than " << + maximumBytes << " bytes in voxel values.") + } + + const char* charBuffer = reinterpret_cast(srcBuf); + + // calculate voxel buffer size after compression + size_t compressedBytes = bloscCompressedSize(charBuffer, srcBytes); + + if (compressedBytes > 0) { + auto bytes16 = static_cast(compressedBytes); // clamp to 16-bit unsigned integer + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } + else { + auto bytes16 = static_cast(maximumBytes); // max value indicates uncompressed + os.write(reinterpret_cast(&bytes16), sizeof(uint16_t)); + } +} + +} // namespace io + + +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED From 6528cb82eb0691514b60f74ac4127026b53d4ee6 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 18 Mar 2026 14:18:25 -0700 Subject: [PATCH 44/80] Remove support for multi-pass I/O except PointDataLeafNode Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 30 +++- openvdb/openvdb/io/io.h | 4 +- openvdb/openvdb/unittest/TestFile.cc | 240 --------------------------- 3 files changed, 27 insertions(+), 247 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index b364e9efc0..b2f1dd8ecd 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -1181,14 +1181,34 @@ struct TreeAdapter > //////////////////////////////////////// +namespace points { + +template class PointDataLeafNode; + +/// @brief Type trait that evaluates to true only for @c PointDataLeafNode instantiations. +template +struct IsPointDataLeafNode : std::false_type {}; + +template +struct IsPointDataLeafNode> : std::true_type {}; + +} // namespace points + + /// @brief Metafunction that specifies whether a given leaf node, tree, or grid type -/// requires multiple passes to read and write voxel data -/// @details Multi-pass I/O allows one to optimize the data layout of leaf nodes -/// for certain access patterns during delayed loading. -/// @sa io::MultiPass +/// requires multiple passes to read and write voxel data. +/// @details Multi-pass I/O allows leaf nodes to optimize their serialization layout +/// for delayed-load access patterns. Only @c PointDataLeafNode supports multi-pass I/O. +/// Defining a custom leaf node that inherits @c io::MultiPass is no longer permitted. +/// @sa points::IsPointDataLeafNode template struct HasMultiPassIO { - static const bool value = std::is_base_of::value; + static_assert( + !std::is_base_of::value + || points::IsPointDataLeafNode::value, + "Only PointDataLeafNode may inherit from io::MultiPass; " + "use points::IsPointDataLeafNode to test for multi-pass I/O support."); + static const bool value = points::IsPointDataLeafNode::value; }; // Partial specialization for Tree types diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index f2ac342ff7..96d2408eb3 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -116,8 +116,8 @@ std::ostream& operator<<(std::ostream&, const StreamMetadata::AuxDataMap&); //////////////////////////////////////// -/// @brief Leaf nodes that require multi-pass I/O must inherit from this struct. -/// @sa Grid::hasMultiPassIO() +/// @brief Tag for multi-pass I/O. Only points::PointDataLeafNode may inherit from this. +/// @sa Grid::hasMultiPassIO(), points::IsPointDataLeafNode struct MultiPass {}; diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index fad51e9567..130947fdf1 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -1522,246 +1522,6 @@ TEST_F(TestFile, testReadClippedGrid) //////////////////////////////////////// -namespace { - -template struct MultiPassLeafNode; // forward declaration - -// Dummy value type -using MultiPassValue = openvdb::PointIndex; - -// Tree configured to match the default OpenVDB configuration -using MultiPassTree = openvdb::tree::Tree< - openvdb::tree::RootNode< - openvdb::tree::InternalNode< - openvdb::tree::InternalNode< - MultiPassLeafNode, 4>, 5>>>; - -using MultiPassGrid = openvdb::Grid; - - -template -struct MultiPassLeafNode: public openvdb::tree::LeafNode, openvdb::io::MultiPass -{ - // The following had to be copied from the LeafNode class - // to make the derived class compatible with the tree structure. - - using LeafNodeType = MultiPassLeafNode; - using Ptr = openvdb::SharedPtr; - using BaseLeaf = openvdb::tree::LeafNode; - using NodeMaskType = openvdb::util::NodeMask; - using ValueType = T; - using ValueOnCIter = typename BaseLeaf::template ValueIter; - using ChildOnIter = typename BaseLeaf::template ChildIter; - using ChildOnCIter = typename BaseLeaf::template ChildIter< - typename NodeMaskType::OnIterator, const MultiPassLeafNode, typename BaseLeaf::ChildOn>; - - MultiPassLeafNode(const openvdb::Coord& coords, const T& value, bool active = false) - : BaseLeaf(coords, value, active) {} - MultiPassLeafNode(openvdb::PartialCreate, const openvdb::Coord& coords, const T& value, - bool active = false): BaseLeaf(openvdb::PartialCreate(), coords, value, active) {} - MultiPassLeafNode(const MultiPassLeafNode& rhs): BaseLeaf(rhs) {} - - ValueOnCIter cbeginValueOn() const { return ValueOnCIter(this->getValueMask().beginOn(),this); } - ChildOnCIter cbeginChildOn() const { return ChildOnCIter(this->getValueMask().endOn(), this); } - ChildOnIter beginChildOn() { return ChildOnIter(this->getValueMask().endOn(), this); } - - // Methods in use for reading and writing multiple buffers - - void readBuffers(std::istream& is, const openvdb::CoordBBox&, bool fromHalf = false) - { - this->readBuffers(is, fromHalf); - } - - void readBuffers(std::istream& is, bool /*fromHalf*/ = false) - { - const openvdb::io::StreamMetadata::Ptr meta = openvdb::io::getStreamMetadataPtr(is); - if (!meta) { - OPENVDB_THROW(openvdb::IoError, - "Cannot write out a MultiBufferLeaf without StreamMetadata."); - } - - // clamp pass to 16-bit integer - const uint32_t pass(static_cast(meta->pass())); - - // Read in the stored pass number. - uint32_t readPass; - is.read(reinterpret_cast(&readPass), sizeof(uint32_t)); - EXPECT_EQ(pass, readPass); - // Record the pass number. - mReadPasses.push_back(readPass); - - if (pass == 0) { - // Read in the node's origin. - openvdb::Coord origin; - is.read(reinterpret_cast(&origin), sizeof(openvdb::Coord)); - EXPECT_EQ(origin, this->origin()); - } - } - - void writeBuffers(std::ostream& os, bool /*toHalf*/ = false) const - { - const openvdb::io::StreamMetadata::Ptr meta = openvdb::io::getStreamMetadataPtr(os); - if (!meta) { - OPENVDB_THROW(openvdb::IoError, - "Cannot read in a MultiBufferLeaf without StreamMetadata."); - } - - // clamp pass to 16-bit integer - const uint32_t pass(static_cast(meta->pass())); - - // Leaf traversal analysis deduces the number of passes to perform for this leaf - // then updates the leaf traversal value to ensure all passes will be written. - if (meta->countingPasses()) { - if (mNumPasses > pass) meta->setPass(mNumPasses); - return; - } - - // Record the pass number. - EXPECT_TRUE(mWritePassesPtr); - const_cast&>(*mWritePassesPtr).push_back(pass); - - // Write out the pass number. - os.write(reinterpret_cast(&pass), sizeof(uint32_t)); - if (pass == 0) { - // Write out the node's origin and the pass number. - const auto origin = this->origin(); - os.write(reinterpret_cast(&origin), sizeof(openvdb::Coord)); - } - } - - - uint32_t mNumPasses = 0; - // Pointer to external vector in which to record passes as they are written - std::vector* mWritePassesPtr = nullptr; - // Vector in which to record passes as they are read - // (this needs to be internal, because leaf nodes are constructed as a grid is read) - std::vector mReadPasses; -}; // struct MultiPassLeafNode - -} // anonymous namespace - - -TEST_F(TestFile, testMultiPassIO) -{ - using namespace openvdb; - - openvdb::initialize(); - MultiPassGrid::registerGrid(); - - // Create a multi-buffer grid. - const MultiPassGrid::Ptr grid = openvdb::createGrid(); - grid->setName("test"); - grid->setTransform(math::Transform::createLinearTransform(1.0)); - MultiPassGrid::TreeType& tree = grid->tree(); - tree.setValue(Coord(0, 0, 0), 5); - tree.setValue(Coord(0, 10, 0), 5); - EXPECT_EQ(2, int(tree.leafCount())); - - const GridPtrVec grids{grid}; - - // Vector in which to record pass numbers (to ensure blocked ordering) - std::vector writePasses; - { - // Specify the required number of I/O passes for each leaf node. - MultiPassGrid::TreeType::LeafIter leafIter = tree.beginLeaf(); - leafIter->mNumPasses = 3; - leafIter->mWritePassesPtr = &writePasses; - ++leafIter; - leafIter->mNumPasses = 2; - leafIter->mWritePassesPtr = &writePasses; - } - - const char* filename = "testMultiPassIO.vdb"; - SharedPtr scopedFile(filename, ::remove); - { - // Verify that passes are written to a file in the correct order. - io::File(filename).write(grids); - EXPECT_EQ(6, int(writePasses.size())); - EXPECT_EQ(0, writePasses[0]); // leaf 0 - EXPECT_EQ(0, writePasses[1]); // leaf 1 - EXPECT_EQ(1, writePasses[2]); // leaf 0 - EXPECT_EQ(1, writePasses[3]); // leaf 1 - EXPECT_EQ(2, writePasses[4]); // leaf 0 - EXPECT_EQ(2, writePasses[5]); // leaf 1 - } - { - // Verify that passes are read in the correct order. - io::File file(filename); - file.open(); - const auto newGrid = GridBase::grid(file.readGrid("test")); - - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - } - { - // Verify that when using multi-pass and bbox clipping that each leaf node - // is still being read before being clipped - io::File file(filename); - file.open(); - const auto newGrid = GridBase::grid( - file.readGrid("test", BBoxd(Vec3d(0), Vec3d(1)))); - EXPECT_EQ(Index64(1), newGrid->tree().leafCount()); - - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_TRUE(!leafIter); // second leaf node has now been clipped - } - - // Clear the pass data. - writePasses.clear(); - - { - // Verify that passes are written to and read from a non-seekable stream - // in the correct order. - std::ostringstream ostr(std::ios_base::binary); - io::Stream(ostr).write(grids); - - EXPECT_EQ(6, int(writePasses.size())); - EXPECT_EQ(0, writePasses[0]); // leaf 0 - EXPECT_EQ(0, writePasses[1]); // leaf 1 - EXPECT_EQ(1, writePasses[2]); // leaf 0 - EXPECT_EQ(1, writePasses[3]); // leaf 1 - EXPECT_EQ(2, writePasses[4]); // leaf 0 - EXPECT_EQ(2, writePasses[5]); // leaf 1 - - std::istringstream is(ostr.str(), std::ios_base::binary); - io::Stream strm(is); - const auto streamedGrids = strm.getGrids(); - EXPECT_EQ(1, int(streamedGrids->size())); - - const auto newGrid = gridPtrCast(*streamedGrids->begin()); - EXPECT_TRUE(bool(newGrid)); - auto leafIter = newGrid->tree().beginLeaf(); - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - ++leafIter; - EXPECT_EQ(3, int(leafIter->mReadPasses.size())); - EXPECT_EQ(0, leafIter->mReadPasses[0]); - EXPECT_EQ(1, leafIter->mReadPasses[1]); - EXPECT_EQ(2, leafIter->mReadPasses[2]); - } -} - - -//////////////////////////////////////// - - TEST_F(TestFile, testHasGrid) { using namespace openvdb; From 0b95d2e9ca190858b4b7fb23b963860f138f8eeb Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 18 Mar 2026 14:16:25 -0700 Subject: [PATCH 45/80] Remove AttributeArray out-of-core atomic Signed-off-by: Dan Bailey --- openvdb/openvdb/points/AttributeArray.cc | 2 -- openvdb/openvdb/points/AttributeArray.h | 8 +++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/openvdb/openvdb/points/AttributeArray.cc b/openvdb/openvdb/points/AttributeArray.cc index 7d83051b52..19e6d14436 100644 --- a/openvdb/openvdb/points/AttributeArray.cc +++ b/openvdb/openvdb/points/AttributeArray.cc @@ -61,7 +61,6 @@ AttributeArray::AttributeArray(const AttributeArray& rhs, const tbb::spin_mutex: : mIsUniform(rhs.mIsUniform) , mFlags(rhs.mFlags) , mUsePagedRead(rhs.mUsePagedRead) - , mOutOfCore(rhs.mOutOfCore.load()) , mPageHandle() { if (mFlags & PARTIALREAD) mCompressedBytes = rhs.mCompressedBytes; @@ -78,7 +77,6 @@ AttributeArray::operator=(const AttributeArray& rhs) mIsUniform = rhs.mIsUniform; mFlags = rhs.mFlags; mUsePagedRead = rhs.mUsePagedRead; - mOutOfCore.store(rhs.mOutOfCore); if (mFlags & PARTIALREAD) mCompressedBytes = rhs.mCompressedBytes; else if (rhs.mPageHandle) mPageHandle = rhs.mPageHandle->copy(); else mPageHandle.reset(); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index bbfba7b156..3ad2d3a48d 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -363,10 +363,12 @@ class OPENVDB_API AttributeArray mutable tbb::spin_mutex mMutex; uint8_t mFlags = 0; uint8_t mUsePagedRead = 0; +#if OPENVDB_ABI_VERSION_NUMBER < 14 std::atomic mOutOfCore{0}; // interpreted as bool +#endif /// used for out-of-core, paged reading union { - compression::PageHandle::Ptr mPageHandle; + std::unique_ptr mPageHandle; size_t mCompressedBytes; }; }; // class AttributeArray @@ -1535,8 +1537,6 @@ TypedAttributeArray::readBuffers(std::istream& is) OPENVDB_THROW(IoError, "Cannot read paged AttributeArray buffers."); } - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); uint8_t bloscCompressed(0); @@ -1586,8 +1586,6 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu OPENVDB_ASSERT(mPageHandle); - tbb::spin_mutex::scoped_lock lock(mMutex); - this->deallocate(); is.read(mPageHandle, std::streamsize(mPageHandle->size()), false); From a9a6cff2ab5a8c248c0319ee2d2545c20ac28157 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 27 Mar 2026 14:36:30 -0700 Subject: [PATCH 46/80] Rename MultiPass tag class to PointDataGridMultiPass Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 6 +++--- openvdb/openvdb/io/io.h | 2 +- openvdb/openvdb/points/PointDataGrid.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index b2f1dd8ecd..41f6e58501 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -1199,14 +1199,14 @@ struct IsPointDataLeafNode> : std::true_type {}; /// requires multiple passes to read and write voxel data. /// @details Multi-pass I/O allows leaf nodes to optimize their serialization layout /// for delayed-load access patterns. Only @c PointDataLeafNode supports multi-pass I/O. -/// Defining a custom leaf node that inherits @c io::MultiPass is no longer permitted. +/// Defining a custom leaf node that inherits @c io::PointDataGridMultiPass is no longer permitted. /// @sa points::IsPointDataLeafNode template struct HasMultiPassIO { static_assert( - !std::is_base_of::value + !std::is_base_of::value || points::IsPointDataLeafNode::value, - "Only PointDataLeafNode may inherit from io::MultiPass; " + "Only PointDataLeafNode may inherit from io::PointDataGridMultiPass; " "use points::IsPointDataLeafNode to test for multi-pass I/O support."); static const bool value = points::IsPointDataLeafNode::value; }; diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index 96d2408eb3..13a9134012 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -118,7 +118,7 @@ std::ostream& operator<<(std::ostream&, const StreamMetadata::AuxDataMap&); /// @brief Tag for multi-pass I/O. Only points::PointDataLeafNode may inherit from this. /// @sa Grid::hasMultiPassIO(), points::IsPointDataLeafNode -struct MultiPass {}; +struct PointDataGridMultiPass {}; //////////////////////////////////////// diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index 946b5764c3..f186717799 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -107,7 +107,7 @@ prefetch(PointDataTreeT&, bool /*position*/ = true, bool /*otherAttributes*/ = t template -class PointDataLeafNode : public tree::LeafNode, io::MultiPass { +class PointDataLeafNode : public tree::LeafNode, io::PointDataGridMultiPass { public: using LeafNodeType = PointDataLeafNode; From 646cdaa609ae204d4095d1823a901a8c32d182b8 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 20 Apr 2026 11:25:29 -0700 Subject: [PATCH 47/80] Add PointDataCodec Signed-off-by: Dan Bailey --- openvdb/openvdb/CMakeLists.txt | 1 + openvdb/openvdb/codecs/PointDataCodec.h | 449 ++++++++++++++++++++ openvdb/openvdb/io/Codec.cc | 2 + openvdb/openvdb/points/AttributeArray.h | 40 ++ openvdb/openvdb/points/StreamCompression.cc | 32 ++ openvdb/openvdb/points/StreamCompression.h | 8 + openvdb/openvdb/unittest/TestPointCodec.cc | 400 +++++++++++++++++ 7 files changed, 932 insertions(+) create mode 100644 openvdb/openvdb/codecs/PointDataCodec.h diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index 97317b1fb7..19172ca380 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -367,6 +367,7 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES codecs/BoolCodec.h + codecs/PointDataCodec.h codecs/PointIndexCodec.h codecs/ScalarCodec.h codecs/TopologyCodec.h diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h new file mode 100644 index 0000000000..e5cc550fe4 --- /dev/null +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -0,0 +1,449 @@ +// Copyright Contributors to the OpenVDB Project +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED +#define OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED + +#include + +#include + +#include +#include +#include + +#include "ScalarLeafCodec.h" +#include "TopologyCodec.h" + +#include +#include + +namespace openvdb { +OPENVDB_USE_VERSION_NAMESPACE +namespace OPENVDB_VERSION_NAME { +namespace codecs { +namespace internal { + +/// Look up an existing paged stream or create a new one for the given attribute index +template +inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream( + std::map& pagedStreams, Index attributeIndex) +{ + auto it = pagedStreams.find(attributeIndex); + if (it != pagedStreams.end()) return it->second.get(); + using PagedStreamT = typename PagedStreamPtrT::element_type; + pagedStreams[attributeIndex] = std::make_shared(); + return pagedStreams[attributeIndex].get(); +} + +//////////////////////////////////////// +// Read-side functions + +template +inline void readPointDataVoxelSizes(const std::vector& leaves, + std::istream& is, std::map& voxelBufferSizes) +{ + for (auto* leaf : leaves) { + uint16_t voxelBufferSize; + is.read(reinterpret_cast(&voxelBufferSize), sizeof(uint16_t)); + voxelBufferSizes[leaf->origin()] = voxelBufferSize; + } +} + +template +inline void readPointDataDescriptors(const std::vector& leaves, + std::istream& is) +{ + points::AttributeSet::Descriptor::Ptr sharedDescriptor; + for (auto* leaf : leaves) { + points::AttributeSet::UniquePtr attrSet = leaf->stealAttributeSet(); + if (sharedDescriptor) { + // Reuse shared descriptor from first leaf + attrSet->resetDescriptor(sharedDescriptor, /*allowMismatchingDescriptors=*/true); + } + else { + uint8_t header; + is.read(reinterpret_cast(&header), sizeof(uint8_t)); + attrSet->readDescriptor(is); + if (header & uint8_t(1)) { + // Store descriptor for subsequent leaves + sharedDescriptor = attrSet->descriptorPtr(); + } + // a forwards-compatibility mechanism for future use, + // if a 0x2 bit is set, read and skip over a specific number of bytes + if (header & uint8_t(2)) { + uint64_t bytesToSkip; + is.read(reinterpret_cast(&bytesToSkip), sizeof(uint64_t)); + if (bytesToSkip > uint64_t(0)) { + std::vector tempData(bytesToSkip); + is.read(reinterpret_cast(&tempData[0]), bytesToSkip); + } + } + // this reader is only able to read headers with 0x1 and 0x2 bits set + if (header > uint8_t(3)) { + OPENVDB_THROW(IoError, "Unrecognised header flags in PointDataLeafNode"); + } + } + attrSet->readMetadata(is); + leaf->replaceAttributeSet(attrSet.release(), /*allowMismatchingDescriptors=*/true); + } +} + +template +inline void readPointDataAttributeSizes(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(true); + array->readPagedBuffers(*pagedStream); + } + } +} + +template +inline void readPointDataVoxelData(const std::vector& leaves, + std::istream& is, bool saveFloatAsHalf, + const typename LeafT::ValueType& background, + const std::map& voxelBufferSizes) +{ + (void) voxelBufferSizes; + using BaseLeaf = typename LeafT::BaseLeaf; + for (auto* leaf : leaves) { + OPENVDB_ASSERT(voxelBufferSizes.find(leaf->origin()) != voxelBufferSizes.end()); + BaseLeaf& baseLeaf = static_cast(*leaf); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background); + } +} + +template +inline void readPointDataAttributeData(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(false); + array->readPagedBuffers(*pagedStream); + } + } +} + +template +inline void skipPointDataAttributeSizes(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(true); + array->skipPagedBuffers(*pagedStream); + } + } +} + +template +inline void skipPointDataAttributeData(const std::vector& leaves, + std::istream& is, Index attributeIndex, + std::map& pagedStreams) +{ + for (auto* leaf : leaves) { + points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + &leaf->attributeArray(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setInputStream(is); + pagedStream->setSizeOnly(false); + array->skipPagedBuffers(*pagedStream); + } + } +} + +//////////////////////////////////////// +// Write-side functions + +template +inline Index countPointDataPasses(const std::vector& leaves) +{ + Index maxRequiredPasses = 0; + for (const auto* leaf : leaves) { + const Index requiredPasses = leaf->buffers(); + if (requiredPasses > maxRequiredPasses) { + maxRequiredPasses = requiredPasses; + } + } + return maxRequiredPasses; +} + +template +inline void writePointDataVoxelSizes(const std::vector& leaves, + std::ostream& os, bool& matching, + points::AttributeSet::Descriptor::Ptr& sharedDescriptor) +{ + bool descriptorChecked = false; + matching = true; + for (const auto* leaf : leaves) { + io::writeCompressedValuesSize(os, leaf->buffer().data(), LeafT::SIZE); + + // Track descriptor matching + const auto& descriptor = leaf->attributeSet().descriptorPtr(); + if (!descriptorChecked) { + // First leaf - store descriptor + descriptorChecked = true; + sharedDescriptor = descriptor; + } + else if (matching && *sharedDescriptor != *descriptor) { + matching = false; + } + } +} + +template +inline void writePointDataDescriptors(const std::vector& leaves, + std::ostream& os, bool matching, + const points::AttributeSet::Descriptor::Ptr&) +{ + bool firstWrite = true; + for (const auto* leaf : leaves) { + const points::AttributeSet& attributeSet = leaf->attributeSet(); + if (matching) { + // Shared descriptor - only write on first leaf + if (firstWrite) { + firstWrite = false; + uint8_t header(1); + os.write(reinterpret_cast(&header), sizeof(uint8_t)); + attributeSet.writeDescriptor(os, /*transient=*/false); + } + } + else { + // Non-shared descriptor - write on every leaf + uint8_t header(0); + os.write(reinterpret_cast(&header), sizeof(uint8_t)); + attributeSet.writeDescriptor(os, /*transient=*/false); + } + attributeSet.writeMetadata(os, /*transient=*/false, /*paged=*/true); + } +} + +template +inline void writePointDataAttributeSizes(const std::vector& leaves, + std::ostream& os, Index attributeIndex) +{ + std::map pagedStreams; + for (const auto* leaf : leaves) { + const points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + leaf->attributeSet().getConst(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setOutputStream(os); + pagedStream->setSizeOnly(true); + array->writePagedBuffers(*pagedStream, /*outputTransient*/false); + } + } + // Flush paged streams to write any remaining buffered page headers + for (auto& pair : pagedStreams) { + pair.second->flush(); + } +} + +template +inline void writePointDataVoxelData(const std::vector& leaves, + std::ostream& os, bool saveFloatAsHalf) +{ + using BaseLeaf = typename LeafT::BaseLeaf; + for (const auto* leaf : leaves) { + const BaseLeaf& baseLeaf = static_cast(*leaf); + writeScalarLeafBuffers(baseLeaf, os, saveFloatAsHalf); + } +} + +template +inline void writePointDataAttributeData(const std::vector& leaves, + std::ostream& os, Index attributeIndex) +{ + std::map pagedStreams; + for (const auto* leaf : leaves) { + const points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ? + leaf->attributeSet().getConst(attributeIndex) : nullptr; + if (array) { + auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex); + pagedStream->setOutputStream(os); + pagedStream->setSizeOnly(false); + array->writePagedBuffers(*pagedStream, /*outputTransient*/false); + } + } + // Flush paged streams to write any remaining buffered page data + for (auto& pair : pagedStreams) { + pair.second->flush(); + } +} + +} // namespace internal + +/// Per-grid-type codec-specific options for PointDataCodec +/// Contains point attribute filtering options +struct OPENVDB_API PointDataCodecTypeData : public io::ReadTypedOptions +{ + // Point Attribute Options - which attributes to read + std::vector pointAttributeNames; +}; // struct PointDataCodecTypeData + +template +struct PointDataCodec : public TopologyCodec +{ + using Ptr = std::unique_ptr>; + + ~PointDataCodec() noexcept = default; + + static inline std::string name() { return GridT::gridType(); } + + void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + { + GridT& grid = static_cast(*data.grid); + + std::vector pointAttributeNames; + + // Look up point-specific options if provided + auto it = options.typeData.find(name()); + if (it != options.typeData.end()) { + auto& pointTypeData = io::ReadTypedOptions::cast(it->second); + pointAttributeNames = pointTypeData.pointAttributeNames; + } + + io::checkFormatVersion(is); + + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + auto& tree = grid.tree(); + tree.clearAllAccessors(); + + uint16_t numPasses = 1; + is.read(reinterpret_cast(&numPasses), sizeof(uint16_t)); + const Index attributes = (numPasses - 4) / 2; + + using LeafT = typename GridT::TreeType::LeafNodeType; + std::vector leaves; + tree.getNodes(leaves); + + // Pass 0: read voxel data sizes + std::map voxelBufferSizes; + internal::readPointDataVoxelSizes(leaves, is, voxelBufferSizes); + + // Pass 1: read descriptor and attribute metadata + internal::readPointDataDescriptors(leaves, is); + + // Build set of attribute indices to skip based on pointAttributeNames. + // An empty pointAttributeNames means no filtering (read all attributes). + std::set skipIndices; + if (!pointAttributeNames.empty() && !leaves.empty()) { + const auto& nameMap = leaves[0]->attributeSet().descriptor().map(); + const std::set wantedNames( + pointAttributeNames.begin(), + pointAttributeNames.end()); + for (const auto& namePos : nameMap) { + if (wantedNames.find(namePos.first) == wantedNames.end()) { + skipIndices.insert(static_cast(namePos.second)); + } + } + } + + // Passes 2..N+1: read attribute buffer sizes + std::map pagedStreams; + for (Index i = 0; i < attributes; ++i) { + if (skipIndices.count(i)) { + internal::skipPointDataAttributeSizes(leaves, is, i, pagedStreams); + } else { + internal::readPointDataAttributeSizes(leaves, is, i, pagedStreams); + } + } + + // Pass N+2: read voxel data + internal::readPointDataVoxelData(leaves, is, saveFloatAsHalf, + tree.background(), voxelBufferSizes); + + // Passes N+3..2N+2: read attribute data buffers + for (Index i = 0; i < attributes; ++i) { + if (skipIndices.count(i)) { + internal::skipPointDataAttributeData(leaves, is, i, pagedStreams); + } else { + internal::readPointDataAttributeData(leaves, is, i, pagedStreams); + } + } + + // Drop skipped attributes from each leaf's AttributeSet + if (!skipIndices.empty() && !leaves.empty()) { + const std::vector dropPositions(skipIndices.begin(), skipIndices.end()); + auto filteredDescriptor = + leaves[0]->attributeSet().descriptorPtr()->duplicateDrop(dropPositions); + for (auto* leaf : leaves) { + leaf->dropAttributes(dropPositions, + leaf->attributeSet().descriptor(), filteredDescriptor); + } + } + + // PointDataGrid uses multiple passes, so clip after reading + // the buffers if bbox is not infinite. + if (options.clipBBox.isSorted()) { + CoordBBox indexBBox = + grid.constTransform().worldToIndexNodeCentered(options.clipBBox); + grid.tree().root().clip(indexBBox); + } + } + + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + { + const GridT& grid = static_cast(gridBase); + bool saveFloatAsHalf = grid.saveFloatAsHalf(); + + using LeafT = typename GridT::TreeType::LeafNodeType; + std::vector leaves; + grid.tree().getNodes(leaves); + + // Determine how many leaf buffer passes are required for this grid + uint16_t numPasses = + static_cast(internal::countPointDataPasses(leaves)); + os.write(reinterpret_cast(&numPasses), sizeof(uint16_t)); + + const Index attributes = (numPasses - 4) / 2; + + // Pass 0: write voxel data sizes + descriptor tracking + bool matching = true; + points::AttributeSet::Descriptor::Ptr sharedDescriptor; + internal::writePointDataVoxelSizes(leaves, os, matching, sharedDescriptor); + + // Pass 1: write descriptor and attribute metadata + internal::writePointDataDescriptors(leaves, os, matching, sharedDescriptor); + + // Passes 2..N+1: write attribute buffer sizes (page headers) + for (Index i = 0; i < attributes; ++i) { + internal::writePointDataAttributeSizes(leaves, os, i); + } + + // Pass N+2: write voxel data + internal::writePointDataVoxelData(leaves, os, saveFloatAsHalf); + + // Passes N+3..2N+2: write attribute data buffers (page data) + for (Index i = 0; i < attributes; ++i) { + internal::writePointDataAttributeData(leaves, os, i); + } + } +}; // struct PointDataCodec + +} // namespace codecs +} // namespace OPENVDB_VERSION_NAME +} // namespace openvdb + +#endif // OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index 3e437fab17..4873f779aa 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -60,6 +61,7 @@ void initialize() CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); CodecRegistry::registerCodec>(); + CodecRegistry::registerCodec>(); // register the plugin that converts from scalar to mask/bool NumericGridTypes::foreach(); diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index 3ad2d3a48d..23ff8f8149 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -308,6 +308,9 @@ class OPENVDB_API AttributeArray /// Read attribute buffers from a paged stream. virtual void readPagedBuffers(compression::PagedInputStream&) = 0; + /// Skip attribute buffers in a paged stream without reading or + /// allocating any data. + void skipPagedBuffers(compression::PagedInputStream&); /// Write attribute buffers to a paged stream. /// @param outputTransient if true, write out transient attributes virtual void writePagedBuffers(compression::PagedOutputStream&, bool outputTransient) const = 0; @@ -1599,6 +1602,43 @@ TypedAttributeArray::readPagedBuffers(compression::PagedInpu } +inline void +AttributeArray::skipPagedBuffers(compression::PagedInputStream& is) +{ + if (!mUsePagedRead) { + if (!is.sizeOnly()) { + // for non-paged data, seek past the raw data in the stream + std::istream& inputStream = is.getInputStream(); + uint8_t bloscCompressed(0); + if (!mIsUniform) inputStream.read(reinterpret_cast(&bloscCompressed), sizeof(uint8_t)); + inputStream.seekg(mCompressedBytes, std::ios_base::cur); + mCompressedBytes = 0; + mFlags = static_cast(mFlags & ~PARTIALREAD); + } + return; + } + + if (is.sizeOnly()) + { + size_t compressedBytes(mCompressedBytes); + mCompressedBytes = 0; + mFlags = static_cast(mFlags & ~PARTIALREAD); + OPENVDB_ASSERT(!mPageHandle); + mPageHandle = is.createHandle(compressedBytes); + return; + } + + OPENVDB_ASSERT(mPageHandle); + + is.skip(mPageHandle, std::streamsize(mPageHandle->size())); + mPageHandle.reset(); + + // clear page state + + mUsePagedRead = 0; +} + + template void TypedAttributeArray::write(std::ostream& os) const diff --git a/openvdb/openvdb/points/StreamCompression.cc b/openvdb/openvdb/points/StreamCompression.cc index 5760f3b4ae..f18d016463 100644 --- a/openvdb/openvdb/points/StreamCompression.cc +++ b/openvdb/openvdb/points/StreamCompression.cc @@ -338,6 +338,21 @@ Page::readBuffers(std::istream&is, bool delayed) } +void +Page::skipBuffers(std::istream& is) +{ + OPENVDB_ASSERT(mInfo); + + bool isCompressed = mInfo->compressedBytes > 0; + std::streamsize bytes = isCompressed ? + mInfo->compressedBytes : -mInfo->compressedBytes; + + is.seekg(bytes, std::ios_base::cur); + + mInfo.reset(); +} + + void Page::copy(const std::unique_ptr& temp, int pageSize) { @@ -438,6 +453,23 @@ PagedInputStream::read(PageHandle::Ptr& pageHandle, std::streamsize n, bool dela } +void +PagedInputStream::skip(PageHandle::Ptr& pageHandle, std::streamsize n) +{ + OPENVDB_ASSERT(mByteIndex <= mUncompressedBytes); + + Page& page = pageHandle->page(); + + if (mByteIndex == mUncompressedBytes) { + mUncompressedBytes = static_cast(page.uncompressedBytes()); + page.skipBuffers(*mIs); + mByteIndex = 0; + } + + mByteIndex += int(n); +} + + //////////////////////////////////////// diff --git a/openvdb/openvdb/points/StreamCompression.h b/openvdb/openvdb/points/StreamCompression.h index d983ea2404..2efbcda35c 100644 --- a/openvdb/openvdb/points/StreamCompression.h +++ b/openvdb/openvdb/points/StreamCompression.h @@ -142,6 +142,10 @@ class OPENVDB_API Page /// pointers will be stored to load the data lazily. void readBuffers(std::istream&, bool delayed); + /// @brief Skip the Page buffers by seeking past the compressed data + /// without reading or decompressing it. + void skipBuffers(std::istream&); + OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.") bool isOutOfCore() const { return false; } @@ -222,6 +226,10 @@ class OPENVDB_API PagedInputStream /// an immediate read of the data. void read(PageHandle::Ptr& pageHandle, std::streamsize n, bool delayed = true); + /// @brief Skip past the page data referenced by @a pageHandle without + /// reading or decompressing it. + void skip(PageHandle::Ptr& pageHandle, std::streamsize n); + private: int mByteIndex = 0; int mUncompressedBytes = 0; diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 2b8911419e..81f8ff2358 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "util.h" // for unittest_util::genPoints @@ -145,3 +146,402 @@ TEST_F(TestPointCodec, testPointIndexCodecIO) CodecRegistry::clear(); std::remove(codecPath.c_str()); } + +TEST_F(TestPointCodec, testPointDataCodecIO) +{ + using namespace openvdb; + using namespace openvdb::io; + using namespace openvdb::points; + using PointDataTree = PointDataGrid::TreeType; + + openvdb::initialize(); + CodecRegistry::clear(); + + // Helper: compare P attribute values leaf-by-leaf between two PointDataGrids + auto comparePositions = [](const PointDataGrid& a, const PointDataGrid& b) { + auto aIt = a.tree().cbeginLeaf(); + auto bIt = b.tree().cbeginLeaf(); + for (; aIt && bIt; ++aIt, ++bIt) { + EXPECT_EQ(aIt->pointCount(), bIt->pointCount()); + AttributeHandle aH(aIt->constAttributeArray("P")); + AttributeHandle bH(bIt->constAttributeArray("P")); + for (Index i = 0; i < aIt->pointCount(); ++i) { + const Vec3f av = aH.get(i); + const Vec3f bv = bH.get(i); + EXPECT_NEAR(av.x(), bv.x(), 1e-6f); + EXPECT_NEAR(av.y(), bv.y(), 1e-6f); + EXPECT_NEAR(av.z(), bv.z(), 1e-6f); + } + } + EXPECT_TRUE(!aIt && !bIt); + }; + + // ----------------------------------------------------------------------- + // Section A: Positions only + // ----------------------------------------------------------------------- + { + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid(positions, *transform); + srcGrid->setName("pdg_positions"); + + const std::string rawPath = "testPDG_A_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("pdg_positions")); + f.close(); + } + ASSERT_TRUE(rawGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(rawGrid->tree())); + + PointDataGrid::Ptr rawTopo; + { + io::File f(rawPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_positions")); + rawTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(rawTopo); + EXPECT_EQ(rawTopo->activeVoxelCount(), Index64(4)); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPDG_A_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType())); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("pdg_positions")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + // Phase 3: compare src vs codec + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(codecGrid->tree())); + comparePositions(*srcGrid, *codecGrid); + + // Phase 4: TopologyOnly read + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + PointDataGrid::Ptr codecTopo; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_positions", topoOpts)); + codecTopo = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecTopo); + EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0)); + EXPECT_TRUE(codecTopo->tree().leafCount() == 0); + + std::remove(codecPath.c_str()); + } + + // ----------------------------------------------------------------------- + // Section B: Multiple attributes + // ----------------------------------------------------------------------- + { + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + const std::vector velocities = { + Vec3f(1.0f, 0.0f, 0.0f), + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(0.0f, 0.0f, 1.0f), + Vec3f(1.0f, 1.0f, 0.5f) + }; + const std::vector ids = {0, 1, 2, 3}; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointAttributeVector posWrapper(positions); + tools::PointIndexGrid::Ptr pointIndexGrid = + tools::createPointIndexGrid(posWrapper, *transform); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid( + *pointIndexGrid, posWrapper, *transform); + srcGrid->setName("pdg_multi"); + + PointDataTree& tree = srcGrid->tree(); + tools::PointIndexTree& indexTree = pointIndexGrid->tree(); + + appendAttribute(tree, "velocity"); + populateAttribute>( + tree, indexTree, "velocity", + PointAttributeVector(velocities)); + + appendAttribute(tree, "id"); + populateAttribute>( + tree, indexTree, "id", + PointAttributeVector(ids)); + + // Verify attribute count on src grid (P, velocity, id) + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(3)); + } + + CodecRegistry::clear(); + + const std::string rawPath = "testPDG_B_raw.vdb"; + + // Phase 1: write/read without codec + { + io::File f(rawPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr rawGrid; + { + io::File f(rawPath); + f.open(); + rawGrid = gridPtrCast(f.readGrid("pdg_multi")); + f.close(); + } + ASSERT_TRUE(rawGrid); + + std::remove(rawPath.c_str()); + + const std::string codecPath = "testPDG_B_codec.vdb"; + + // Phase 2: register codec, write/read with codec + io::internal::initialize(); + + { + io::File f(codecPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + codecGrid = gridPtrCast(f.readGrid("pdg_multi")); + f.close(); + } + ASSERT_TRUE(codecGrid); + + EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(codecGrid->tree())); + + // Verify attribute count on codec grid + { + auto leafIt = codecGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(3)); + } + + // Compare all three attributes leaf-by-leaf + { + auto srcIt = srcGrid->tree().cbeginLeaf(); + auto codecIt = codecGrid->tree().cbeginLeaf(); + for (; srcIt && codecIt; ++srcIt, ++codecIt) { + EXPECT_EQ(srcIt->pointCount(), codecIt->pointCount()); + AttributeHandle srcP(srcIt->constAttributeArray("P")); + AttributeHandle codecP(codecIt->constAttributeArray("P")); + AttributeHandle srcVel(srcIt->constAttributeArray("velocity")); + AttributeHandle codecVel(codecIt->constAttributeArray("velocity")); + AttributeHandle srcId(srcIt->constAttributeArray("id")); + AttributeHandle codecId(codecIt->constAttributeArray("id")); + for (Index i = 0; i < srcIt->pointCount(); ++i) { + const Vec3f rp = srcP.get(i); + const Vec3f cp = codecP.get(i); + EXPECT_NEAR(rp.x(), cp.x(), 1e-6f); + EXPECT_NEAR(rp.y(), cp.y(), 1e-6f); + EXPECT_NEAR(rp.z(), cp.z(), 1e-6f); + const Vec3f rv = srcVel.get(i); + const Vec3f cv = codecVel.get(i); + EXPECT_NEAR(rv.x(), cv.x(), 1e-6f); + EXPECT_NEAR(rv.y(), cv.y(), 1e-6f); + EXPECT_NEAR(rv.z(), cv.z(), 1e-6f); + EXPECT_EQ(srcId.get(i), codecId.get(i)); + } + } + EXPECT_TRUE(!srcIt && !codecIt); + } + + CodecRegistry::clear(); + std::remove(codecPath.c_str()); + } + + // ----------------------------------------------------------------------- + // Section C: Shared vs non-shared descriptors + // ----------------------------------------------------------------------- + { + std::vector pts; + unittest_util::genPoints(100, pts); + + std::vector positions; + positions.reserve(pts.size()); + for (const auto& p : pts) { + positions.emplace_back(float(p.x()), float(p.y()), float(p.z())); + } + + const double voxelSize = 0.1; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid(positions, *transform); + srcGrid->setName("pdg_desc"); + + // All leaves should share one Descriptor::Ptr initially + ASSERT_GT(srcGrid->tree().leafCount(), Index32(1)); + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + auto firstDescPtr = leafIt->attributeSet().descriptorPtr(); + for (; leafIt; ++leafIt) { + EXPECT_EQ(leafIt->attributeSet().descriptorPtr(), firstDescPtr); + } + } + + io::internal::initialize(); + + // -- C1: Shared descriptors (default) -- + // All leaves already share one pointer; this exercises the header=1 write path. + const std::string sharedPath = "testPDG_C_shared.vdb"; + { + io::File f(sharedPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr sharedGrid; + { + io::File f(sharedPath); + f.open(); + sharedGrid = gridPtrCast(f.readGrid("pdg_desc")); + f.close(); + } + ASSERT_TRUE(sharedGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(sharedGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(sharedGrid->tree())); + comparePositions(*srcGrid, *sharedGrid); + + // -- C2: After makeDescriptorUnique -- + // makeDescriptorUnique() creates ONE new descriptor and assigns it to every + // leaf, so all leaves still share a single pointer (the new copy). + // The codec still detects matching descriptors and writes header=1; + // this is a regression check that the round-trip remains correct. + makeDescriptorUnique(srcGrid->tree()); + { + auto leafIt = srcGrid->tree().cbeginLeaf(); + auto firstDescPtr = leafIt->attributeSet().descriptorPtr(); + ++leafIt; + if (leafIt) { + // All leaves share the same new pointer + EXPECT_EQ(leafIt->attributeSet().descriptorPtr(), firstDescPtr); + } + } + + const std::string nonSharedPath = "testPDG_C_nonshared.vdb"; + { + io::File f(nonSharedPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr nonSharedGrid; + { + io::File f(nonSharedPath); + f.open(); + nonSharedGrid = gridPtrCast(f.readGrid("pdg_desc")); + f.close(); + } + ASSERT_TRUE(nonSharedGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(nonSharedGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(nonSharedGrid->tree())); + comparePositions(*sharedGrid, *nonSharedGrid); + + // -- C3: Genuinely different descriptors (exercises the header=0 write path) -- + // Add "extra" to all leaves, then drop it from only the first leaf so that + // leaf descriptors differ by value, triggering matching=false in the codec. + appendAttribute(srcGrid->tree(), "extra"); + makeDescriptorUnique(srcGrid->tree()); + srcGrid->setName("pdg_desc_diff"); + + { + auto leafIt = srcGrid->tree().beginLeaf(); + ASSERT_TRUE(leafIt); + const size_t extraIdx = + leafIt->attributeSet().descriptor().find("extra"); + ASSERT_NE(extraIdx, AttributeSet::INVALID_POS); + const std::vector dropIndices = {extraIdx}; + AttributeSet::Descriptor::Ptr newDesc = + leafIt->attributeSet().descriptor().duplicateDrop(dropIndices); + leafIt->dropAttributes( + dropIndices, leafIt->attributeSet().descriptor(), newDesc); + } + + const std::string diffPath = "testPDG_C_diff.vdb"; + { + io::File f(diffPath); + f.write(GridPtrVec{srcGrid}); + } + + PointDataGrid::Ptr diffGrid; + { + io::File f(diffPath); + f.open(); + diffGrid = gridPtrCast(f.readGrid("pdg_desc_diff")); + f.close(); + } + ASSERT_TRUE(diffGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(diffGrid->tree())); + EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(diffGrid->tree())); + + // First leaf has {P} only; remaining leaves have {P, extra} + { + auto diffIt = diffGrid->tree().cbeginLeaf(); + ASSERT_TRUE(diffIt); + EXPECT_EQ(diffIt->attributeSet().size(), size_t(1)); + ++diffIt; + if (diffIt) { + EXPECT_EQ(diffIt->attributeSet().size(), size_t(2)); + } + } + + std::remove(sharedPath.c_str()); + std::remove(nonSharedPath.c_str()); + std::remove(diffPath.c_str()); + } +} From 056b3223cc203637b5417ed22654dc20dec7b697 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 14 May 2026 14:29:07 -0700 Subject: [PATCH 48/80] Address feedback Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 18 ++++++++++-------- openvdb/openvdb/codecs/PointIndexCodec.h | 6 ++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index e5cc550fe4..8ad1d829db 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -12,7 +12,7 @@ #include #include -#include "ScalarLeafCodec.h" +#include "impl/ScalarLeafCodec.h" #include "TopologyCodec.h" #include @@ -32,8 +32,9 @@ inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream( auto it = pagedStreams.find(attributeIndex); if (it != pagedStreams.end()) return it->second.get(); using PagedStreamT = typename PagedStreamPtrT::element_type; - pagedStreams[attributeIndex] = std::make_shared(); - return pagedStreams[attributeIndex].get(); + auto ptr = std::make_shared(); + auto& stored = (pagedStreams[attributeIndex] = std::move(ptr)); + return stored.get(); } //////////////////////////////////////// @@ -41,7 +42,7 @@ inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream( template inline void readPointDataVoxelSizes(const std::vector& leaves, - std::istream& is, std::map& voxelBufferSizes) + std::istream& is, std::unordered_map& voxelBufferSizes) { for (auto* leaf : leaves) { uint16_t voxelBufferSize; @@ -110,9 +111,8 @@ template inline void readPointDataVoxelData(const std::vector& leaves, std::istream& is, bool saveFloatAsHalf, const typename LeafT::ValueType& background, - const std::map& voxelBufferSizes) + const std::unordered_map& voxelBufferSizes) { - (void) voxelBufferSizes; using BaseLeaf = typename LeafT::BaseLeaf; for (auto* leaf : leaves) { OPENVDB_ASSERT(voxelBufferSizes.find(leaf->origin()) != voxelBufferSizes.end()); @@ -302,7 +302,7 @@ struct OPENVDB_API PointDataCodecTypeData : public io::ReadTypedOptions }; // struct PointDataCodecTypeData template -struct PointDataCodec : public TopologyCodec +struct PointDataCodec final: public TopologyCodec { using Ptr = std::unique_ptr>; @@ -312,6 +312,8 @@ struct PointDataCodec : public TopologyCodec void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { + OPENVDB_ASSERT(dynamic_cast(data.grid.get())); + GridT& grid = static_cast(*data.grid); std::vector pointAttributeNames; @@ -339,7 +341,7 @@ struct PointDataCodec : public TopologyCodec tree.getNodes(leaves); // Pass 0: read voxel data sizes - std::map voxelBufferSizes; + std::unordered_map voxelBufferSizes; internal::readPointDataVoxelSizes(leaves, is, voxelBufferSizes); // Pass 1: read descriptor and attribute metadata diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h index d8d9f1ee32..851ee140fd 100644 --- a/openvdb/openvdb/codecs/PointIndexCodec.h +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -8,7 +8,7 @@ #include -#include "ScalarLeafCodec.h" +#include "impl/ScalarLeafCodec.h" #include "TopologyCodec.h" namespace openvdb { @@ -106,7 +106,7 @@ struct WritePointIndexBuffersOp } // namespace internal template -struct PointIndexCodec : public TopologyCodec +struct PointIndexCodec final: public TopologyCodec { using Ptr = std::unique_ptr>; @@ -116,6 +116,8 @@ struct PointIndexCodec : public TopologyCodec void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final { + OPENVDB_ASSERT(dynamic_cast(data.grid.get())); + GridT& grid = static_cast(*data.grid); if (grid.hasMultiPassIO()) { From 031489e38980ea92a57ef0f69df08a44d6994567 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 14 May 2026 15:01:25 -0700 Subject: [PATCH 49/80] Address feedback (fix unused error) Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index 8ad1d829db..f4bc713bd7 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -111,7 +111,7 @@ template inline void readPointDataVoxelData(const std::vector& leaves, std::istream& is, bool saveFloatAsHalf, const typename LeafT::ValueType& background, - const std::unordered_map& voxelBufferSizes) + [[maybe_unused]] const std::unordered_map& voxelBufferSizes) { using BaseLeaf = typename LeafT::BaseLeaf; for (auto* leaf : leaves) { From c1118e298aef7dc45bd307c722aa3f4ab086f99c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 9 Jun 2026 21:26:27 -0700 Subject: [PATCH 50/80] Fix underflow bug when reading or writing empty grids Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 10 ++++-- openvdb/openvdb/unittest/TestPointCodec.cc | 42 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index f4bc713bd7..ac3d353f77 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -334,7 +334,11 @@ struct PointDataCodec final: public TopologyCodec uint16_t numPasses = 1; is.read(reinterpret_cast(&numPasses), sizeof(uint16_t)); - const Index attributes = (numPasses - 4) / 2; + // The pass layout is: voxel sizes (1) + descriptors (1) + attribute + // sizes (N) + voxel data (1) + attribute data (N) = 2N + 4 passes. + // A leafless grid stores numPasses == 0, and malformed files may store + // numPasses < 4; guard against unsigned underflow in either case. + const Index attributes = numPasses >= 4 ? Index(numPasses - 4) / 2 : 0; using LeafT = typename GridT::TreeType::LeafNodeType; std::vector leaves; @@ -419,7 +423,9 @@ struct PointDataCodec final: public TopologyCodec static_cast(internal::countPointDataPasses(leaves)); os.write(reinterpret_cast(&numPasses), sizeof(uint16_t)); - const Index attributes = (numPasses - 4) / 2; + // See readBuffers(): a leafless grid yields numPasses == 0, so guard + // against unsigned underflow rather than computing (numPasses - 4) / 2. + const Index attributes = numPasses >= 4 ? Index(numPasses - 4) / 2 : 0; // Pass 0: write voxel data sizes + descriptor tracking bool matching = true; diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 81f8ff2358..52cb6a34bc 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -545,3 +545,45 @@ TEST_F(TestPointCodec, testPointDataCodecIO) std::remove(diffPath.c_str()); } } + +// A leafless PointDataGrid stores numPasses == 0. The attribute count is +// derived as (numPasses - 4) / 2; without an underflow guard this wraps to a +// huge unsigned value and the attribute loops spin ~4.3e9 times, hanging +// both write and read. This test exercises an empty grid round-trip. +TEST_F(TestPointCodec, testPointDataCodecEmptyGrid) +{ + using namespace openvdb; + using namespace openvdb::io; + using namespace openvdb::points; + + openvdb::initialize(); + CodecRegistry::clear(); + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType())); + + PointDataGrid::Ptr srcGrid = PointDataGrid::create(); + srcGrid->setName("pdg_empty"); + EXPECT_EQ(srcGrid->tree().leafCount(), Index32(0)); + + const std::string codecPath = "testPDG_empty_codec.vdb"; + + { + io::File f(codecPath); + EXPECT_NO_THROW(f.write(GridPtrVec{srcGrid})); + } + + PointDataGrid::Ptr codecGrid; + { + io::File f(codecPath); + f.open(); + GridBase::Ptr base; + EXPECT_NO_THROW(base = f.readGrid("pdg_empty")); + codecGrid = gridPtrCast(base); + f.close(); + } + ASSERT_TRUE(codecGrid); + EXPECT_EQ(codecGrid->tree().leafCount(), Index32(0)); + EXPECT_EQ(codecGrid->activeVoxelCount(), Index64(0)); + + std::remove(codecPath.c_str()); +} From 74db979826464fe734f29ba76ab0ec815b4f4619 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 9 Jun 2026 21:43:57 -0700 Subject: [PATCH 51/80] Fix an issue in the old version codepath for reading topology where the value mask was used instead of the child mask Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/TopologyCodec.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index e602f176ce..8b64d60420 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -194,8 +194,12 @@ struct ReadTopologyOp // Copy values from the array into this node's table. if (oldVersion) { + // The node's member child mask is still empty at this point + // (PartialCreate; setChildUnsafe runs below), so iterate the + // local childMask's off-bits to match the legacy ordering and + // avoid over-reading the countOff-sized values array. Index n = 0; - for (auto iter = node.beginValueAll(); iter; ++iter) { + for (auto iter = childMask.beginOff(); iter; ++iter) { node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n++])); } OPENVDB_ASSERT(n == numValues); From b8f7f6c6176d9a3c4a05b21ad7973fe5709851ce Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 09:10:35 -0700 Subject: [PATCH 52/80] Fix issue where instanced grids were not being clipped using the correct transform Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 19 +++++++- openvdb/openvdb/unittest/TestFile.cc | 71 ++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 3aaa31c5b1..d380cfb50a 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -490,11 +490,26 @@ File::readGrid(const Name& name, const io::ReadOptions& readOptions) << " in file " << mFilename); } + // Read the parent without clipping. Archive::readGrid() converts the + // world-space clip region into index space using the grid's own + // transform, but an instance has its own transform that may differ + // from the parent's. Instead, read the full parent tree and clip the + // assembled instance below using the instance's transform, so that the + // retained region matches the requested world-space bbox. + io::ReadOptions parentOptions = readOptions; + parentOptions.clipBBox = BBoxd(); + GridBase::Ptr parent; OPENVDB_ASSERT(inputHasGridOffsets()); parentIt->second.seekToGrid(inputStream()); - parent = Archive::readGrid(parentIt->second, inputStream(), readOptions, mReadDiagnostics); - if (parent) grid->setTree(parent->baseTreePtr()); + parent = Archive::readGrid(parentIt->second, inputStream(), parentOptions, mReadDiagnostics); + if (parent) { + grid->setTree(parent->baseTreePtr()); + const auto& clipBBox = readOptions.clipBBox; + if (clipBBox.isSorted()) { + grid->clipGrid(clipBBox); + } + } } return grid; } diff --git a/openvdb/openvdb/unittest/TestFile.cc b/openvdb/openvdb/unittest/TestFile.cc index 130947fdf1..9aa7693d6a 100644 --- a/openvdb/openvdb/unittest/TestFile.cc +++ b/openvdb/openvdb/unittest/TestFile.cc @@ -557,6 +557,77 @@ TEST_F(TestFile, testWriteInstancedGrids) } +// Verify that clipping an instanced grid uses the instance's own transform, +// not the parent's. The bug was that Archive::readGrid() converted the +// world-space bbox to index space with the parent's transform before the +// instance's transform was applied, so the clipped region was wrong when the +// two transforms differed. +TEST_F(TestFile, testReadClippedInstancedGrid) +{ + using namespace openvdb; + + const char* filename = "testReadClippedInstancedGrid.vdb"; + SharedPtr scopedFile(filename, ::remove); + + // Parent grid: voxel size 1.0. Fill index [-5, 5] with value 1. + FloatTree::Ptr tree(new FloatTree(0.0f)); + tree->fill(CoordBBox(Coord(-5), Coord(5)), 1.0f, /*active=*/true); + + GridBase::Ptr parent = FloatGrid::create(tree); + parent->setName("parent"); + parent->setTransform(math::Transform::createLinearTransform(1.0)); + + // Instance grid: same tree, but voxel size 2.0. + // Index coord n → world coord 2n (double the parent's world-space extent). + GridBase::Ptr instance = FloatGrid::create(tree); + instance->setName("instance"); + instance->setTransform(math::Transform::createLinearTransform(2.0)); + + GridPtrVec grids; + grids.push_back(parent); + grids.push_back(instance); + + { + io::File vdbfile(filename); + vdbfile.write(grids); + } + + // World-space clip: [0, 6]. + // Via parent transform (voxel 1.0): index [0, 6] → voxels 0..5 survive. + // Via instance transform (voxel 2.0): index [0, 3] → voxels 0..3 survive. + // The correct answer uses the instance's transform. + const BBoxd clipBox(Vec3d(0.0), Vec3d(6.0)); + + io::File vdbfile(filename); + vdbfile.open(); + + GridBase::Ptr readGrid = vdbfile.readGrid("instance", clipBox); + EXPECT_TRUE(readGrid.get() != nullptr); + FloatGrid::Ptr clipped = gridPtrCast(readGrid); + EXPECT_TRUE(clipped.get() != nullptr); + + const CoordBBox bbox = clipped->evalActiveVoxelBoundingBox(); + // The instance's transform maps world [0,6] to index [0,3]. + EXPECT_EQ(Coord(0, 0, 0), bbox.min()); + EXPECT_EQ(Coord(3, 3, 3), bbox.max()); + + // No active voxels should survive outside [0,3] in any axis. + FloatGrid::ConstAccessor acc = clipped->getConstAccessor(); + for (int i = -5; i <= 5; ++i) { + for (int j = -5; j <= 5; ++j) { + for (int k = -5; k <= 5; ++k) { + const Coord xyz(i, j, k); + if (i >= 0 && j >= 0 && k >= 0 && i <= 3 && j <= 3 && k <= 3) { + EXPECT_EQ(1.0f, acc.getValue(xyz)); + } else { + EXPECT_EQ(0.0f, acc.getValue(xyz)); + } + } + } + } +} + + void TestFile::testReadGridDescriptors() { From 98d9d0e4f279dadff347ac900dc42457c431260a Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:58:13 -0700 Subject: [PATCH 53/80] Fix a bug where the Stream assignment operator was not calling the Archive assignment operator Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Stream.cc | 1 + openvdb/openvdb/unittest/TestStream.cc | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 938cc321b0..91b3ca4b7d 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -83,6 +83,7 @@ Stream& Stream::operator=(const Stream& other) { if (&other != this) { + Archive::operator=(other); mMeta = other.mMeta; mGrids = other.mGrids; mOutputStream = other.mOutputStream; diff --git a/openvdb/openvdb/unittest/TestStream.cc b/openvdb/openvdb/unittest/TestStream.cc index 35850fab3d..6091e60c2b 100644 --- a/openvdb/openvdb/unittest/TestStream.cc +++ b/openvdb/openvdb/unittest/TestStream.cc @@ -216,3 +216,20 @@ TestStream::testFileReadFromStream() verifyTestGrids(grids, meta); } TEST_F(TestStream, testFileReadFromStream) { testFileReadFromStream(); } + + +TEST_F(TestStream, testAssignmentPreservesArchiveFlags) +{ + using namespace openvdb; + + std::ostringstream os1(std::ios_base::binary), os2(std::ios_base::binary); + io::Stream src(os1); + src.setCompression(io::COMPRESS_ZIP); + src.setInstancingEnabled(false); + + io::Stream dst(os2); + dst = src; + + EXPECT_EQ(src.compression(), dst.compression()); + EXPECT_EQ(src.isInstancingEnabled(), dst.isInstancingEnabled()); +} From 8cb8cebbce641b03a6706ff95762fad9abd1ad4c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 16:59:50 -0700 Subject: [PATCH 54/80] Fix topology-only and data race condition Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 22 +++++++++++++++++++++ openvdb/openvdb/codecs/TopologyCodec.h | 14 +++++++++++++ openvdb/openvdb/io/Archive.cc | 15 +++++++++++++- openvdb/openvdb/io/Codec.h | 14 +++++++++---- openvdb/openvdb/io/File.cc | 4 ++-- openvdb/openvdb/io/io.h | 5 +++++ openvdb/openvdb/tree/InternalNode.h | 4 ++++ openvdb/openvdb/tree/LeafBuffer.h | 23 +++++++++++++++------- openvdb/openvdb/tree/LeafNode.h | 3 +++ openvdb/openvdb/unittest/TestCodec.cc | 13 ++++++++++-- openvdb/openvdb/unittest/TestPointCodec.cc | 21 ++++++++++++++++---- 11 files changed, 118 insertions(+), 20 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index 41f6e58501..3d7a34e63b 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -9,6 +9,7 @@ #include "Types.h" #include "io/io.h" #include "math/Transform.h" +#include "tree/LeafManager.h" #include "tree/Tree.h" #include "util/Assert.h" #include "util/logging.h" @@ -1630,6 +1631,27 @@ inline void Grid::readTopology(std::istream& is) { tree().readTopology(is, saveFloatAsHalf()); + // When called from the legacy (non-codec) TopologyOnly path, the stream + // metadata carries a flag requesting allocation+zero-fill of leaf buffers + // (PartialCreate leaves them unallocated after readTopology). + if (io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is)) { + if (meta->allocateLeafBuffers()) { + meta->setAllocateLeafBuffers(false); + if constexpr (!std::is_void_v) { + const auto background = tree().root().background(); + tree::LeafManager leafManager(tree()); + leafManager.foreach([&background](auto& leaf, size_t) { + using LeafType = std::decay_t; + if constexpr (!std::is_same_v) { + if (leaf.buffer().empty()) { + leaf.buffer().allocate(); + leaf.buffer().fill(background); + } + } + }); + } + } + } } diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 8b64d60420..cbccf5eef2 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -291,6 +292,19 @@ void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::R if (options.readMode == io::ReadMode::TopologyOnly) { internal::setTilesToBackground(grid.tree()); + // allocate leaf buffers in parallel and fill with the background value; + // ReadTopologyOp uses PartialCreate which leaves buffers unallocated. + const auto background = grid.tree().root().background(); + tree::LeafManager leafManager(grid.tree()); + leafManager.foreach([&background](auto& leaf, size_t) { + using LeafType = std::decay_t; + if constexpr (!std::is_same_v) { + if (leaf.buffer().empty()) { + leaf.buffer().allocate(); + leaf.buffer().fill(background); + } + } + }); return; } } diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 86dd2755cc..e0fc5dc599 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -182,6 +182,7 @@ struct StreamMetadata::Impl bool mDelayedLoadMeta = false; uint64_t mLeaf = 0; uint32_t mTest = 0; // for testing only + bool mAllocateLeafBuffers = false; }; // struct StreamMetadata @@ -243,6 +244,7 @@ bool StreamMetadata::writeGridStats() const { return mImpl->mWriteGr bool StreamMetadata::seekable() const { return mImpl->mSeekable; } bool StreamMetadata::delayedLoadMeta() const { return mImpl->mDelayedLoadMeta; } bool StreamMetadata::countingPasses() const { return mImpl->mCountingPasses; } +bool StreamMetadata::allocateLeafBuffers() const { return mImpl->mAllocateLeafBuffers; } uint32_t StreamMetadata::pass() const { return mImpl->mPass; } MetaMap& StreamMetadata::gridMetadata() { return mImpl->mGridMetadata; } const MetaMap& StreamMetadata::gridMetadata() const { return mImpl->mGridMetadata; } @@ -260,6 +262,7 @@ void StreamMetadata::setHalfFloat(bool b) { mImpl->mHalfFloat = b; void StreamMetadata::setWriteGridStats(bool b) { mImpl->mWriteGridStats = b; } void StreamMetadata::setSeekable(bool b) { mImpl->mSeekable = b; } void StreamMetadata::setCountingPasses(bool b) { mImpl->mCountingPasses = b; } +void StreamMetadata::setAllocateLeafBuffers(bool b) { mImpl->mAllocateLeafBuffers = b; } void StreamMetadata::setPass(uint32_t i) { mImpl->mPass = i; } void StreamMetadata::__setTest(uint32_t t) { mImpl->mTest = t; } @@ -1000,13 +1003,23 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti io::setGridClass(is, gridClass); grid->readTransform(is); - if (readOptions.readMode != io::ReadMode::TopologyOnly && !gd.isInstance()) { + const bool readTopology = readOptions.readMode != io::ReadMode::MetadataOnly && !gd.isInstance(); + const bool readBuffers = readTopology && readOptions.readMode != io::ReadMode::TopologyOnly; + if (readTopology) { // read topology if (codec) { codec->readTopology(is, *codecData, readOptions, diagnostics); } else { + if (readOptions.readMode == io::ReadMode::TopologyOnly) { + // signal Grid::readTopology to allocate+zero-fill leaf buffers + if (auto meta = io::getStreamMetadataPtr(is)) { + meta->setAllocateLeafBuffers(true); + } + } grid->readTopology(is); } + } + if (readBuffers) { // read buffers if (codec) { codec->readBuffers(is, *codecData, readOptions, diagnostics); diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 4fbf3412d1..f236732cc9 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -79,10 +79,14 @@ enum class ReadMode { Mask, /// Deserialize topology only; value buffers are skipped. The resulting /// grid has a valid tree structure (active/inactive state, node - /// hierarchy) but leaf buffer data is left at its default (background) - /// value. Useful when only the active-voxel mask is needed and - /// avoiding the cost of reading large value buffers is desirable. - TopologyOnly + /// hierarchy) and all leaf buffers are allocated and zero-filled. + /// Useful when only the active-voxel mask is needed and avoiding the + /// cost of reading large value buffers is desirable. + TopologyOnly, + /// Deserialize grid metadata and transform only; no topology, no value + /// buffers. The codec is still used to construct the correct grid type, + /// but its @c readTopology()/@c readBuffers() are not called. + MetadataOnly }; /// @brief Base class for per-grid-type, codec-specific read options. @@ -144,6 +148,8 @@ struct OPENVDB_API ReadTypedOptions /// in-place type conversion as data is read. /// @c ReadMode::TopologyOnly skips value buffers entirely, which can be /// significantly faster when only the active-voxel mask is needed. +/// @c ReadMode::MetadataOnly skips both topology and value buffers, +/// reading only grid metadata and transform. /// /// @par Per-type options /// @c typeData allows callers to attach codec-specific configuration for diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index d380cfb50a..b783008986 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -386,7 +386,7 @@ File::readAllGridMetadata() // Seek to the grid in the file. gd.seekToGrid(inputStream()); io::ReadOptions readOptions; - readOptions.readMode = io::ReadMode::TopologyOnly; + readOptions.readMode = io::ReadMode::MetadataOnly; GridBase::ConstPtr grid = Archive::readGrid(gd, inputStream(), readOptions); // Return copies of the grids, but with empty trees. // (As of 0.98.0, at least, it would suffice to just const cast @@ -426,7 +426,7 @@ File::readGridMetadata(const Name& name) const GridDescriptor& gd = it->second; gd.seekToGrid(inputStream()); io::ReadOptions readOptions; - readOptions.readMode = io::ReadMode::TopologyOnly; + readOptions.readMode = io::ReadMode::MetadataOnly; ret = Archive::readGrid(gd, inputStream(), readOptions); } return ret->copyGridWithNewTree(); diff --git a/openvdb/openvdb/io/io.h b/openvdb/openvdb/io/io.h index 13a9134012..26e513b7f6 100644 --- a/openvdb/openvdb/io/io.h +++ b/openvdb/openvdb/io/io.h @@ -73,6 +73,11 @@ class OPENVDB_API StreamMetadata bool countingPasses() const; void setCountingPasses(bool); + /// @brief Return @c true if readTopology() should allocate and zero-fill + /// leaf buffers after loading the tree structure (topology-only read mode). + bool allocateLeafBuffers() const; + void setAllocateLeafBuffers(bool); + uint32_t pass() const; void setPass(uint32_t); diff --git a/openvdb/openvdb/tree/InternalNode.h b/openvdb/openvdb/tree/InternalNode.h index fb8c5a8ef9..7dbe5349dc 100644 --- a/openvdb/openvdb/tree/InternalNode.h +++ b/openvdb/openvdb/tree/InternalNode.h @@ -82,6 +82,10 @@ class InternalNode /// @param active State assigned to all the tiles InternalNode(const Coord& origin, const ValueType& fillValue, bool active = false); + /// @brief Construct a node without allocating child memory. Children are + /// left unallocated and must be populated single-threaded, or with external + /// synchronization. The valid advanced pattern is: create all nodes + /// single-threaded, then allocate leaf buffers in parallel across distinct leaves. InternalNode(PartialCreate, const Coord&, const ValueType& fillValue, bool active = false); /// @brief Deep copy constructor diff --git a/openvdb/openvdb/tree/LeafBuffer.h b/openvdb/openvdb/tree/LeafBuffer.h index bb6222cf84..7d1a965144 100644 --- a/openvdb/openvdb/tree/LeafBuffer.h +++ b/openvdb/openvdb/tree/LeafBuffer.h @@ -44,7 +44,12 @@ class LeafBuffer explicit inline LeafBuffer(const ValueType&); /// Copy constructor inline LeafBuffer(const LeafBuffer&); - /// Construct a buffer but don't allocate memory for the full array of values. + /// @brief Construct a buffer without allocating the value array. + /// The buffer is left unallocated; call @c allocate() before use. + /// Populating the buffer (via @c allocate() or @c data()) must be done + /// single-threaded per leaf, or externally synchronized. The valid + /// advanced pattern is: create all nodes single-threaded, then call + /// @c allocate() in parallel across distinct leaves. LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) {} /// Destructor inline ~LeafBuffer(); @@ -87,11 +92,15 @@ class LeafBuffer static Index size() { return SIZE; } /// @brief Return a const pointer to the array of voxel values. - /// @details This method guarantees that the buffer is allocated and loaded. + /// @warning The buffer must already be allocated (call @c allocate() first). + /// First-touch allocation via @c data() is not thread-safe; concurrent access + /// to an unallocated buffer is a programming error flagged in debug builds. /// @warning This method should only be used by experts seeking low-level optimizations. const ValueType* data() const; /// @brief Return a pointer to the array of voxel values. - /// @details This method guarantees that the buffer is allocated and loaded. + /// @warning The buffer must already be allocated (call @c allocate() first). + /// First-touch allocation via @c data() is not thread-safe; concurrent access + /// to an unallocated buffer is a programming error flagged in debug builds. /// @warning This method should only be used by experts seeking low-level optimizations. ValueType* data(); @@ -229,9 +238,10 @@ template inline const typename LeafBuffer::ValueType* LeafBuffer::data() const { + OPENVDB_ASSERT(mData != nullptr); if (mData == nullptr) { LeafBuffer* self = const_cast(this); - if (mData == nullptr) self->mData = new ValueType[SIZE]; + self->mData = new ValueType[SIZE]; } return mData; } @@ -240,9 +250,8 @@ template inline typename LeafBuffer::ValueType* LeafBuffer::data() { - if (mData == nullptr) { - if (mData == nullptr) mData = new ValueType[SIZE]; - } + OPENVDB_ASSERT(mData != nullptr); + if (mData == nullptr) mData = new ValueType[SIZE]; return mData; } diff --git a/openvdb/openvdb/tree/LeafNode.h b/openvdb/openvdb/tree/LeafNode.h index 12692e76f1..4a53a3927f 100644 --- a/openvdb/openvdb/tree/LeafNode.h +++ b/openvdb/openvdb/tree/LeafNode.h @@ -83,6 +83,9 @@ class LeafNode /// @param value a value with which to fill the buffer /// @param active the active state to which to initialize all voxels /// @details This constructor does not allocate memory for voxel values. + /// Call @c buffer().allocate() before accessing voxel data. The valid + /// advanced pattern is: create all leaves single-threaded, then call + /// @c allocate() in parallel across distinct leaves. LeafNode(PartialCreate, const Coord& coords, const ValueType& value = zeroVal(), diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index d8981ca3f7..c9e96add01 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -219,8 +219,17 @@ void testIOImpl( f.close(); } ASSERT_TRUE(readTopo); - EXPECT_EQ(readTopo->activeVoxelCount(), Index64(0)); - EXPECT_TRUE(readTopo->tree().leafCount() == 0); + // TopologyOnly: full tree structure is read (topology + active-voxel masks), + // leaf buffers are allocated and zero-filled, values are not read. + EXPECT_EQ(readTopo->tree().leafCount(), srcGrid->tree().leafCount()); + EXPECT_TRUE(readTopo->tree().leafCount() > 0); + EXPECT_EQ(readTopo->activeVoxelCount(), srcGrid->activeVoxelCount()); + // verify leaf buffers are allocated (bool/mask buffers are always present, skip empty() check) + if constexpr (!std::is_same_v) { + for (auto leafIter = readTopo->tree().cbeginLeaf(); leafIter; ++leafIter) { + EXPECT_FALSE(leafIter->buffer().empty()); + } + } EXPECT_EQ(readTopo->getName(), gridName); // Cleanup diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 52cb6a34bc..9c8660a4a6 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -138,8 +138,15 @@ TEST_F(TestPointCodec, testPointIndexCodecIO) f.close(); } ASSERT_TRUE(codecTopo); - EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0)); - EXPECT_TRUE(codecTopo->tree().leafCount() == 0); + // TopologyOnly: topology and active state are preserved; voxel values are zero-filled + EXPECT_EQ(codecTopo->tree().leafCount(), srcGrid->tree().leafCount()); + EXPECT_EQ(codecTopo->activeVoxelCount(), srcGrid->activeVoxelCount()); + for (auto leafIt = codecTopo->tree().cbeginLeaf(); leafIt; ++leafIt) { + EXPECT_FALSE(leafIt->buffer().empty()); + for (auto voxIt = leafIt->cbeginValueOn(); voxIt; ++voxIt) { + EXPECT_EQ(*voxIt, PointIndexGrid::ValueType(0)); + } + } EXPECT_EQ(codecTopo->getName(), std::string("point_index_grid")); // Cleanup @@ -265,8 +272,14 @@ TEST_F(TestPointCodec, testPointDataCodecIO) f.close(); } ASSERT_TRUE(codecTopo); - EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0)); - EXPECT_TRUE(codecTopo->tree().leafCount() == 0); + // TopologyOnly: topology and active state are preserved; voxel values are zero-filled + EXPECT_EQ(codecTopo->tree().leafCount(), srcGrid->tree().leafCount()); + EXPECT_EQ(codecTopo->activeVoxelCount(), srcGrid->activeVoxelCount()); + for (auto leafIt = codecTopo->tree().cbeginLeaf(); leafIt; ++leafIt) { + for (auto voxIt = leafIt->cbeginValueOn(); voxIt; ++voxIt) { + EXPECT_EQ(*voxIt, PointDataGrid::ValueType(0)); + } + } std::remove(codecPath.c_str()); } From 863900fe2c6f9dfce32d4d9efe489ced5da5ee30 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:44:44 -0700 Subject: [PATCH 55/80] Add conversion codec fallback and read diagnostics Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/TopologyCodec.h | 17 +++++++++++++++++ openvdb/openvdb/io/Archive.cc | 9 +++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index cbccf5eef2..6f704cbe89 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -339,6 +339,23 @@ struct TopologyCodec : public io::Codec void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final { + // Warn when a conversion readMode was requested but this codec is a + // non-conversion instance (GridT == StorageGridT), meaning no conversion + // codec was registered for this grid type and we are falling back to the + // original type. + if constexpr (std::is_same_v) { + if (options.readMode == io::ReadMode::Half || + options.readMode == io::ReadMode::Bool || + options.readMode == io::ReadMode::Mask) + { + const std::string modeStr = + options.readMode == io::ReadMode::Half ? "Half" : + options.readMode == io::ReadMode::Bool ? "Bool" : "Mask"; + diagnostics.addWarning(data.grid->getName(), + "ReadMode::" + modeStr + " conversion is not supported for grid type '" + + GridT::gridType() + "'; reading as original type"); + } + } internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics); } diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index e0fc5dc599..987013e9c1 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -893,14 +893,15 @@ Archive::findCodec(const std::string& gridType, const io::ReadOptions& options) // if readMode is Half, then search for a codec that converts // from the storage grid type to the grid type if (options.readMode == ReadMode::Half) { - return io::CodecRegistry::get(gridType + "_to_half"); + if (auto* codec = io::CodecRegistry::get(gridType + "_to_half")) return codec; } else if (options.readMode == ReadMode::Bool) { - return io::CodecRegistry::get(gridType + "_to_bool"); + if (auto* codec = io::CodecRegistry::get(gridType + "_to_bool")) return codec; } else if (options.readMode == ReadMode::Mask) { - return io::CodecRegistry::get(gridType + "_to_mask"); + if (auto* codec = io::CodecRegistry::get(gridType + "_to_mask")) return codec; } - // Determine the I/O codec to use to read this grid + // Determine the I/O codec to use to read this grid (also the fallback when + // no conversion codec is registered for a Half/Bool/Mask readMode). return io::CodecRegistry::get(gridType); } From 8dceecf27e4eb4103bae45703195c6442f2474f1 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:46:48 -0700 Subject: [PATCH 56/80] Fix a bug in PointDataGrid where read-and-discard not being used when stream is not seekable Signed-off-by: Dan Bailey --- openvdb/openvdb/points/AttributeArray.h | 8 +- openvdb/openvdb/points/StreamCompression.cc | 9 +- openvdb/openvdb/unittest/TestPointCodec.cc | 127 ++++++++++++++++++++ 3 files changed, 142 insertions(+), 2 deletions(-) diff --git a/openvdb/openvdb/points/AttributeArray.h b/openvdb/openvdb/points/AttributeArray.h index 23ff8f8149..26f03b71cc 100644 --- a/openvdb/openvdb/points/AttributeArray.h +++ b/openvdb/openvdb/points/AttributeArray.h @@ -1611,7 +1611,13 @@ AttributeArray::skipPagedBuffers(compression::PagedInputStream& is) std::istream& inputStream = is.getInputStream(); uint8_t bloscCompressed(0); if (!mIsUniform) inputStream.read(reinterpret_cast(&bloscCompressed), sizeof(uint8_t)); - inputStream.seekg(mCompressedBytes, std::ios_base::cur); + auto meta = io::getStreamMetadataPtr(inputStream); + if (meta && meta->seekable()) { + inputStream.seekg(mCompressedBytes, std::ios_base::cur); + } else { + std::vector tempData(mCompressedBytes); + inputStream.read(tempData.data(), mCompressedBytes); + } mCompressedBytes = 0; mFlags = static_cast(mFlags & ~PARTIALREAD); } diff --git a/openvdb/openvdb/points/StreamCompression.cc b/openvdb/openvdb/points/StreamCompression.cc index f18d016463..fb99554b70 100644 --- a/openvdb/openvdb/points/StreamCompression.cc +++ b/openvdb/openvdb/points/StreamCompression.cc @@ -7,6 +7,7 @@ #include #include #include +#include #ifdef OPENVDB_USE_BLOSC #include #endif @@ -347,7 +348,13 @@ Page::skipBuffers(std::istream& is) std::streamsize bytes = isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes; - is.seekg(bytes, std::ios_base::cur); + auto meta = io::getStreamMetadataPtr(is); + if (meta && meta->seekable()) { + is.seekg(bytes, std::ios_base::cur); + } else { + std::vector tempData(bytes); + is.read(tempData.data(), bytes); + } mInfo.reset(); } diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 9c8660a4a6..25ffab976a 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -1,11 +1,16 @@ // Copyright Contributors to the OpenVDB Project // SPDX-License-Identifier: Apache-2.0 +#include #include #include +#include +#include +#include #include #include #include +#include #include #include "util.h" // for unittest_util::genPoints @@ -600,3 +605,125 @@ TEST_F(TestPointCodec, testPointDataCodecEmptyGrid) std::remove(codecPath.c_str()); } + +// Regression test for the fix in AttributeArray::skipPagedBuffers and +// Page::skipBuffers: when the stream is not seekable (written via io::Stream), +// skip must read-and-discard rather than seekg. Without the fix, both code +// paths called seekg unconditionally, corrupting the stream position on +// non-seekable streams. +TEST_F(TestPointCodec, testPointDataCodecSkipNonSeekable) +{ + using namespace openvdb; + using namespace openvdb::io; + using namespace openvdb::points; + + openvdb::initialize(); + CodecRegistry::clear(); + io::internal::initialize(); + ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType())); + + const std::vector positions = { + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(1.5f, 3.5f, 1.0f), + Vec3f(-1.0f, 6.0f, -2.0f), + Vec3f(1.1f, 1.25f, 0.06f) + }; + const std::vector velocities = { + Vec3f(1.0f, 0.0f, 0.0f), + Vec3f(0.0f, 1.0f, 0.0f), + Vec3f(0.0f, 0.0f, 1.0f), + Vec3f(1.0f, 1.0f, 0.5f) + }; + const std::vector ids = {0, 1, 2, 3}; + + const float voxelSize = 0.5f; + math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize); + + PointAttributeVector posWrapper(positions); + tools::PointIndexGrid::Ptr pointIndexGrid = + tools::createPointIndexGrid(posWrapper, *transform); + + PointDataGrid::Ptr srcGrid = + createPointDataGrid( + *pointIndexGrid, posWrapper, *transform); + srcGrid->setName("pdg_skip"); + + PointDataTree& tree = srcGrid->tree(); + tools::PointIndexTree& indexTree = pointIndexGrid->tree(); + + appendAttribute(tree, "velocity"); + populateAttribute>( + tree, indexTree, "velocity", + PointAttributeVector(velocities)); + + appendAttribute(tree, "id"); + populateAttribute>( + tree, indexTree, "id", + PointAttributeVector(ids)); + + // Write via io::Stream (seekable=false by construction) + std::ostringstream ostr(std::ios_base::binary); + io::Stream(ostr).write(GridPtrVec{srcGrid}); + + // Build ReadOptions requesting only "P" — "velocity" and "id" will be skipped + io::ReadOptions readOptions; + auto typeData = std::make_shared(); + typeData->pointAttributeNames = {"P"}; + readOptions.typeData[PointDataGrid::gridType()] = typeData; + + // Parse the stream manually, replicating Stream::Stream(istream&) but using + // our custom ReadOptions so that the skip path is exercised with seekable=false. + std::istringstream is(ostr.str(), std::ios_base::binary); + + io::Stream headerReader(is); + // StreamMetadata created without setSeekable(true) => seekable() == false + StreamMetadata::Ptr streamMeta(new StreamMetadata); + EXPECT_FALSE(streamMeta->seekable()); + + std::istringstream isFinal(ostr.str(), std::ios_base::binary); + headerReader.readHeader(isFinal); + io::setStreamMetadataPtr(isFinal, streamMeta, /*transfer=*/false); + io::setVersion(isFinal, headerReader.libraryVersion(), headerReader.fileVersion()); + io::setDataCompression(isFinal, headerReader.compression()); + + MetaMap fileMeta; + fileMeta.readMeta(isFinal); + + const int32_t gridCount = Archive::readGridCount(isFinal); + ASSERT_EQ(gridCount, 1); + + GridDescriptor gd; + gd.readHeader(isFinal); + gd.readStreamPos(isFinal); + + GridBase::Ptr baseGrid = Archive::readGrid(gd, isFinal, readOptions); + ASSERT_TRUE(baseGrid); + + PointDataGrid::Ptr resultGrid = gridPtrCast(baseGrid); + ASSERT_TRUE(resultGrid); + + // Only "P" should be present; "velocity" and "id" were skipped + { + auto leafIt = resultGrid->tree().cbeginLeaf(); + ASSERT_TRUE(leafIt); + EXPECT_EQ(leafIt->attributeSet().size(), size_t(1)); + EXPECT_NE(leafIt->attributeSet().find("P"), AttributeSet::INVALID_POS); + EXPECT_EQ(leafIt->attributeSet().find("velocity"), AttributeSet::INVALID_POS); + EXPECT_EQ(leafIt->attributeSet().find("id"), AttributeSet::INVALID_POS); + } + + // All 4 points should be present and readable + EXPECT_EQ(pointCount(resultGrid->tree()), Index64(4)); + { + std::vector readPositions; + for (auto it = resultGrid->tree().cbeginLeaf(); it; ++it) { + AttributeHandle posHandle(it->constAttributeArray("P")); + for (Index i = 0; i < it->pointCount(); ++i) { + readPositions.push_back(posHandle.get(i)); + } + } + EXPECT_EQ(readPositions.size(), size_t(4)); + } +} From 132ee154040d72f48a3e20f576ae6c9110c119d0 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:52:32 -0700 Subject: [PATCH 57/80] Calling initialize will now clear the registry making it idempotent and align with all the other initialize methods Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Codec.cc | 1 + openvdb/openvdb/unittest/TestCodec.cc | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/openvdb/openvdb/io/Codec.cc b/openvdb/openvdb/io/Codec.cc index 4873f779aa..e2a3213814 100644 --- a/openvdb/openvdb/io/Codec.cc +++ b/openvdb/openvdb/io/Codec.cc @@ -55,6 +55,7 @@ struct RegisterConvertCodec { void initialize() { + CodecRegistry::clear(); NumericGridTypes::foreach(); Vec3GridTypes::foreach(); diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index c9e96add01..db7de5c70d 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -78,6 +78,24 @@ TEST_F(TestCodec, testCodecRegistry) } +TEST_F(TestCodec, testInitializeIdempotent) +{ + using namespace openvdb::io; + + // Calling initialize() twice without uninitialize() in between must not throw. + // Previously registerCodecByName() threw KeyError on the duplicate registration. + CodecRegistry::clear(); + EXPECT_NO_THROW(internal::initialize()); + EXPECT_NO_THROW(internal::initialize()); + + // Codecs must still be registered after the second call. + EXPECT_TRUE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType())); + EXPECT_TRUE(CodecRegistry::isRegistered(openvdb::FloatGrid::gridType())); + + internal::uninitialize(); +} + + TEST_F(TestCodec, testReadDiagnostics) { using namespace openvdb; From 6624c4dce40eed2eb67e569a8510a83041c6e475 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:54:27 -0700 Subject: [PATCH 58/80] Fix a potential memory leak when reading topology Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/TopologyCodec.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 6f704cbe89..6970b99bcd 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -212,11 +212,12 @@ struct ReadTopologyOp } // Read in all child nodes and insert them into the table at their proper locations. + // Register the child before recursing so that node's destructor frees it on a throw. for (auto iter = childMask.beginOn(); iter; ++iter) { Coord origin = node.offsetToGlobalCoord(iter.pos()); auto* child = new ChildT(PartialCreate(), origin, background); - (*this)(*child); node.setChildUnsafe(iter.pos(), child); + (*this)(*child); } } From f173582e16b5eb9a25d55bad69491c37307e6f3c Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 10 Jun 2026 20:55:03 -0700 Subject: [PATCH 59/80] Throw an error if the attribute descriptor is not homogeneous Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index ac3d353f77..512eb0b67a 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -355,7 +355,17 @@ struct PointDataCodec final: public TopologyCodec // An empty pointAttributeNames means no filtering (read all attributes). std::set skipIndices; if (!pointAttributeNames.empty() && !leaves.empty()) { - const auto& nameMap = leaves[0]->attributeSet().descriptor().map(); + // Attribute filtering requires homogeneous descriptors across all + // leaves because skip decisions are made per-index across all leaves. + const auto* firstDesc = &leaves[0]->attributeSet().descriptor(); + for (size_t i = 1; i < leaves.size(); ++i) { + if (&leaves[i]->attributeSet().descriptor() != firstDesc) { + OPENVDB_THROW(IoError, + "Attribute filtering is not supported for PointDataGrids " + "with heterogeneous descriptors"); + } + } + const auto& nameMap = firstDesc->map(); const std::set wantedNames( pointAttributeNames.begin(), pointAttributeNames.end()); From 4208d6b1f3c5c22dad63c53c5855f4bd8400cddd Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 9 Jun 2026 21:39:58 -0700 Subject: [PATCH 60/80] Fix an issue where the background value was previously using ValueT instead of StorageValueT causing inactive values to be read incorrectly Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/PointDataCodec.h | 9 +- openvdb/openvdb/codecs/PointIndexCodec.h | 12 ++- openvdb/openvdb/codecs/ScalarCodec.h | 31 ++++--- openvdb/openvdb/codecs/TopologyCodec.h | 53 ++++++++---- openvdb/openvdb/codecs/impl/ScalarLeafCodec.h | 24 +++--- openvdb/openvdb/io/Compression.h | 27 +++--- openvdb/openvdb/unittest/TestCodec.cc | 82 +++++++++++++++++++ 7 files changed, 182 insertions(+), 56 deletions(-) diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index 512eb0b67a..ab203fe4de 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -111,13 +111,14 @@ template inline void readPointDataVoxelData(const std::vector& leaves, std::istream& is, bool saveFloatAsHalf, const typename LeafT::ValueType& background, - [[maybe_unused]] const std::unordered_map& voxelBufferSizes) + [[maybe_unused]] const std::unordered_map& voxelBufferSizes, + const typename LeafT::ValueType* storageBackground = nullptr) { using BaseLeaf = typename LeafT::BaseLeaf; for (auto* leaf : leaves) { OPENVDB_ASSERT(voxelBufferSizes.find(leaf->origin()) != voxelBufferSizes.end()); BaseLeaf& baseLeaf = static_cast(*leaf); - readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr, storageBackground); } } @@ -387,8 +388,10 @@ struct PointDataCodec final: public TopologyCodec } // Pass N+2: read voxel data + using ValueT = typename GridT::TreeType::ValueType; + auto& topoData = static_cast&>(data); internal::readPointDataVoxelData(leaves, is, saveFloatAsHalf, - tree.background(), voxelBufferSizes); + tree.background(), voxelBufferSizes, &topoData.storageBackground); // Passes N+3..2N+2: read attribute data buffers for (Index i = 0; i < attributes; ++i) { diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h index 851ee140fd..0dec4fd77b 100644 --- a/openvdb/openvdb/codecs/PointIndexCodec.h +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -26,10 +26,11 @@ struct ReadPointIndexBuffersOp using ValueT = typename TreeT::ValueType; ReadPointIndexBuffersOp(std::istream& _is, bool _saveFloatAsHalf, - const ValueT& _background) + const ValueT& _background, const ValueT* _storageBackground = nullptr) : is(_is) , saveFloatAsHalf(_saveFloatAsHalf) - , background(_background) { } + , background(_background) + , storageBackground(_storageBackground) { } template void operator()(NodeT&, size_t) { } @@ -40,7 +41,7 @@ struct ReadPointIndexBuffersOp // Read the value mask and voxel data via base class BaseLeaf& baseLeaf = static_cast(leaf); - readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr); + readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr, storageBackground); // Read the number of indices. Index64 numIndices = Index64(0); @@ -63,6 +64,7 @@ struct ReadPointIndexBuffersOp std::istream& is; const bool saveFloatAsHalf; const ValueT& background; + const ValueT* storageBackground = nullptr; }; // struct ReadPointIndexBuffersOp template @@ -135,7 +137,9 @@ struct PointIndexCodec final: public TopologyCodec diagnostics.addWarning(grid.getName(), "bounding box clipping is not supported for PointIndexGrids"); } - internal::ReadPointIndexBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background()); + using ValueT = typename GridT::TreeType::ValueType; + auto& topoData = static_cast&>(data); + internal::ReadPointIndexBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), &topoData.storageBackground); tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); } diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index 114c575bcf..440325d2b3 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -23,21 +23,26 @@ template struct WriteBuffersOp { using LeafT = typename TreeT::LeafNodeType; + using ValueT = typename TreeT::ValueType; - WriteBuffersOp(std::ostream& _os, bool _saveFloatAsHalf) + WriteBuffersOp(std::ostream& _os, bool _saveFloatAsHalf, const ValueT& _background) : os(_os) - , saveFloatAsHalf(_saveFloatAsHalf) { } + , saveFloatAsHalf(_saveFloatAsHalf) + , background(_background) { } template void operator()(const NodeT&, size_t) { } void operator()(const LeafT& leaf, size_t) { - writeScalarLeafBuffers(leaf, os, saveFloatAsHalf); + // Pass the background explicitly so leaf compression does not depend on + // the stream's background pointer (which the codec path no longer sets). + writeScalarLeafBuffers(leaf, os, saveFloatAsHalf, &background); } std::ostream& os; const bool saveFloatAsHalf; + const ValueT& background; }; // struct WriteBuffersOp @@ -48,13 +53,15 @@ struct ReadBuffersOp using LeafT = typename TreeT::LeafNodeType; using ValueT = typename TreeT::ValueType; using StorageLeafT = typename StorageTreeT::LeafNodeType; + using StorageValueT = typename StorageTreeT::ValueType; ReadBuffersOp(std::istream& _is, bool _saveFloatAsHalf, const ValueT& _background, - const CoordBBox* _clipBBox) + const CoordBBox* _clipBBox, const StorageValueT* _storageBackground = nullptr) : is(_is) , saveFloatAsHalf(_saveFloatAsHalf) , background(_background) - , clipBBox(_clipBBox) { } + , clipBBox(_clipBBox) + , storageBackground(_storageBackground) { } void operator()(RootT& root, size_t) { @@ -73,20 +80,22 @@ struct ReadBuffersOp void operator()(LeafT& leaf, size_t) { - readScalarLeafBuffers(leaf, is, saveFloatAsHalf, background, /*skip=*/false, clipBBox); + readScalarLeafBuffers(leaf, is, saveFloatAsHalf, background, /*skip=*/false, clipBBox, storageBackground); } std::istream& is; const bool saveFloatAsHalf; const ValueT& background; const CoordBBox* clipBBox = nullptr; + const StorageValueT* storageBackground = nullptr; }; // struct ReadBuffersOp // Free-standing function for both standard and conversion codec cases // Uses StorageGridT = GridT by default, but allows different storage type for conversions template -void scalarCodecReadBuffers(GridT& grid, std::istream& is, const io::ReadOptions& options) +void scalarCodecReadBuffers(GridT& grid, std::istream& is, const io::ReadOptions& options, + const typename StorageGridT::TreeType::ValueType* storageBackground) { if (grid.hasMultiPassIO()) { OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec"); @@ -109,7 +118,7 @@ void scalarCodecReadBuffers(GridT& grid, std::istream& is, const io::ReadOptions // Works for both standard (TreeT == StorageTreeT) and conversion cases ReadBuffersOp readBuffersOp(is, saveFloatAsHalf, tree.background(), - clipIndexBBox.get()); + clipIndexBBox.get(), storageBackground); tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); } @@ -123,7 +132,7 @@ void scalarCodecWriteBuffers(const GridT& grid, std::ostream& os) OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec"); } - WriteBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf()); + WriteBuffersOp writeBuffersOp(os, grid.saveFloatAsHalf(), grid.tree().background()); tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp); } @@ -152,8 +161,10 @@ struct ScalarCodec final: public TopologyCodec void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { + using StorageValueT = typename StorageGridT::TreeType::ValueType; GridT& grid = static_cast(*data.grid); - internal::scalarCodecReadBuffers(grid, is, options); + auto& topoData = static_cast&>(data); + internal::scalarCodecReadBuffers(grid, is, options, &topoData.storageBackground); } void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 6970b99bcd..305d9c0129 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -16,6 +16,16 @@ namespace openvdb { OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { namespace codecs { + +/// Per-read-operation state shared between readTopology() and readBuffers(). +/// Stores the storage-typed background so readBuffers() can pass it directly +/// to readCompressedValues(), bypassing the stream background ptr entirely. +template +struct TopologyCodecData : public io::CodecData +{ + StorageValueT storageBackground{}; +}; // struct TopologyCodecData + namespace internal { template @@ -41,7 +51,6 @@ struct WriteTopologyOp ValueT truncatedVal = io::truncateRealToHalf(*background); os.write(reinterpret_cast(&truncatedVal), sizeof(ValueT)); } - io::setGridBackgroundValuePtr(os, background); const Index numTiles = root.tileCount(), numChildren = root.childCount(); os.write(reinterpret_cast(&numTiles), sizeof(Index)); @@ -84,7 +93,7 @@ struct WriteTopologyOp values[i] = (node.isChildMaskOff(i) ? node.getValueUnsafe(i) : zero); } // Compress (optionally) and write out the contents of the array. - io::writeCompressedValues(os, values, NodeT::NUM_VALUES, valueMask, childMask, saveFloatAsHalf); + io::writeCompressedValues(os, values, NodeT::NUM_VALUES, valueMask, childMask, saveFloatAsHalf, background); } } @@ -130,16 +139,8 @@ struct ReadTopologyOp // Read a RootNode that was stored in the current format. - if constexpr (std::is_same_v) { - is.read(reinterpret_cast(&background), sizeof(ValueT)); - } else { - StorageValueT _background; - is.read(reinterpret_cast(&_background), sizeof(StorageValueT)); - background = static_cast(_background); - } - - root.setBackground(background, false); - io::setGridBackgroundValuePtr(is, &root.background()); + is.read(reinterpret_cast(&storageBackground), sizeof(StorageValueT)); + background = static_cast(storageBackground); Index numTiles = 0, numChildren = 0; is.read(reinterpret_cast(&numTiles), sizeof(Index)); @@ -191,7 +192,7 @@ struct ReadTopologyOp // into a contiguous array. std::unique_ptr valuePtr(new StorageValueT[numValues]); StorageValueT* values = valuePtr.get(); - io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf, &storageBackground); // Copy values from the array into this node's table. if (oldVersion) { @@ -231,6 +232,7 @@ struct ReadTopologyOp std::istream& is; bool saveFloatAsHalf; ValueT background; + StorageValueT storageBackground; io::ReadDiagnostics& diagnostics; std::string gridName; }; // struct ReadTopologyOp @@ -279,9 +281,12 @@ void setTilesToBackground(TreeT& tree) nodeManager.foreachTopDown(op); } -// Free-standing function for read case (supports type conversion via StorageGridT) +// Free-standing function for read case (supports type conversion via StorageGridT). +// codecData receives the storage-typed background value so readBuffers() callers +// can pass it explicitly to readCompressedValues(), avoiding the stream background ptr. template -void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) +void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options, + io::ReadDiagnostics& diagnostics, TopologyCodecData& codecData) { io::checkFormatVersion(is); @@ -291,6 +296,18 @@ void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::R internal::ReadTopologyOp readTopologyOp(is, grid.saveFloatAsHalf(), diagnostics, grid.getName()); readTopologyOp(grid.tree().root()); + // Restore the (value-typed) background on the root. ReadTopologyOp only reads + // the on-disk background into a local; without this the grid would retain the + // default background from GridT::create(). Pass updateChildNodes=false so the + // already-populated child nodes are left untouched. + grid.tree().root().setBackground(readTopologyOp.background, /*updateChildNodes=*/false); + + // Copy storageBackground out of the stack-local ReadTopologyOp into codecData + // so it stays alive until readBuffers() completes. readBuffers() passes it + // explicitly to readCompressedValues(), so the stream background ptr is never + // used and setGridBackgroundValuePtr() is not needed in the codec path. + codecData.storageBackground = readTopologyOp.storageBackground; + if (options.readMode == io::ReadMode::TopologyOnly) { internal::setTilesToBackground(grid.tree()); // allocate leaf buffers in parallel and fill with the background value; @@ -326,13 +343,14 @@ void topologyCodecWriteTopology(const GridBase& gridBase, std::ostream& os) template struct TopologyCodec : public io::Codec { + using StorageValueT = typename StorageGridT::TreeType::ValueType; using Ptr = std::unique_ptr>; ~TopologyCodec() noexcept = default; io::CodecData::Ptr createData() override { - auto data = std::make_unique(); + auto data = std::make_unique>(); data->grid = GridT::create(); return data; } @@ -357,7 +375,8 @@ struct TopologyCodec : public io::Codec + GridT::gridType() + "'; reading as original type"); } } - internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics); + auto& topoData = static_cast&>(data); + internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics, topoData); } void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final diff --git a/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h b/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h index ad9e088604..2288f8a5a2 100644 --- a/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h +++ b/openvdb/openvdb/codecs/impl/ScalarLeafCodec.h @@ -13,7 +13,8 @@ namespace codecs { namespace internal { template -void writeScalarLeafBuffers(const LeafT& leaf, std::ostream& os, bool saveFloatAsHalf) +void writeScalarLeafBuffers(const LeafT& leaf, std::ostream& os, bool saveFloatAsHalf, + const typename LeafT::ValueType* background = nullptr) { using NodeMaskT = typename LeafT::NodeMaskType; @@ -23,14 +24,15 @@ void writeScalarLeafBuffers(const LeafT& leaf, std::ostream& os, bool saveFloatA leaf.buffer().data(); // load values io::writeCompressedValues(os, leaf.buffer().data(), LeafT::SIZE, - leaf.getValueMask(), /*childMask=*/NodeMaskT(), saveFloatAsHalf); + leaf.getValueMask(), /*childMask=*/NodeMaskT(), saveFloatAsHalf, background); } template void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, const typename LeafT::ValueType& background, bool skip = false, - const math::CoordBBox* clipBBox = nullptr) + const math::CoordBBox* clipBBox = nullptr, + const typename StorageLeafT::ValueType* storageBackground = nullptr) { using ValueT = typename LeafT::ValueType; using NodeMaskT = typename LeafT::NodeMaskType; @@ -65,10 +67,10 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, if (skip) { if (seekable) { - io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf, storageBackground); } else { StorageBufferT storageTemp; - io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); } // Clear the value mask so that the skipped leaf has no active // voxels. Without this, the leaf retains its on-disk active @@ -81,18 +83,18 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, // ValueMask leaf: value == active state, already captured in the value mask above. // Seek/consume past the storage buffer without populating any separate leaf buffer. if (seekable) { - io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, nullptr, SIZE, valueMask, saveFloatAsHalf, storageBackground); } else { StorageBufferT storageTemp; - io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); } } else if constexpr (std::is_same_v) { // Bool leaf: must read storage values regardless of seekability, then convert. if constexpr (std::is_same_v) { - io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); } else { StorageBufferT storageTemp; - io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); for (Index i = 0; i < SIZE; ++i) { leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); } @@ -100,10 +102,10 @@ void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf, } else { leaf.buffer().allocate(); if constexpr (std::is_same_v) { - io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); } else { StorageBufferT storageTemp; - io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf); + io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf, storageBackground); for (Index i = 0; i < SIZE; ++i) { leaf.buffer().setValue(i, static_cast(storageTemp.getValue(i))); } diff --git a/openvdb/openvdb/io/Compression.h b/openvdb/openvdb/io/Compression.h index ae572c4105..ade464275d 100644 --- a/openvdb/openvdb/io/Compression.h +++ b/openvdb/openvdb/io/Compression.h @@ -451,7 +451,7 @@ struct HalfWriter { template inline void readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, - const MaskT& valueMask, bool fromHalf) + const MaskT& valueMask, bool fromHalf, const ValueT* background = nullptr) { checkFormatVersion(is); @@ -475,13 +475,15 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, } } - ValueT background = zeroVal(); - if (const void* bgPtr = getGridBackgroundValuePtr(is)) { - background = *static_cast(bgPtr); + ValueT bgValue = zeroVal(); + if (background) { + bgValue = *background; + } else if (const void* bgPtr = getGridBackgroundValuePtr(is)) { + bgValue = *static_cast(bgPtr); } - ValueT inactiveVal1 = background; + ValueT inactiveVal1 = bgValue; ValueT inactiveVal0 = - ((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : math::negative(background)); + ((metadata == NO_MASK_OR_INACTIVE_VALS) ? bgValue : math::negative(bgValue)); if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL || metadata == MASK_AND_ONE_INACTIVE_VAL || @@ -620,7 +622,8 @@ writeCompressedValuesSize(ValueT* srcBuf, Index srcCount, template inline void writeCompressedValues(std::ostream& os, const ValueT* srcBuf, Index srcCount, - const MaskT& valueMask, const MaskT& childMask, bool toHalf) + const MaskT& valueMask, const MaskT& childMask, bool toHalf, + const ValueT* background = nullptr) { // Get the stream's compression settings. const uint32_t compress = getDataCompression(os); @@ -642,12 +645,14 @@ writeCompressedValues(std::ostream& os, const ValueT* srcBuf, Index srcCount, // an inside/outside bitmask. const ValueT zero = zeroVal(); - ValueT background = zero; - if (const void* bgPtr = getGridBackgroundValuePtr(os)) { - background = *static_cast(bgPtr); + ValueT bgValue = zero; + if (background) { + bgValue = *background; + } else if (const void* bgPtr = getGridBackgroundValuePtr(os)) { + bgValue = *static_cast(bgPtr); } - MaskCompress maskCompressData(valueMask, childMask, srcBuf, background); + MaskCompress maskCompressData(valueMask, childMask, srcBuf, bgValue); metadata = maskCompressData.metadata; os.write(reinterpret_cast(&metadata), /*bytes=*/1); diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index db7de5c70d..cfa14de91e 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -430,3 +430,85 @@ TEST_F(TestCodec, testNumericToMaskCodecConversion) testConvertCodecImpl(); testConvertCodecImpl(); } + +// Regression test for the dangling storageBackground pointer bug. +// +// ReadTopologyOp stores storageBackground on its stack frame and registers +// &storageBackground with the stream. Before the fix, topologyCodecReadTopology +// returned and destroyed ReadTopologyOp before readBuffers() ran; readCompressedValues +// then dereferenced the dead pointer to reconstruct inactive voxels under +// COMPRESS_ACTIVE_MASK, producing garbage inactive values. +// +// The test is deliberately structured to maximize the chance that the freed +// stack frame has been overwritten: a non-zero background (3.0f / 5) forces the +// reconstructed inactive value to be wrong if the pointer is stale, and +// COMPRESS_ACTIVE_MASK (flag 0x2, always on by default) is the code path that +// uses the background pointer. +TEST_F(TestCodec, testInactiveValuesAfterReadBuffers) +{ + using namespace openvdb; + using namespace openvdb::io; + + openvdb::io::CodecRegistry::clear(); + openvdb::io::internal::initialize(); + + // Float: non-zero background, active region surrounded by inactive background voxels. + { + const float bg = 3.0f; + FloatGrid::Ptr src = FloatGrid::create(bg); + src->setName("float_bg"); + src->fill(CoordBBox(Coord(0), Coord(15)), 1.0f, /*active=*/true); + src->fill(CoordBBox(Coord(4), Coord(11)), bg, /*active=*/false); + + const std::string path = "testInactiveVals_float.vdb"; + { + io::File f(path); + f.setCompression(COMPRESS_ACTIVE_MASK); + f.write(GridPtrVec{src}); + } + FloatGrid::Ptr result; + { + io::File f(path); + f.open(); + result = gridPtrCast(f.readGrid("float_bg")); + f.close(); + } + ASSERT_TRUE(result); + EXPECT_EQ(result->background(), bg); + FloatGrid::ConstAccessor refAcc = src->getConstAccessor(); + for (FloatGrid::ValueAllCIter it = result->cbeginValueAll(); it; ++it) { + EXPECT_EQ(*it, refAcc.getValue(it.getCoord())); + } + std::remove(path.c_str()); + } + + // Int32: non-zero background (5), verify inactive values round-trip. + { + const int bg = 5; + Int32Grid::Ptr src = Int32Grid::create(bg); + src->setName("int_bg"); + src->fill(CoordBBox(Coord(0), Coord(15)), 99, /*active=*/true); + src->fill(CoordBBox(Coord(4), Coord(11)), bg, /*active=*/false); + + const std::string path = "testInactiveVals_int.vdb"; + { + io::File f(path); + f.setCompression(COMPRESS_ACTIVE_MASK); + f.write(GridPtrVec{src}); + } + Int32Grid::Ptr result; + { + io::File f(path); + f.open(); + result = gridPtrCast(f.readGrid("int_bg")); + f.close(); + } + ASSERT_TRUE(result); + EXPECT_EQ(result->background(), bg); + Int32Grid::ConstAccessor refAcc = src->getConstAccessor(); + for (Int32Grid::ValueAllCIter it = result->cbeginValueAll(); it; ++it) { + EXPECT_EQ(*it, refAcc.getValue(it.getCoord())); + } + std::remove(path.c_str()); + } +} From ca04e210b54c1cbf0346512ecdc46b9297ca6265 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Thu, 11 Jun 2026 15:25:56 -0700 Subject: [PATCH 61/80] Add ReadOptions argument to Stream Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/ScalarCodec.h | 13 ++++++--- openvdb/openvdb/codecs/TopologyCodec.h | 11 ++++--- openvdb/openvdb/io/Stream.cc | 8 ++++- openvdb/openvdb/io/Stream.h | 8 ++++- openvdb/openvdb/points/PointDataIO.h | 6 ++-- openvdb/openvdb/unittest/TestPointCodec.cc | 34 +++++----------------- 6 files changed, 41 insertions(+), 39 deletions(-) diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index 440325d2b3..0999707068 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -169,10 +169,15 @@ struct ScalarCodec final: public TopologyCodec void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final { - if constexpr (Mode == io::CodecMode::ReadOnly) return; - - const GridT& grid = static_cast(gridBase); - internal::scalarCodecWriteBuffers(grid, os); + // Note: the write body must live inside the negated if constexpr branch + // so it is not instantiated for read-only codecs. A bare + // `if constexpr (Mode == ReadOnly) return;` would still instantiate the + // code that follows, which fails to compile for the scalar-to-mask/bool + // convert codecs (their leaf buffers expose WordType*, not ValueType*). + if constexpr (Mode != io::CodecMode::ReadOnly) { + const GridT& grid = static_cast(gridBase); + internal::scalarCodecWriteBuffers(grid, os); + } } }; // struct ScalarCodec diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 305d9c0129..eee4ead6e5 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -381,10 +381,13 @@ struct TopologyCodec : public io::Codec void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final { - // disable implementation when read only - if constexpr (Mode == io::CodecMode::ReadOnly) return; - - internal::topologyCodecWriteTopology(gridBase, os); + // Disable implementation when read only. The body must live inside the + // negated if constexpr branch so it is not instantiated for read-only + // codecs; a bare `if constexpr (...) return;` still instantiates what + // follows. + if constexpr (Mode != io::CodecMode::ReadOnly) { + internal::topologyCodecWriteTopology(gridBase, os); + } } }; // struct TopologyCodec diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 91b3ca4b7d..5b7ae46601 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -21,6 +21,12 @@ namespace io { Stream::Stream(std::istream& is) + : Stream(is, io::ReadOptions{}) +{ +} + + +Stream::Stream(std::istream& is, const io::ReadOptions& readOptions) { if (!is) return; @@ -51,7 +57,7 @@ Stream::Stream(std::istream& is) gd.readHeader(is); gd.readStreamPos(is); descriptors.push_back(gd); - GridBase::Ptr grid = Archive::readGrid(gd, is, io::ReadOptions{}); + GridBase::Ptr grid = Archive::readGrid(gd, is, readOptions); mGrids->push_back(grid); namedGrids[gd.uniqueName()] = grid; } diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index 418a06b142..fd302c8547 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -23,7 +23,13 @@ class OPENVDB_API Stream: public Archive public: /// @brief Read grids from an input stream. /// @param is The input stream to read from - explicit Stream(std::istream&); + explicit Stream(std::istream& is); + + /// @brief Read grids from an input stream using the given read options. + /// @param is The input stream to read from + /// @param readOptions Options controlling how grids are read (e.g. attribute + /// skipping for point data grids) + Stream(std::istream& is, const io::ReadOptions& readOptions); OPENVDB_DEPRECATED_MESSAGE("Use Stream(std::istream&) instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") Stream(std::istream& is, bool /*delayLoad*/) : Stream(is) { } diff --git a/openvdb/openvdb/points/PointDataIO.h b/openvdb/openvdb/points/PointDataIO.h index fddbaef6c4..110b93972b 100644 --- a/openvdb/openvdb/points/PointDataIO.h +++ b/openvdb/openvdb/points/PointDataIO.h @@ -21,7 +21,8 @@ namespace io template<> inline void readCompressedValues( std::istream& is, PointDataIndex32* destBuf, Index destCount, - const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/) + const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/, + const PointDataIndex32* /*background*/) { using compression::bloscDecompress; @@ -82,7 +83,8 @@ template<> inline void writeCompressedValues( std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount, const util::NodeMask<3>& /*valueMask*/, - const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/) + const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/, + const PointDataIndex32* /*background*/) { using compression::bloscCompress; diff --git a/openvdb/openvdb/unittest/TestPointCodec.cc b/openvdb/openvdb/unittest/TestPointCodec.cc index 25ffab976a..0729f6cc98 100644 --- a/openvdb/openvdb/unittest/TestPointCodec.cc +++ b/openvdb/openvdb/unittest/TestPointCodec.cc @@ -673,35 +673,15 @@ TEST_F(TestPointCodec, testPointDataCodecSkipNonSeekable) typeData->pointAttributeNames = {"P"}; readOptions.typeData[PointDataGrid::gridType()] = typeData; - // Parse the stream manually, replicating Stream::Stream(istream&) but using - // our custom ReadOptions so that the skip path is exercised with seekable=false. + // Read via io::Stream (non-seekable by construction) with our ReadOptions, so the + // skip path is exercised with seekable == false. std::istringstream is(ostr.str(), std::ios_base::binary); + io::Stream strm(is, readOptions); + GridPtrVecPtr grids = strm.getGrids(); + ASSERT_TRUE(grids); + ASSERT_EQ(grids->size(), size_t(1)); - io::Stream headerReader(is); - // StreamMetadata created without setSeekable(true) => seekable() == false - StreamMetadata::Ptr streamMeta(new StreamMetadata); - EXPECT_FALSE(streamMeta->seekable()); - - std::istringstream isFinal(ostr.str(), std::ios_base::binary); - headerReader.readHeader(isFinal); - io::setStreamMetadataPtr(isFinal, streamMeta, /*transfer=*/false); - io::setVersion(isFinal, headerReader.libraryVersion(), headerReader.fileVersion()); - io::setDataCompression(isFinal, headerReader.compression()); - - MetaMap fileMeta; - fileMeta.readMeta(isFinal); - - const int32_t gridCount = Archive::readGridCount(isFinal); - ASSERT_EQ(gridCount, 1); - - GridDescriptor gd; - gd.readHeader(isFinal); - gd.readStreamPos(isFinal); - - GridBase::Ptr baseGrid = Archive::readGrid(gd, isFinal, readOptions); - ASSERT_TRUE(baseGrid); - - PointDataGrid::Ptr resultGrid = gridPtrCast(baseGrid); + PointDataGrid::Ptr resultGrid = gridPtrCast((*grids)[0]); ASSERT_TRUE(resultGrid); // Only "P" should be present; "velocity" and "id" were skipped From 392162bbcec5cccd32af9491e9229201cd5c8531 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 24 Jun 2026 09:41:35 -0700 Subject: [PATCH 62/80] Add an Index64 size to readBuffers() to aid seeking for faster performance Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/BoolCodec.h | 2 +- openvdb/openvdb/codecs/PointDataCodec.h | 2 +- openvdb/openvdb/codecs/PointIndexCodec.h | 2 +- openvdb/openvdb/codecs/ScalarCodec.h | 2 +- openvdb/openvdb/codecs/ValueMaskCodec.h | 2 +- openvdb/openvdb/io/Archive.cc | 4 +++- openvdb/openvdb/io/Codec.h | 5 +++-- 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h index e8537fa662..82c7ae5a38 100644 --- a/openvdb/openvdb/codecs/BoolCodec.h +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -110,7 +110,7 @@ struct BoolCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { GridT& grid = static_cast(*data.grid); diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index ab203fe4de..7cb6219cb7 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -311,7 +311,7 @@ struct PointDataCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { OPENVDB_ASSERT(dynamic_cast(data.grid.get())); diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h index 0dec4fd77b..d1694d5b96 100644 --- a/openvdb/openvdb/codecs/PointIndexCodec.h +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -116,7 +116,7 @@ struct PointIndexCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final + void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final { OPENVDB_ASSERT(dynamic_cast(data.grid.get())); diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index 0999707068..60b1eb6425 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -159,7 +159,7 @@ struct ScalarCodec final: public TopologyCodec } } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { using StorageValueT = typename StorageGridT::TreeType::ValueType; GridT& grid = static_cast(*data.grid); diff --git a/openvdb/openvdb/codecs/ValueMaskCodec.h b/openvdb/openvdb/codecs/ValueMaskCodec.h index 27f6289cb9..3d2b8d7bd0 100644 --- a/openvdb/openvdb/codecs/ValueMaskCodec.h +++ b/openvdb/openvdb/codecs/ValueMaskCodec.h @@ -102,7 +102,7 @@ struct ValueMaskCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final { GridT& grid = static_cast(*data.grid); diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 61e498c48c..0fd0db1dab 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -1033,7 +1033,9 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti if (readBuffers) { // read buffers if (codec) { - codec->readBuffers(is, *codecData, readOptions, diagnostics); + OPENVDB_ASSERT(gd.getEndPos() >= gd.getGridPos()); + const Index64 size = static_cast(gd.getEndPos() - gd.getGridPos()); + codec->readBuffers(is, size, *codecData, readOptions, diagnostics); } else { const auto& worldBBox = readOptions.clipBBox; const bool clip = worldBBox.isSorted(); diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index f236732cc9..86b93b6b3c 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -408,8 +408,9 @@ struct OPENVDB_API Codec /// that mode internally for safety. If @c options.clipBBox is non-empty, /// restrict the loaded data to the region that intersects it; if the codec /// cannot honour clipping natively, fall back to a post-process and record - /// a warning via @a diagnostics. - virtual void readBuffers(std::istream& /*is*/, CodecData& /*data*/, + /// a warning via @a diagnostics. The @c size argument is the number of bytes + /// occupied by the entire readBuffers data section. + virtual void readBuffers(std::istream& /*is*/, Index64 /*size*/, CodecData& /*data*/, const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { } /// @brief Serialize the grid topology (tree structure and active-voxel From b846e9351bb2663d09f921c839b5720c94ae9b2f Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 5 Aug 2026 10:38:42 -0700 Subject: [PATCH 63/80] Address feedback Signed-off-by: Dan Bailey --- openvdb/openvdb/Grid.h | 5 +-- openvdb/openvdb/io/Archive.cc | 20 +++++++++--- openvdb/openvdb/io/Codec.h | 7 ++-- openvdb/openvdb/io/Stream.cc | 15 +++++++++ openvdb/openvdb/io/Stream.h | 3 ++ openvdb/openvdb/unittest/TestStream.cc | 45 ++++++++++++++++++++++++++ 6 files changed, 86 insertions(+), 9 deletions(-) diff --git a/openvdb/openvdb/Grid.h b/openvdb/openvdb/Grid.h index 3d7a34e63b..133a5866db 100644 --- a/openvdb/openvdb/Grid.h +++ b/openvdb/openvdb/Grid.h @@ -1632,8 +1632,9 @@ Grid::readTopology(std::istream& is) { tree().readTopology(is, saveFloatAsHalf()); // When called from the legacy (non-codec) TopologyOnly path, the stream - // metadata carries a flag requesting allocation+zero-fill of leaf buffers - // (PartialCreate leaves them unallocated after readTopology). + // metadata carries a flag requesting that leaf buffers be allocated and + // filled with the background value (PartialCreate leaves them + // unallocated after readTopology). if (io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is)) { if (meta->allocateLeafBuffers()) { meta->setAllocateLeafBuffers(false); diff --git a/openvdb/openvdb/io/Archive.cc b/openvdb/openvdb/io/Archive.cc index 0fd0db1dab..d1d2451943 100644 --- a/openvdb/openvdb/io/Archive.cc +++ b/openvdb/openvdb/io/Archive.cc @@ -1021,13 +1021,25 @@ Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOpti if (codec) { codec->readTopology(is, *codecData, readOptions, diagnostics); } else { + io::StreamMetadata::Ptr allocateLeafBuffersMeta; if (readOptions.readMode == io::ReadMode::TopologyOnly) { - // signal Grid::readTopology to allocate+zero-fill leaf buffers - if (auto meta = io::getStreamMetadataPtr(is)) { - meta->setAllocateLeafBuffers(true); + // Signal Grid::readTopology to allocate leaf buffers and + // fill them with the background value. + allocateLeafBuffersMeta = io::getStreamMetadataPtr(is); + if (allocateLeafBuffersMeta) { + allocateLeafBuffersMeta->setAllocateLeafBuffers(true); } } - grid->readTopology(is); + try { + grid->readTopology(is); + } catch (...) { + // Grid::readTopology() clears the flag on success, but if + // it throws the flag must not leak into subsequent grid reads. + if (allocateLeafBuffersMeta) { + allocateLeafBuffersMeta->setAllocateLeafBuffers(false); + } + throw; + } } } if (readBuffers) { diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 86b93b6b3c..70eea1c468 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -79,9 +79,10 @@ enum class ReadMode { Mask, /// Deserialize topology only; value buffers are skipped. The resulting /// grid has a valid tree structure (active/inactive state, node - /// hierarchy) and all leaf buffers are allocated and zero-filled. - /// Useful when only the active-voxel mask is needed and avoiding the - /// cost of reading large value buffers is desirable. + /// hierarchy) and all leaf buffers are allocated and filled with the + /// grid's background value. Useful when only the active-voxel mask is + /// needed and avoiding the cost of reading large value buffers is + /// desirable. TopologyOnly, /// Deserialize grid metadata and transform only; no topology, no value /// buffers. The codec is still used to construct the correct grid type, diff --git a/openvdb/openvdb/io/Stream.cc b/openvdb/openvdb/io/Stream.cc index 5b7ae46601..87f9f99058 100644 --- a/openvdb/openvdb/io/Stream.cc +++ b/openvdb/openvdb/io/Stream.cc @@ -28,6 +28,21 @@ Stream::Stream(std::istream& is) Stream::Stream(std::istream& is, const io::ReadOptions& readOptions) { + // Read modes that stop before consuming all of a grid's data are not + // supported, because a stream is read sequentially - the position after + // a partial read is still inside the previous grid's data, so the next + // grid header would be read from the wrong offset. + // TODO: Skip over the unread bytes of each grid instead of disallowing + // these read modes. This is best implemented alongside the extension that + // adds support for byte skipping in non-seekable streams. + if (readOptions.readMode == io::ReadMode::MetadataOnly || + readOptions.readMode == io::ReadMode::TopologyOnly) { + OPENVDB_THROW(ValueError, "io::ReadMode::" + << (readOptions.readMode == io::ReadMode::MetadataOnly + ? "MetadataOnly" : "TopologyOnly") + << " is not supported when reading from a stream"); + } + if (!is) return; // Delayed loading has been removed - always read directly from the stream diff --git a/openvdb/openvdb/io/Stream.h b/openvdb/openvdb/io/Stream.h index fd302c8547..432e5eb9a9 100644 --- a/openvdb/openvdb/io/Stream.h +++ b/openvdb/openvdb/io/Stream.h @@ -29,6 +29,9 @@ class OPENVDB_API Stream: public Archive /// @param is The input stream to read from /// @param readOptions Options controlling how grids are read (e.g. attribute /// skipping for point data grids) + /// @throw ValueError if @a readOptions requests @c io::ReadMode::MetadataOnly + /// or @c io::ReadMode::TopologyOnly. These modes leave a grid's data + /// partially unread, which a sequentially read stream cannot skip over. Stream(std::istream& is, const io::ReadOptions& readOptions); OPENVDB_DEPRECATED_MESSAGE("Use Stream(std::istream&) instead. This method is deprecated and will be removed. Delayed loading is no longer supported.") diff --git a/openvdb/openvdb/unittest/TestStream.cc b/openvdb/openvdb/unittest/TestStream.cc index 6091e60c2b..6146b8f96e 100644 --- a/openvdb/openvdb/unittest/TestStream.cc +++ b/openvdb/openvdb/unittest/TestStream.cc @@ -218,6 +218,51 @@ TestStream::testFileReadFromStream() TEST_F(TestStream, testFileReadFromStream) { testFileReadFromStream(); } +TEST_F(TestStream, testUnsupportedReadModes) +{ + using namespace openvdb; + + Int32Grid::Ptr grid1 = Int32Grid::create(0); + grid1->setName("first"); + grid1->tree().setValue(Coord(0, 0, 0), 1); + + FloatGrid::Ptr grid2 = FloatGrid::create(0.0f); + grid2->setName("second"); + grid2->tree().setValue(Coord(1, 2, 3), 2.0f); + + std::ostringstream ostr(std::ios_base::binary); + io::Stream(ostr).write(GridPtrVec{grid1, grid2}); + + // A stream is read sequentially, so read modes that leave part of a grid + // unread cannot be supported - the next grid header would be read from + // the wrong offset. + for (auto readMode: {io::ReadMode::MetadataOnly, io::ReadMode::TopologyOnly}) { + io::ReadOptions readOptions; + readOptions.readMode = readMode; + + std::istringstream is(ostr.str(), std::ios_base::binary); + EXPECT_THROW({ io::Stream strm(is, readOptions); }, ValueError); + } + + // Modes that read every byte of each grid are supported. + for (auto readMode: {io::ReadMode::Original, io::ReadMode::Half, + io::ReadMode::Bool, io::ReadMode::Mask}) + { + io::ReadOptions readOptions; + readOptions.readMode = readMode; + + std::istringstream is(ostr.str(), std::ios_base::binary); + io::Stream strm(is, readOptions); + + GridPtrVecPtr grids = strm.getGrids(); + ASSERT_TRUE(grids); + ASSERT_EQ(grids->size(), size_t(2)); + EXPECT_EQ((*grids)[0]->getName(), std::string("first")); + EXPECT_EQ((*grids)[1]->getName(), std::string("second")); + } +} + + TEST_F(TestStream, testAssignmentPreservesArchiveFlags) { using namespace openvdb; From 6bff7fa0d1337ad0ac737f50bce9d36cf6f8f12e Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 11 Aug 2026 17:12:57 -0700 Subject: [PATCH 64/80] Remove delayed loading option in vdb tool Signed-off-by: Dan Bailey --- openvdb_cmd/vdb_tool/include/Tool.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openvdb_cmd/vdb_tool/include/Tool.h b/openvdb_cmd/vdb_tool/include/Tool.h index 2833bf1505..bc9824c4e6 100644 --- a/openvdb_cmd/vdb_tool/include/Tool.h +++ b/openvdb_cmd/vdb_tool/include/Tool.h @@ -622,8 +622,7 @@ void Tool::init() mParser.addAction( {"read", "import", "load", "i"}, "Read one or more geometry or VDB files from disk or STDIN.", {{"files", "", "{file|stdin}.{obj|ply|abc|stl|off|pts|xyz|e57|vdb|nvdb|gltf|glb|geo|usd|usda|usdc|usdz}", "list of files or the input stream, e.g. file.vdb,stdin.vdb. Note that \"files=\" is optional since any argument without \"=\" is intrepreted as a file and appended to \"files\""}, - {"grids", "*", "*|grid_name,...", "list of VDB grids name to be imported (defaults to \"*\", i.e. import all available grids)"}, - {"delayed", "true", "1|0|true|false", "toggle delayed loading of VDB grids (enabled by default). This option is ignored by other file types"}}, + {"grids", "*", "*|grid_name,...", "list of VDB grids name to be imported (defaults to \"*\", i.e. import all available grids)"}}, [](){}, [&](){this->read();}, 0);// anonymous options are treated as to the first option,i.e. "files" mParser.addAction( @@ -1466,7 +1465,7 @@ void Tool::readVDB(const std::string &fileName) } else { if (mParser.verbose) mTimer.start("Reading VDB grid(s) from file named \""+fileName+"\""); io::File file(fileName); - file.open(mParser.get("delayed")); + file.open(); grids = file.getGrids(); } const size_t count = mGrid.size(); From 5354d1408010559281228b7a352e7954ffe85257 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 11 Aug 2026 18:20:31 -0700 Subject: [PATCH 65/80] Fix doxygen errors Signed-off-by: Dan Bailey --- doc/changes.txt | 4 ++-- openvdb/openvdb/io/Archive.h | 2 ++ openvdb/openvdb/io/Compression.h | 2 ++ openvdb/openvdb/points/PointDataGrid.h | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/changes.txt b/doc/changes.txt index 3c05074c81..033231bbaa 100644 --- a/doc/changes.txt +++ b/doc/changes.txt @@ -2034,7 +2034,7 @@ Bug fixes: New features: - Added @vdblink{tools::FindActiveValues,FindActiveValues}, which counts the active values in a tree that intersect a given bounding box. -- Added @vdblink{io::DelayedLoadMetadata,DelayedLoadMetadata}, which stores +- Added @c io::DelayedLoadMetadata, which stores mask offsets and compression sizes on write to accelerate delayed load reading. @@ -2933,7 +2933,7 @@ New features: - Added a toggle to the @vdblink::tools::clip() clip@endlink tool to invert the clipping mask. - Custom leaf node implementations may now optimize their file layout - by inheriting from @vdblink::io::MultiPass io::MultiPass@endlink. + by inheriting from @c io::MultiPass. Voxel data for grids with such leaf nodes will be written and read in multiple passes, allowing blocks of related data to be stored contiguously. [Contributed by Double Negative] diff --git a/openvdb/openvdb/io/Archive.h b/openvdb/openvdb/io/Archive.h index e0414f166a..5d15b9cdc2 100644 --- a/openvdb/openvdb/io/Archive.h +++ b/openvdb/openvdb/io/Archive.h @@ -162,12 +162,14 @@ class OPENVDB_API Archive /// Write the given grid descriptor and grid to an output stream /// and update the GridDescriptor offsets. /// @param seekable if true, the output stream supports seek operations + /// @param writeOptions options controlling how grid data is written void writeGrid(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable, const io::WriteOptions& writeOptions = io::WriteOptions{}) const; /// Write the given grid descriptor and grid metadata to an output stream /// and update the GridDescriptor offsets, but don't write the grid's tree, /// since it is shared with another grid. /// @param seekable if true, the output stream supports seek operations + /// @param writeOptions options controlling how grid data is written void writeGridInstance(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable, const io::WriteOptions& writeOptions = io::WriteOptions{}) const; diff --git a/openvdb/openvdb/io/Compression.h b/openvdb/openvdb/io/Compression.h index ade464275d..a81b1ef1e5 100644 --- a/openvdb/openvdb/io/Compression.h +++ b/openvdb/openvdb/io/Compression.h @@ -448,6 +448,7 @@ struct HalfWriter { /// which positions in the buffer correspond to active values /// @param fromHalf if true, read 16-bit half floats from the input stream /// and convert them to full floats +/// @param background optional background value used when mask compressed template inline void readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount, @@ -619,6 +620,7 @@ writeCompressedValuesSize(ValueT* srcBuf, Index srcCount, /// @param childMask a bitmask (typically, a node's child mask) indicating /// which positions in the buffer correspond to child node pointers /// @param toHalf if true, convert floating-point values to 16-bit half floats +/// @param background optional background value used when mask compressed template inline void writeCompressedValues(std::ostream& os, const ValueT* srcBuf, Index srcCount, diff --git a/openvdb/openvdb/points/PointDataGrid.h b/openvdb/openvdb/points/PointDataGrid.h index f186717799..35988239af 100644 --- a/openvdb/openvdb/points/PointDataGrid.h +++ b/openvdb/openvdb/points/PointDataGrid.h @@ -94,7 +94,7 @@ makeDescriptorUnique(PointDataTreeT& tree); template OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.") inline void -setStreamingMode(PointDataTreeT&, bool /*on*/ = true) { } +setStreamingMode(PointDataTreeT& tree, bool on = true) { (void)tree; (void)on; } template From d85dead9094e28c0fcf8aa760865bc4f6261e1e2 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 11 Aug 2026 19:35:28 -0700 Subject: [PATCH 66/80] Retry vcpkg install in Windows CI when a download fails Signed-off-by: Dan Bailey --- ci/install_windows.ps1 | 52 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/ci/install_windows.ps1 b/ci/install_windows.ps1 index 6054cb96bf..a4d44da353 100644 --- a/ci/install_windows.ps1 +++ b/ci/install_windows.ps1 @@ -21,20 +21,52 @@ $vcpkgPackages = @( "nanobind" ) +$maxAttempts = 3 + +# curl's schannel backend reports an unreachable CRL/OCSP responder as a +# certificate verification failure (error 60), which vcpkg then treats as +# permanent. Downgrade a missing revocation answer to a warning; the rest of +# certificate validation still applies. +$env:VCPKG_SSL_REVOKE_BEST_EFFORT = "1" + # Update vcpkg vcpkg update -# Allow the vcpkg command to fail once so we can retry with the latest -try { - vcpkg install $vcpkgPackages -} catch { - Write-Host "vcpkg install failed, retrying with latest ports..." - # Retry the installation with updated ports - Push-Location $env:VCPKG_INSTALLATION_ROOT - git pull - Pop-Location - vcpkg update +$installed = $false + +for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { vcpkg install $vcpkgPackages + + # A failing native command does not raise a terminating error, so the exit + # code has to be inspected explicitly rather than relying on try/catch. + if ($LASTEXITCODE -eq 0) { + $installed = $true + break + } + + if ($attempt -eq $maxAttempts) { + break + } + + # vcpkg fetches port sources directly from upstream hosts and won't retry + # downloads it classifies as permanent failures, so a single flaky TLS + # handshake aborts the whole install. + Write-Host "vcpkg install failed (attempt $attempt of $maxAttempts), retrying..." + Start-Sleep -Seconds 15 + + # Refresh the ports before the last attempt in case the failure is caused + # by a stale port rather than the network. + if ($attempt -eq ($maxAttempts - 1)) { + Write-Host "Retrying with latest ports..." + Push-Location $env:VCPKG_INSTALLATION_ROOT + git pull + Pop-Location + vcpkg update + } +} + +if (-not $installed) { + throw "vcpkg install failed after $maxAttempts attempts" } Write-Host "vcpkg install completed successfully" From 610e8cce31b6d6ab6696838e74ea1b9d00ba7d21 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 12 Aug 2026 11:51:25 -0700 Subject: [PATCH 67/80] Add pending changes Signed-off-by: Dan Bailey --- pendingchanges/codecs.txt | 7 +++++++ pendingchanges/delayedloading.txt | 6 ++++++ 2 files changed, 13 insertions(+) create mode 100644 pendingchanges/codecs.txt create mode 100644 pendingchanges/delayedloading.txt diff --git a/pendingchanges/codecs.txt b/pendingchanges/codecs.txt new file mode 100644 index 0000000000..1e049f78e4 --- /dev/null +++ b/pendingchanges/codecs.txt @@ -0,0 +1,7 @@ +OpenVDB: + Features: + - Introduced new openvdb::codecs subsystem and CodecRegistry for stream encoding and decoding. + - Added support for implicit codec conversion fallback (e.g. float to half, scalar to mask) + and ReadDiagnostics. + API changes: + - Extracted Point Data I/O function overloads from PointDataGrid.h into new openvdb/points/PointDataIO.h header. diff --git a/pendingchanges/delayedloading.txt b/pendingchanges/delayedloading.txt new file mode 100644 index 0000000000..6a445fe74f --- /dev/null +++ b/pendingchanges/delayedloading.txt @@ -0,0 +1,6 @@ +OpenVDB: + Highlights: + - Completely removed support for out-of-core / delayed loading across OpenVDB core, + including removal of the OPENVDB_USE_DELAYED_LOADING build option, DelayedLoadMetadata, + and TempFile. This was done to improve performance, reduce codebase complexity and to + entirely eliminate the optional Boost dependency. From fc294a0ab09aee30975564e9bf9aa70eada0b8e3 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 12 Aug 2026 12:27:15 -0700 Subject: [PATCH 68/80] More pending changes Signed-off-by: Dan Bailey --- pendingchanges/ioreadconversion.txt | 5 +++++ pendingchanges/removedelayedloading.txt | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 pendingchanges/ioreadconversion.txt create mode 100644 pendingchanges/removedelayedloading.txt diff --git a/pendingchanges/ioreadconversion.txt b/pendingchanges/ioreadconversion.txt new file mode 100644 index 0000000000..46cddca5db --- /dev/null +++ b/pendingchanges/ioreadconversion.txt @@ -0,0 +1,5 @@ +OpenVDB: + New features: + - io::ReadOptions now supports read-time grid type conversion, allowing a grid + to be read as a different registered type than the one stored in the file + (for example, reading a FloatGrid as a HalfGrid). diff --git a/pendingchanges/removedelayedloading.txt b/pendingchanges/removedelayedloading.txt new file mode 100644 index 0000000000..7d31734800 --- /dev/null +++ b/pendingchanges/removedelayedloading.txt @@ -0,0 +1,20 @@ +OpenVDB: + API changes: + - Delayed loading has been removed. The OPENVDB_USE_DELAYED_LOADING CMake option, + MappedFile support and DelayedLoadMetadata have all been removed. + - io::File::open() and io::Stream constructors that accepted delayed-loading + parameters are deprecated and now behave as their non-delayed-loading + equivalents. + - Out-of-core tree APIs are deprecated, including LeafBuffer::isOutOfCore(), + memUsageIfLoaded() and readNonResidentBuffers(). The latter are now no-ops + and memUsageIfLoaded() forwards to memUsage(). + - Free functions that previously accepted an inCoreOnly parameter are deprecated + in favor of overloads without that parameter. + + Improvements: + - OpenVDB no longer depends on Boost. The boost::iostreams and Boost + interprocess optional dependencies have been removed along with io::TempFile. + + Build: + - Removed the OPENVDB_USE_DELAYED_LOADING CMake option. + - OpenVDB no longer requires or links against Boost. From 53af683310d0ac962b89b47e5e71814c003751a6 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 17 Aug 2026 23:13:05 -0700 Subject: [PATCH 69/80] Make Codec read/write methods const Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/BoolCodec.h | 4 ++-- openvdb/openvdb/codecs/PointDataCodec.h | 4 ++-- openvdb/openvdb/codecs/PointIndexCodec.h | 4 ++-- openvdb/openvdb/codecs/ScalarCodec.h | 4 ++-- openvdb/openvdb/codecs/TopologyCodec.h | 4 ++-- openvdb/openvdb/codecs/ValueMaskCodec.h | 4 ++-- openvdb/openvdb/io/Codec.h | 22 ++++++++++++---------- 7 files changed, 24 insertions(+), 22 deletions(-) diff --git a/openvdb/openvdb/codecs/BoolCodec.h b/openvdb/openvdb/codecs/BoolCodec.h index 82c7ae5a38..3da1839101 100644 --- a/openvdb/openvdb/codecs/BoolCodec.h +++ b/openvdb/openvdb/codecs/BoolCodec.h @@ -110,7 +110,7 @@ struct BoolCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final { GridT& grid = static_cast(*data.grid); @@ -134,7 +134,7 @@ struct BoolCodec final: public TopologyCodec tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); } - void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { const GridT& grid = static_cast(gridBase); diff --git a/openvdb/openvdb/codecs/PointDataCodec.h b/openvdb/openvdb/codecs/PointDataCodec.h index 7cb6219cb7..18484facd1 100644 --- a/openvdb/openvdb/codecs/PointDataCodec.h +++ b/openvdb/openvdb/codecs/PointDataCodec.h @@ -311,7 +311,7 @@ struct PointDataCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final { OPENVDB_ASSERT(dynamic_cast(data.grid.get())); @@ -422,7 +422,7 @@ struct PointDataCodec final: public TopologyCodec } } - void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { const GridT& grid = static_cast(gridBase); bool saveFloatAsHalf = grid.saveFloatAsHalf(); diff --git a/openvdb/openvdb/codecs/PointIndexCodec.h b/openvdb/openvdb/codecs/PointIndexCodec.h index d1694d5b96..5b4d6b62cf 100644 --- a/openvdb/openvdb/codecs/PointIndexCodec.h +++ b/openvdb/openvdb/codecs/PointIndexCodec.h @@ -116,7 +116,7 @@ struct PointIndexCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) const final { OPENVDB_ASSERT(dynamic_cast(data.grid.get())); @@ -143,7 +143,7 @@ struct PointIndexCodec final: public TopologyCodec tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); } - void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { const GridT& grid = static_cast(gridBase); diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index 60b1eb6425..199a376232 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -159,7 +159,7 @@ struct ScalarCodec final: public TopologyCodec } } - void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final { using StorageValueT = typename StorageGridT::TreeType::ValueType; GridT& grid = static_cast(*data.grid); @@ -167,7 +167,7 @@ struct ScalarCodec final: public TopologyCodec internal::scalarCodecReadBuffers(grid, is, options, &topoData.storageBackground); } - void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { // Note: the write body must live inside the negated if constexpr branch // so it is not instantiated for read-only codecs. A bare diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index eee4ead6e5..47f07b7ba0 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -356,7 +356,7 @@ struct TopologyCodec : public io::Codec } void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options, - io::ReadDiagnostics& diagnostics) final + io::ReadDiagnostics& diagnostics) const final { // Warn when a conversion readMode was requested but this codec is a // non-conversion instance (GridT == StorageGridT), meaning no conversion @@ -379,7 +379,7 @@ struct TopologyCodec : public io::Codec internal::topologyCodecReadTopology(*data.grid, is, options, diagnostics, topoData); } - void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { // Disable implementation when read only. The body must live inside the // negated if constexpr branch so it is not instantiated for read-only diff --git a/openvdb/openvdb/codecs/ValueMaskCodec.h b/openvdb/openvdb/codecs/ValueMaskCodec.h index 3d2b8d7bd0..d3e3e2a938 100644 --- a/openvdb/openvdb/codecs/ValueMaskCodec.h +++ b/openvdb/openvdb/codecs/ValueMaskCodec.h @@ -102,7 +102,7 @@ struct ValueMaskCodec final: public TopologyCodec static inline std::string name() { return GridT::gridType(); } - void readBuffers(std::istream& is, Index64 /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final + void readBuffers(std::istream& is, int64_t /*size*/, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) const final { GridT& grid = static_cast(*data.grid); @@ -126,7 +126,7 @@ struct ValueMaskCodec final: public TopologyCodec tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false); } - void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final + void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) const final { const GridT& grid = static_cast(gridBase); diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 70eea1c468..473102b57a 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -284,7 +284,7 @@ struct OPENVDB_API ReadDiagnostics { /// In the read and write methods, downcast the reference back to the concrete /// type to access the extra fields: /// @code -/// void readBuffers(std::istream& is, io::CodecData& data, ...) override { +/// void readBuffers(std::istream& is, int64_t size, io::CodecData& data, ...) const override { /// auto& myData = static_cast(data); /// MyGridType& grid = static_cast(*myData.grid); /// // Use myData.intermediateBuffer, grid, etc. @@ -350,15 +350,15 @@ struct OPENVDB_API CodecData /// @code /// void readTopology(std::istream& is, io::CodecData& data, /// const io::ReadOptions& options, -/// io::ReadDiagnostics& diagnostics) override +/// io::ReadDiagnostics& diagnostics) const override /// { /// MyGridType& grid = static_cast(*data.grid); /// // Deserialize the tree structure into grid... /// } /// -/// void readBuffers(std::istream& is, io::CodecData& data, +/// void readBuffers(std::istream& is, int64_t size, CodecData& data, /// const io::ReadOptions& options, -/// io::ReadDiagnostics& diagnostics) override +/// io::ReadDiagnostics& diagnostics) const override /// { /// if (options.readMode == io::ReadMode::TopologyOnly) return; /// MyGridType& grid = static_cast(*data.grid); @@ -397,7 +397,7 @@ struct OPENVDB_API Codec /// @brief Deserialize the grid topology (tree structure and active-voxel /// mask) from @a is into the grid held by @a data. virtual void readTopology(std::istream& /*is*/, CodecData& /*data*/, - const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { } + const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) const { } /// @brief Deserialize all voxel-value buffers from @a is into the grid /// held by @a data. @@ -410,9 +410,11 @@ struct OPENVDB_API Codec /// restrict the loaded data to the region that intersects it; if the codec /// cannot honour clipping natively, fall back to a post-process and record /// a warning via @a diagnostics. The @c size argument is the number of bytes - /// occupied by the entire readBuffers data section. - virtual void readBuffers(std::istream& /*is*/, Index64 /*size*/, CodecData& /*data*/, - const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { } + /// occupied by the readBuffers data section, measured from the stream + /// position at which this method is called, or -1 when the size is not known + /// (a non-seekable stream, or a stream without grid offsets). + virtual void readBuffers(std::istream& /*is*/, int64_t /*size*/, CodecData& /*data*/, + const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) const { } /// @brief Serialize the grid topology (tree structure and active-voxel /// mask) from @a grid to @a os. @@ -421,14 +423,14 @@ struct OPENVDB_API Codec /// topology as a distinct section that precedes the value buffers in the /// stream. virtual void writeTopology(std::ostream& /*os*/, const GridBase& /*grid*/, - const WriteOptions& /*options*/) { } + const WriteOptions& /*options*/) const { } /// @brief Serialize all voxel-value buffers from @a grid to @a os. /// /// The default implementation is a no-op. Override to write the leaf-node /// value buffers that follow the topology section. virtual void writeBuffers(std::ostream& /*os*/, const GridBase& /*grid*/, - const WriteOptions& /*options*/) { } + const WriteOptions& /*options*/) const { } }; // struct Codec /// @brief A thread-safe, process-global registry that maps codec names to From d940d205f82c5b712690966db861782466f5ad9a Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 18 Aug 2026 15:47:03 -0700 Subject: [PATCH 70/80] Skip readBuffers() logic in ScalarCodec Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/ScalarCodec.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openvdb/openvdb/codecs/ScalarCodec.h b/openvdb/openvdb/codecs/ScalarCodec.h index 199a376232..e181f54df7 100644 --- a/openvdb/openvdb/codecs/ScalarCodec.h +++ b/openvdb/openvdb/codecs/ScalarCodec.h @@ -163,6 +163,11 @@ struct ScalarCodec final: public TopologyCodec { using StorageValueT = typename StorageGridT::TreeType::ValueType; GridT& grid = static_cast(*data.grid); + + // readTopology() has already set the tiles and leaf buffers to the + // background, so there is nothing left to read. + if (options.readMode == io::ReadMode::TopologyOnly) return; + auto& topoData = static_cast&>(data); internal::scalarCodecReadBuffers(grid, is, options, &topoData.storageBackground); } From 50d3e42a35b2b65936d52aa0be9d3e2f8ac79f74 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 18 Aug 2026 15:47:03 -0700 Subject: [PATCH 71/80] Add in-memory conversion for VDB grids with no file offsets Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 173 +++++++++++- openvdb/openvdb/unittest/TestCodec.cc | 372 ++++++++++++++++++++++++++ 2 files changed, 541 insertions(+), 4 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index b783008986..6c0952ff0a 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -6,6 +6,7 @@ #include "File.h" #include +#include // for GridTypes #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include namespace openvdb { @@ -25,6 +27,18 @@ OPENVDB_USE_VERSION_NAMESPACE namespace OPENVDB_VERSION_NAME { namespace io { +namespace { + +/// @brief Convert @a source to the grid type that @a readOptions would have +/// produced had it been read through @a codec (looked up by the caller via +/// the protected @c Archive::findCodec(), since this is a free function). +/// Returns null if no conversion is needed or possible, in which case the +/// caller keeps the original grid and @a diagnostics is set accordingly. +GridBase::Ptr convertGridForReadMode(const GridBase& source, const ReadOptions& readOptions, + Codec* codec, ReadDiagnostics& diagnostics); + +} // anonymous namespace + File::File(const std::string& filename) : Archive() @@ -308,7 +322,40 @@ File::getGrids(const io::ReadOptions& readOptions) const if (!inputHasGridOffsets()) { // If the input file doesn't have grid offsets, then all of the grids // have already been streamed in and stored in mGrids. - ret = mGrids; + const auto& bbox = readOptions.clipBBox; + const bool clip = bbox.isSorted(); + + if (readOptions.readMode == io::ReadMode::Original && !clip) { + // Nothing to convert or clip: preserve pointer identity with mGrids. + ret = mGrids; + } else { + ret.reset(new GridPtrVec); + for (const auto& cachedGrid : *mGrids) { + io::Codec* codec = Archive::findCodec(cachedGrid->type(), readOptions); + GridBase::Ptr grid = + convertGridForReadMode(*cachedGrid, readOptions, codec, mReadDiagnostics); + if (!grid) { + grid = cachedGrid; + if (readOptions.readMode == io::ReadMode::Half || + readOptions.readMode == io::ReadMode::Bool || + readOptions.readMode == io::ReadMode::Mask) + { + OPENVDB_LOG_WARN(mFilename << ": grid \"" << cachedGrid->getName() + << "\" requested a read mode conversion, but no conversion is " + "available for grid type \"" << cachedGrid->type() + << "\"; returning the original type"); + } + } + if (clip) { + if (grid == cachedGrid) { + // Never mutate the cached grid in place; it stays owned by mGrids. + grid = grid->deepCopyGrid(); + } + grid->clipGrid(bbox); + } + ret->push_back(grid); + } + } } else { ret.reset(new GridPtrVec); @@ -455,12 +502,42 @@ File::readGrid(const Name& name, const io::ReadOptions& readOptions) // If a grid with the given name was already read and cached // (along with the entire contents of the file, because the file // doesn't support random access), retrieve and return it. - GridBase::Ptr grid = retrieveCachedGrid(name); - if (grid) { + GridBase::Ptr cachedGrid = retrieveCachedGrid(name); + GridBase::Ptr grid; + if (cachedGrid) { + grid = cachedGrid; + + if (readOptions.readMode == io::ReadMode::TopologyOnly) { + mReadDiagnostics.addWarning(grid->getName(), + "ReadMode::TopologyOnly is not supported for cached grids; " + "reading as original type"); + OPENVDB_LOG_WARN(mFilename << ": grid \"" << grid->getName() + << "\" requested ReadMode::TopologyOnly, but this file has no grid offsets " + "and the grid is already fully cached; returning the original type"); + } else { + io::Codec* codec = Archive::findCodec(grid->type(), readOptions); + GridBase::Ptr converted = + convertGridForReadMode(*grid, readOptions, codec, mReadDiagnostics); + if (converted) { + grid = converted; + } else if (readOptions.readMode == io::ReadMode::Half || + readOptions.readMode == io::ReadMode::Bool || + readOptions.readMode == io::ReadMode::Mask) + { + OPENVDB_LOG_WARN(mFilename << ": grid \"" << grid->getName() + << "\" requested a read mode conversion, but no conversion is " + "available for grid type \"" << grid->type() + << "\"; returning the original type"); + } + } + const auto& bbox = readOptions.clipBBox; const bool clip = bbox.isSorted(); if (clip) { - grid = grid->deepCopyGrid(); + if (grid == cachedGrid) { + // Never mutate the cached grid in place; it stays owned by mNamedGrids. + grid = grid->deepCopyGrid(); + } grid->clipGrid(bbox); } return grid; @@ -608,6 +685,94 @@ File::endName() const } +//////////////////////////////////////// + + +namespace { + +namespace convert_grid_internal { + +/// @brief Convert @a source to the grid type @c ValueConverter::Type, +/// provided the registry-reported @a targetType agrees and the value +/// conversion is legal. Returns null otherwise. +template +inline GridBase::Ptr +convertToBuildType(const GridBase& source, const std::string& targetType) +{ + GridBase::Ptr result; + source.apply([&](const auto& typedSource) { + using SourceGridT = std::decay_t; + using TargetGridT = + typename SourceGridT::template ValueConverter::Type; + // No LeafNode conversion constructor exists from ValueMask to any + // other build type, so exclude it here; nested if constexpr avoids + // instantiating the ambiguous CanConvertType. + if constexpr (!std::is_same_v) { + if constexpr (CanConvertType::value) { + if (TargetGridT::gridType() == targetType) { + result = typename TargetGridT::Ptr(new TargetGridT(typedSource)); + } + } + } + }); + return result; +} + +} // namespace convert_grid_internal + +GridBase::Ptr +convertGridForReadMode(const GridBase& source, const ReadOptions& readOptions, + Codec* codec, ReadDiagnostics& diagnostics) +{ + if (readOptions.readMode != ReadMode::Half && + readOptions.readMode != ReadMode::Bool && + readOptions.readMode != ReadMode::Mask) + { + return GridBase::Ptr(); + } + + const std::string modeStr = + readOptions.readMode == ReadMode::Half ? "Half" : + readOptions.readMode == ReadMode::Bool ? "Bool" : "Mask"; + + CodecData::Ptr codecData = codec ? codec->createData() : CodecData::Ptr(); + const std::string targetType = + (codecData && codecData->grid) ? codecData->grid->type() : std::string(); + + // No conversion codec registered for this grid type (falls through to the + // plain gridType codec), or the registry agrees the type is unchanged. + if (targetType.empty() || targetType == source.type()) { + diagnostics.addWarning(source.getName(), + "ReadMode::" + modeStr + " conversion is not supported for grid type '" + + source.type() + "'; reading as original type"); + return GridBase::Ptr(); + } + + GridBase::Ptr result; + if (readOptions.readMode == ReadMode::Half) { + result = convert_grid_internal::convertToBuildType(source, targetType); + } else if (readOptions.readMode == ReadMode::Bool) { + result = convert_grid_internal::convertToBuildType(source, targetType); + } else { + result = convert_grid_internal::convertToBuildType(source, targetType); + } + + // The registry named a target type that this dispatch could not produce + // (e.g. a custom-registered grid type outside GridTypes, or the + // CanConvertType guard rejected the pair): fall through with a warning + // rather than silently diverging from what the registry reported. + if (!result) { + diagnostics.addWarning(source.getName(), + "ReadMode::" + modeStr + " conversion is not supported for grid type '" + + source.type() + "'; reading as original type"); + } + + return result; +} + +} // anonymous namespace + + } // namespace io } // namespace OPENVDB_VERSION_NAME } // namespace openvdb diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index cfa14de91e..d1727543be 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -3,11 +3,14 @@ #include #include +#include #include #include #include #include #include +#include // for remove() +#include class TestCodec: public ::testing::Test { @@ -348,6 +351,375 @@ TEST_F(TestCodec, testFloatToHalfCodecConversion) std::remove(floatPath.c_str()); } +TEST_F(TestCodec, testFloatToHalfCodecConversionNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatGrid::Ptr srcGrid = FloatGrid::create(3.25f); + srcGrid->setName("float_to_half"); + srcGrid->tree().setValue(Coord(0, 0, 0), 1.0f / 3.0f); + + const std::string path = "test_float_to_half_no_offsets.vdb"; + + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{srcGrid}); + } + + ReadOptions readOptions; + readOptions.readMode = ReadMode::Half; + + { + io::File f(path); + f.open(); + GridPtrVecPtr grids = f.getGrids(readOptions); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(1), grids->size()); + EXPECT_TRUE((*grids)[0]->isType()); + HalfGrid::Ptr halfGrid = gridPtrCast((*grids)[0]); + ASSERT_TRUE(halfGrid); + EXPECT_EQ(halfGrid->tree().getValue(Coord(0, 0, 0)), Half(1.0f / 3.0f)); + EXPECT_EQ(halfGrid->background(), Half(3.25f)); + f.close(); + } + + { + io::File f(path); + f.open(); + GridBase::Ptr grid = f.readGrid(srcGrid->getName(), readOptions); + ASSERT_TRUE(grid); + EXPECT_TRUE(grid->isType()); + HalfGrid::Ptr halfGrid = gridPtrCast(grid); + ASSERT_TRUE(halfGrid); + EXPECT_EQ(halfGrid->tree().getValue(Coord(0, 0, 0)), Half(1.0f / 3.0f)); + EXPECT_EQ(halfGrid->background(), Half(3.25f)); + f.close(); + } + + std::remove(path.c_str()); +} + +TEST_F(TestCodec, testBoolAndMaskConversionNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + auto runCase = [](ReadMode mode) { + FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f); + srcGrid->setName("float_grid"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + const std::string path = + "test_bool_mask_no_offsets_" + std::to_string(int(mode)) + ".vdb"; + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{srcGrid}); + } + + ReadOptions readOptions; + readOptions.readMode = mode; + + auto checkGrid = [&](const GridBase::Ptr& grid) { + ASSERT_TRUE(grid); + if (mode == ReadMode::Bool) { + EXPECT_TRUE(grid->isType()); + BoolGrid::Ptr dstGrid = gridPtrCast(grid); + ASSERT_TRUE(dstGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(dstGrid->tree())); + for (BoolGrid::ValueOnCIter it = dstGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, true); + } + } else { + EXPECT_TRUE(grid->isType()); + MaskGrid::Ptr dstGrid = gridPtrCast(grid); + ASSERT_TRUE(dstGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(dstGrid->tree())); + for (MaskGrid::ValueOnCIter it = dstGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, true); + } + } + }; + + { + io::File f(path); + f.open(); + GridPtrVecPtr grids = f.getGrids(readOptions); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(1), grids->size()); + checkGrid((*grids)[0]); + f.close(); + } + + { + io::File f(path); + f.open(); + checkGrid(f.readGrid(srcGrid->getName(), readOptions)); + f.close(); + } + + std::remove(path.c_str()); + }; + + runCase(ReadMode::Bool); + runCase(ReadMode::Mask); +} + +TEST_F(TestCodec, testVec3FallsBackWithWarningNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + Vec3SGrid::Ptr srcGrid = Vec3SGrid::create(Vec3s(0.0f)); + srcGrid->setName("vec3_grid"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), Vec3s(1.0f), true); + + const std::string path = "test_vec3_fallback_no_offsets.vdb"; + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{srcGrid}); + } + + ReadOptions readOptions; + readOptions.readMode = ReadMode::Bool; + + // Determine the warning the offsets path produces for the same request, + // so the two paths can be compared for parity. + std::string offsetsWarning; + { + const std::string offsetsPath = "test_vec3_fallback_offsets.vdb"; + { + io::File f(offsetsPath); + f.write(GridPtrVec{srcGrid}); + } + io::File f(offsetsPath); + f.open(); + f.enableReadDiagnostics(); + GridBase::Ptr grid = f.readGrid(srcGrid->getName(), readOptions); + ASSERT_TRUE(grid); + EXPECT_TRUE(grid->isType()); + ASSERT_EQ(size_t(1), f.readDiagnostics().diagnostics().size()); + offsetsWarning = f.readDiagnostics().diagnostics()[0].message; + f.close(); + std::remove(offsetsPath.c_str()); + } + + { + io::File f(path); + f.open(); + f.enableReadDiagnostics(); + GridBase::Ptr grid; + EXPECT_NO_THROW(grid = f.readGrid(srcGrid->getName(), readOptions)); + ASSERT_TRUE(grid); + EXPECT_TRUE(grid->isType()); + ASSERT_EQ(size_t(1), f.readDiagnostics().diagnostics().size()); + EXPECT_EQ(f.readDiagnostics().diagnostics()[0].message, offsetsWarning); + f.close(); + } + + std::remove(path.c_str()); +} + +TEST_F(TestCodec, testOffsetsAndNoOffsetsParity) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatGrid::Ptr srcGrid = FloatGrid::create(2.0f); + srcGrid->setName("parity_grid"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 0.25f, true); + + const std::string offsetsPath = "test_parity_offsets.vdb"; + const std::string noOffsetsPath = "test_parity_no_offsets.vdb"; + { + io::File f(offsetsPath); + f.write(GridPtrVec{srcGrid}); + } + { + std::ofstream os(noOffsetsPath, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{srcGrid}); + } + + ReadOptions readOptions; + readOptions.readMode = ReadMode::Half; + + HalfGrid::Ptr offsetsGrid; + { + io::File f(offsetsPath); + f.open(); + offsetsGrid = gridPtrCast(f.readGrid(srcGrid->getName(), readOptions)); + f.close(); + } + HalfGrid::Ptr noOffsetsGrid; + { + io::File f(noOffsetsPath); + f.open(); + noOffsetsGrid = gridPtrCast(f.readGrid(srcGrid->getName(), readOptions)); + f.close(); + } + + ASSERT_TRUE(offsetsGrid); + ASSERT_TRUE(noOffsetsGrid); + EXPECT_EQ(offsetsGrid->type(), noOffsetsGrid->type()); + EXPECT_TRUE(offsetsGrid->tree().hasSameTopology(noOffsetsGrid->tree())); + for (HalfGrid::ValueOnCIter it = offsetsGrid->cbeginValueOn(); it; ++it) { + EXPECT_EQ(*it, noOffsetsGrid->tree().getValue(it.getCoord())); + } + + std::remove(offsetsPath.c_str()); + std::remove(noOffsetsPath.c_str()); +} + +TEST_F(TestCodec, testClipBBoxInGetGridsNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f); + srcGrid->setName("clip_grid"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + const std::string path = "test_clip_getgrids_no_offsets.vdb"; + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{srcGrid}); + } + + const BBoxd clipBBox(Vec3d(0.0), Vec3d(3.5)); + auto srcClipped = tools::clip(*srcGrid, clipBBox); + + io::File f(path); + f.open(); + + ReadOptions clipOptions; + clipOptions.clipBBox = clipBBox; + + { + GridPtrVecPtr grids = f.getGrids(clipOptions); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(1), grids->size()); + FloatGrid::Ptr clipped = gridPtrCast((*grids)[0]); + ASSERT_TRUE(clipped); + EXPECT_TRUE(srcClipped->tree().hasSameTopology(clipped->tree())); + } + + // mGrids must be unmutated: a default-options call still sees the full topology. + { + GridPtrVecPtr grids = f.getGrids(); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(1), grids->size()); + FloatGrid::Ptr full = gridPtrCast((*grids)[0]); + ASSERT_TRUE(full); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(full->tree())); + } + + f.close(); + std::remove(path.c_str()); +} + +TEST_F(TestCodec, testClipAndConversionTogetherNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f); + srcGrid->setName("clip_convert_grid"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + const std::string path = "test_clip_and_convert_no_offsets.vdb"; + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{srcGrid}); + } + + const BBoxd clipBBox(Vec3d(0.0), Vec3d(3.5)); + auto srcClipped = tools::clip(*srcGrid, clipBBox); + + ReadOptions readOptions; + readOptions.readMode = ReadMode::Half; + readOptions.clipBBox = clipBBox; + + { + io::File f(path); + f.open(); + GridPtrVecPtr grids = f.getGrids(readOptions); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(1), grids->size()); + EXPECT_TRUE((*grids)[0]->isType()); + HalfGrid::Ptr clipped = gridPtrCast((*grids)[0]); + ASSERT_TRUE(clipped); + EXPECT_TRUE(srcClipped->tree().hasSameTopology(clipped->tree())); + f.close(); + } + + { + io::File f(path); + f.open(); + GridBase::Ptr grid = f.readGrid(srcGrid->getName(), readOptions); + ASSERT_TRUE(grid); + EXPECT_TRUE(grid->isType()); + HalfGrid::Ptr clipped = gridPtrCast(grid); + ASSERT_TRUE(clipped); + EXPECT_TRUE(srcClipped->tree().hasSameTopology(clipped->tree())); + f.close(); + } + + std::remove(path.c_str()); +} + +TEST_F(TestCodec, testTopologyOnlyWarnsAndIgnoresNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f); + srcGrid->setName("topology_only_grid"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + const std::string path = "test_topology_only_no_offsets.vdb"; + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{srcGrid}); + } + + ReadOptions readOptions; + readOptions.readMode = ReadMode::TopologyOnly; + + io::File f(path); + f.open(); + f.enableReadDiagnostics(); + + GridBase::Ptr grid; + EXPECT_NO_THROW(grid = f.readGrid(srcGrid->getName(), readOptions)); + ASSERT_TRUE(grid); + EXPECT_TRUE(grid->isType()); + EXPECT_EQ(size_t(1), f.readDiagnostics().diagnostics().size()); + + f.close(); + std::remove(path.c_str()); +} + template void testConvertCodecImpl() { From e4dbaaf905085c41685c72dcb30da873fb0c1cc4 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 24 Aug 2026 21:46:27 -0700 Subject: [PATCH 72/80] Fix clipping and in-memory conversion for instanced grids Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 161 ++++++++----- openvdb/openvdb/io/File.h | 6 + openvdb/openvdb/unittest/TestCodec.cc | 312 ++++++++++++++++++++++++++ 3 files changed, 422 insertions(+), 57 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 6c0952ff0a..2f6cea1f14 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,9 @@ namespace { GridBase::Ptr convertGridForReadMode(const GridBase& source, const ReadOptions& readOptions, Codec* codec, ReadDiagnostics& diagnostics); +/// @brief Return the name used in diagnostics and log messages for @a mode. +std::string readModeName(ReadMode mode); + } // anonymous namespace @@ -330,28 +334,43 @@ File::getGrids(const io::ReadOptions& readOptions) const ret = mGrids; } else { ret.reset(new GridPtrVec); + + // Instances (grids sharing a source tree) share the converted tree + // too, unless instancing is disabled. Under a clip, share only + // when transforms agree, since a clip depends on each grid's own + // transform. + const bool shareConvertedTrees = isInstancingEnabled() && + readOptions.readMode != io::ReadMode::MetadataOnly; + struct Resolved { GridBase::Ptr grid; math::Transform::ConstPtr transform; }; + std::map resolvedBySourceTree; + for (const auto& cachedGrid : *mGrids) { - io::Codec* codec = Archive::findCodec(cachedGrid->type(), readOptions); - GridBase::Ptr grid = - convertGridForReadMode(*cachedGrid, readOptions, codec, mReadDiagnostics); - if (!grid) { - grid = cachedGrid; - if (readOptions.readMode == io::ReadMode::Half || - readOptions.readMode == io::ReadMode::Bool || - readOptions.readMode == io::ReadMode::Mask) + const TreeBase* sourceTree = &cachedGrid->constBaseTree(); + GridBase::Ptr grid; + + if (shareConvertedTrees) { + auto it = resolvedBySourceTree.find(sourceTree); + if (it != resolvedBySourceTree.end() && + (!clip || *it->second.transform == cachedGrid->transform())) { - OPENVDB_LOG_WARN(mFilename << ": grid \"" << cachedGrid->getName() - << "\" requested a read mode conversion, but no conversion is " - "available for grid type \"" << cachedGrid->type() - << "\"; returning the original type"); + const GridBase::Ptr& resolved = it->second.grid; + grid = resolved->copyGridWithNewTree(); + grid->clearMetadata(); + grid->insertMeta(*cachedGrid); + grid->setTransform(cachedGrid->transformPtr()); + grid->setTree(resolved->baseTreePtr()); + ret->push_back(grid); + continue; } } - if (clip) { - if (grid == cachedGrid) { - // Never mutate the cached grid in place; it stays owned by mGrids. - grid = grid->deepCopyGrid(); - } - grid->clipGrid(bbox); + + grid = resolveCachedGrid(cachedGrid, readOptions, mReadDiagnostics); + + if (shareConvertedTrees) { + // Keep the first-seen entry as canonical, so a later + // mismatched transform under clip doesn't overwrite it. + resolvedBySourceTree.try_emplace( + sourceTree, Resolved{grid, cachedGrid->transformPtr()}); } ret->push_back(grid); } @@ -505,42 +524,7 @@ File::readGrid(const Name& name, const io::ReadOptions& readOptions) GridBase::Ptr cachedGrid = retrieveCachedGrid(name); GridBase::Ptr grid; if (cachedGrid) { - grid = cachedGrid; - - if (readOptions.readMode == io::ReadMode::TopologyOnly) { - mReadDiagnostics.addWarning(grid->getName(), - "ReadMode::TopologyOnly is not supported for cached grids; " - "reading as original type"); - OPENVDB_LOG_WARN(mFilename << ": grid \"" << grid->getName() - << "\" requested ReadMode::TopologyOnly, but this file has no grid offsets " - "and the grid is already fully cached; returning the original type"); - } else { - io::Codec* codec = Archive::findCodec(grid->type(), readOptions); - GridBase::Ptr converted = - convertGridForReadMode(*grid, readOptions, codec, mReadDiagnostics); - if (converted) { - grid = converted; - } else if (readOptions.readMode == io::ReadMode::Half || - readOptions.readMode == io::ReadMode::Bool || - readOptions.readMode == io::ReadMode::Mask) - { - OPENVDB_LOG_WARN(mFilename << ": grid \"" << grid->getName() - << "\" requested a read mode conversion, but no conversion is " - "available for grid type \"" << grid->type() - << "\"; returning the original type"); - } - } - - const auto& bbox = readOptions.clipBBox; - const bool clip = bbox.isSorted(); - if (clip) { - if (grid == cachedGrid) { - // Never mutate the cached grid in place; it stays owned by mNamedGrids. - grid = grid->deepCopyGrid(); - } - grid->clipGrid(bbox); - } - return grid; + return resolveCachedGrid(cachedGrid, readOptions, mReadDiagnostics); } NameMapCIter it = findDescriptor(name); @@ -720,6 +704,20 @@ convertToBuildType(const GridBase& source, const std::string& targetType) } // namespace convert_grid_internal +/// @brief Return the name used in diagnostics and log messages for @a mode. +std::string +readModeName(ReadMode mode) +{ + switch (mode) { + case ReadMode::Half: return "Half"; + case ReadMode::Bool: return "Bool"; + case ReadMode::Mask: return "Mask"; + case ReadMode::TopologyOnly: return "TopologyOnly"; + case ReadMode::MetadataOnly: return "MetadataOnly"; + default: return "Original"; + } +} + GridBase::Ptr convertGridForReadMode(const GridBase& source, const ReadOptions& readOptions, Codec* codec, ReadDiagnostics& diagnostics) @@ -731,9 +729,7 @@ convertGridForReadMode(const GridBase& source, const ReadOptions& readOptions, return GridBase::Ptr(); } - const std::string modeStr = - readOptions.readMode == ReadMode::Half ? "Half" : - readOptions.readMode == ReadMode::Bool ? "Bool" : "Mask"; + const std::string modeStr = readModeName(readOptions.readMode); CodecData::Ptr codecData = codec ? codec->createData() : CodecData::Ptr(); const std::string targetType = @@ -773,6 +769,57 @@ convertGridForReadMode(const GridBase& source, const ReadOptions& readOptions, } // anonymous namespace +GridBase::Ptr +File::resolveCachedGrid(const GridBase::Ptr& cachedGrid, const io::ReadOptions& readOptions, + ReadDiagnostics& diagnostics) const +{ + if (readOptions.readMode == ReadMode::MetadataOnly) { + return cachedGrid->copyGridWithNewTree(); + } + + GridBase::Ptr grid = cachedGrid; + + if (readOptions.readMode == ReadMode::TopologyOnly) { + diagnostics.addWarning(grid->getName(), + "ReadMode::TopologyOnly is not supported for grids cached from a file " + "without grid offsets; returning the original grid with values intact"); + OPENVDB_LOG_WARN(mFilename << ": grid \"" << grid->getName() + << "\" requested ReadMode::TopologyOnly, but this file has no grid offsets " + "and the grid is already fully cached; returning the original grid " + "with values intact"); + } else { + io::Codec* codec = Archive::findCodec(grid->type(), readOptions); + GridBase::Ptr converted = convertGridForReadMode(*grid, readOptions, codec, diagnostics); + if (converted) { + grid = converted; + } else if (readOptions.readMode == ReadMode::Half || + readOptions.readMode == ReadMode::Bool || + readOptions.readMode == ReadMode::Mask) + { + const std::string modeStr = readModeName(readOptions.readMode); + OPENVDB_LOG_WARN(mFilename << ": grid \"" << grid->getName() + << "\" requested ReadMode::" << modeStr << ", but no conversion is " + "available for grid type \"" << grid->type() + << "\"; returning the original type"); + } + } + + const auto& bbox = readOptions.clipBBox; + if (bbox.isSorted()) { + if (grid == cachedGrid) { + // Don't mutate the cached grid in place, it stays owned by the caller. + grid = grid->deepCopyGrid(); + } + grid->clipGrid(bbox); + diagnostics.addWarning(cachedGrid->getName(), + "bounding box clipping was applied as a post-process because the grid " + "was cached from a file without grid offsets"); + } + + return grid; +} + + } // namespace io } // namespace OPENVDB_VERSION_NAME } // namespace openvdb diff --git a/openvdb/openvdb/io/File.h b/openvdb/openvdb/io/File.h index 920b86a50f..b8934b4463 100644 --- a/openvdb/openvdb/io/File.h +++ b/openvdb/openvdb/io/File.h @@ -157,6 +157,12 @@ class OPENVDB_API File: public Archive /// @throw KeyError if no grid with the given name exists in this file. GridBase::Ptr retrieveCachedGrid(const Name&) const; + /// @brief Return the grid that @a readOptions asks for, given the already + /// cached @a cachedGrid. Doesn't mutate @a cachedGrid, and returns it + /// unchanged when no option applies. + GridBase::Ptr resolveCachedGrid(const GridBase::Ptr& cachedGrid, + const io::ReadOptions& readOptions, ReadDiagnostics& diagnostics) const; + void writeGrids(const GridCPtrVec&, const MetaMap&, const io::WriteOptions&) const; MetaMap::Ptr fileMetadata(); diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index d1727543be..7522e13075 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -605,6 +605,7 @@ TEST_F(TestCodec, testClipBBoxInGetGridsNoGridOffsets) io::File f(path); f.open(); + f.enableReadDiagnostics(); ReadOptions clipOptions; clipOptions.clipBBox = clipBBox; @@ -616,6 +617,7 @@ TEST_F(TestCodec, testClipBBoxInGetGridsNoGridOffsets) FloatGrid::Ptr clipped = gridPtrCast((*grids)[0]); ASSERT_TRUE(clipped); EXPECT_TRUE(srcClipped->tree().hasSameTopology(clipped->tree())); + EXPECT_EQ(size_t(1), f.readDiagnostics().diagnostics().size()); } // mGrids must be unmutated: a default-options call still sees the full topology. @@ -632,6 +634,46 @@ TEST_F(TestCodec, testClipBBoxInGetGridsNoGridOffsets) std::remove(path.c_str()); } +TEST_F(TestCodec, testClipOnOffsetsFileRecordsNoDiagnostic) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f); + srcGrid->setName("clip_grid_offsets"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + const std::string path = "test_clip_getgrids_offsets.vdb"; + { + io::File f(path); + f.write(GridPtrVec{srcGrid}); + } + + const BBoxd clipBBox(Vec3d(0.0), Vec3d(3.5)); + + ReadOptions clipOptions; + clipOptions.clipBBox = clipBBox; + + io::File f(path); + f.open(); + f.enableReadDiagnostics(); + + GridPtrVecPtr grids = f.getGrids(clipOptions); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(1), grids->size()); + EXPECT_EQ(size_t(0), f.readDiagnostics().diagnostics().size()); + + GridBase::Ptr grid = f.readGrid(srcGrid->getName(), clipOptions); + ASSERT_TRUE(grid); + EXPECT_EQ(size_t(0), f.readDiagnostics().diagnostics().size()); + + f.close(); + std::remove(path.c_str()); +} + TEST_F(TestCodec, testClipAndConversionTogetherNoGridOffsets) { using namespace openvdb; @@ -660,6 +702,7 @@ TEST_F(TestCodec, testClipAndConversionTogetherNoGridOffsets) { io::File f(path); f.open(); + f.enableReadDiagnostics(); GridPtrVecPtr grids = f.getGrids(readOptions); ASSERT_TRUE(grids); ASSERT_EQ(size_t(1), grids->size()); @@ -667,6 +710,8 @@ TEST_F(TestCodec, testClipAndConversionTogetherNoGridOffsets) HalfGrid::Ptr clipped = gridPtrCast((*grids)[0]); ASSERT_TRUE(clipped); EXPECT_TRUE(srcClipped->tree().hasSameTopology(clipped->tree())); + // Only the clip diagnostic is expected, the Half conversion succeeded. + EXPECT_EQ(size_t(1), f.readDiagnostics().diagnostics().size()); f.close(); } @@ -720,6 +765,273 @@ TEST_F(TestCodec, testTopologyOnlyWarnsAndIgnoresNoGridOffsets) std::remove(path.c_str()); } +TEST_F(TestCodec, testGetGridsTopologyOnlyWarnsNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f); + srcGrid->setName("topology_only_grid"); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + const std::string path = "test_getgrids_topology_only_no_offsets.vdb"; + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{srcGrid}); + } + + ReadOptions readOptions; + readOptions.readMode = ReadMode::TopologyOnly; + + // Pin getGrids() and readGrid() to the same diagnostic message. + std::string readGridMessage; + { + io::File f(path); + f.open(); + f.enableReadDiagnostics(); + GridBase::Ptr grid; + EXPECT_NO_THROW(grid = f.readGrid(srcGrid->getName(), readOptions)); + ASSERT_TRUE(grid); + EXPECT_TRUE(grid->isType()); + ASSERT_EQ(size_t(1), f.readDiagnostics().diagnostics().size()); + readGridMessage = f.readDiagnostics().diagnostics()[0].message; + f.close(); + } + + { + io::File f(path); + f.open(); + f.enableReadDiagnostics(); + GridPtrVecPtr grids; + EXPECT_NO_THROW(grids = f.getGrids(readOptions)); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(1), grids->size()); + EXPECT_TRUE((*grids)[0]->isType()); + ASSERT_EQ(size_t(1), f.readDiagnostics().diagnostics().size()); + EXPECT_EQ(f.readDiagnostics().diagnostics()[0].message, readGridMessage); + f.close(); + } + + std::remove(path.c_str()); +} + +TEST_F(TestCodec, testMetadataOnlyNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f); + srcGrid->setName("metadata_only_grid"); + srcGrid->insertMeta("author", StringMetadata("Einstein")); + srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + const std::string path = "test_metadata_only_no_offsets.vdb"; + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{srcGrid}); + } + + ReadOptions readOptions; + readOptions.readMode = ReadMode::MetadataOnly; + + GridBase::Ptr expected; + { + io::File f(path); + f.open(); + expected = f.readGridMetadata(srcGrid->getName()); + f.close(); + } + + { + io::File f(path); + f.open(); + f.enableReadDiagnostics(); + GridPtrVecPtr grids = f.getGrids(readOptions); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(1), grids->size()); + GridBase::Ptr grid = (*grids)[0]; + EXPECT_TRUE(grid->isType()); + EXPECT_TRUE(gridPtrCast(grid)->tree().empty()); + EXPECT_EQ(std::string("Einstein"), grid->metaValue("author")); + EXPECT_EQ(expected->transform(), grid->transform()); + EXPECT_EQ(size_t(0), f.readDiagnostics().diagnostics().size()); + f.close(); + } + + { + io::File f(path); + f.open(); + f.enableReadDiagnostics(); + GridBase::Ptr grid = f.readGrid(srcGrid->getName(), readOptions); + ASSERT_TRUE(grid); + EXPECT_TRUE(grid->isType()); + EXPECT_TRUE(gridPtrCast(grid)->tree().empty()); + EXPECT_EQ(std::string("Einstein"), grid->metaValue("author")); + EXPECT_EQ(expected->transform(), grid->transform()); + EXPECT_EQ(size_t(0), f.readDiagnostics().diagnostics().size()); + f.close(); + } + + std::remove(path.c_str()); +} + +TEST_F(TestCodec, testClipInstancingNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatTree::Ptr tree(new FloatTree(0.0f)); + tree->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + GridBase::Ptr grid1 = createGrid(tree); + grid1->setName("parent"); + GridBase::Ptr grid2 = createGrid(tree); // instance of grid1 + grid2->setName("instance"); + + const std::string path = "test_clip_instancing_no_offsets.vdb"; + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{grid1, grid2}); + } + + ReadOptions readOptions; + readOptions.clipBBox = BBoxd(Vec3d(0.0), Vec3d(3.5)); + + io::File f(path); + f.open(); + GridPtrVecPtr grids = f.getGrids(readOptions); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(2), grids->size()); + + FloatGrid::Ptr resultParent = gridPtrCast(findGridByName(*grids, "parent")); + FloatGrid::Ptr resultInstance = gridPtrCast(findGridByName(*grids, "instance")); + ASSERT_TRUE(resultParent); + ASSERT_TRUE(resultInstance); + + // Same transform, so the clipped tree is shared. + EXPECT_EQ(resultParent->treePtr(), resultInstance->treePtr()); + + f.close(); + std::remove(path.c_str()); +} + +TEST_F(TestCodec, testConversionInstancingNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatTree::Ptr tree(new FloatTree(0.0f)); + tree->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + GridBase::Ptr grid1 = createGrid(tree); + grid1->setName("parent"); + GridBase::Ptr grid2 = createGrid(tree); // instance of grid1 + grid2->setName("instance"); + + const std::string path = "test_conversion_instancing_no_offsets.vdb"; + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{grid1, grid2}); + } + + ReadOptions readOptions; + readOptions.readMode = ReadMode::Half; + + { + io::File f(path); + f.open(); + GridPtrVecPtr grids = f.getGrids(readOptions); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(2), grids->size()); + + HalfGrid::Ptr resultParent = gridPtrCast(findGridByName(*grids, "parent")); + HalfGrid::Ptr resultInstance = gridPtrCast(findGridByName(*grids, "instance")); + ASSERT_TRUE(resultParent); + ASSERT_TRUE(resultInstance); + + // Conversion doesn't break instancing, both results share the tree. + EXPECT_EQ(resultParent->treePtr(), resultInstance->treePtr()); + f.close(); + } + + // Instancing disabled, so the two results don't share a tree. + { + io::File f(path); + f.setInstancingEnabled(false); + f.open(); + GridPtrVecPtr grids = f.getGrids(readOptions); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(2), grids->size()); + + HalfGrid::Ptr resultParent = gridPtrCast(findGridByName(*grids, "parent")); + HalfGrid::Ptr resultInstance = gridPtrCast(findGridByName(*grids, "instance")); + ASSERT_TRUE(resultParent); + ASSERT_TRUE(resultInstance); + EXPECT_NE(resultParent->treePtr(), resultInstance->treePtr()); + f.close(); + } + + std::remove(path.c_str()); +} + +TEST_F(TestCodec, testInstanceKeepsOwnTransformNoGridOffsets) +{ + using namespace openvdb; + using namespace openvdb::io; + + CodecRegistry::clear(); + io::internal::initialize(); + + FloatTree::Ptr tree(new FloatTree(0.0f)); + tree->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true); + + GridBase::Ptr grid1 = createGrid(tree); + grid1->setName("parent"); + grid1->setTransform(math::Transform::createLinearTransform(1.0)); + GridBase::Ptr grid2 = createGrid(tree); // instance of grid1 + grid2->setName("instance"); + grid2->setTransform(math::Transform::createLinearTransform(2.0)); + + const std::string path = "test_instance_own_transform_no_offsets.vdb"; + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{grid1, grid2}); + } + + ReadOptions readOptions; + readOptions.readMode = ReadMode::Half; + + io::File f(path); + f.open(); + GridPtrVecPtr grids = f.getGrids(readOptions); + ASSERT_TRUE(grids); + ASSERT_EQ(size_t(2), grids->size()); + + HalfGrid::Ptr resultParent = gridPtrCast(findGridByName(*grids, "parent")); + HalfGrid::Ptr resultInstance = gridPtrCast(findGridByName(*grids, "instance")); + ASSERT_TRUE(resultParent); + ASSERT_TRUE(resultInstance); + + EXPECT_EQ(resultParent->treePtr(), resultInstance->treePtr()); + EXPECT_EQ(1.0, resultParent->voxelSize()[0]); + EXPECT_EQ(2.0, resultInstance->voxelSize()[0]); + + f.close(); + std::remove(path.c_str()); +} + template void testConvertCodecImpl() { From af6988fcea22a0e1be3da9b454ec1de210887ac0 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 26 Aug 2026 21:30:16 -0700 Subject: [PATCH 73/80] Fix TopologyOnly mode for BoolGrids and MaskGrids Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/TopologyCodec.h | 15 ++-- openvdb/openvdb/unittest/TestCodec.cc | 115 +++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 7 deletions(-) diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 47f07b7ba0..10816609ab 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -312,17 +312,18 @@ void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::R internal::setTilesToBackground(grid.tree()); // allocate leaf buffers in parallel and fill with the background value; // ReadTopologyOp uses PartialCreate which leaves buffers unallocated. - const auto background = grid.tree().root().background(); - tree::LeafManager leafManager(grid.tree()); - leafManager.foreach([&background](auto& leaf, size_t) { - using LeafType = std::decay_t; - if constexpr (!std::is_same_v) { + // (skip BoolGrids/MaskGrids whose buffers are bit masks that are always + // allocated and so provide no empty()/allocate()) + if constexpr (!std::is_same_v) { + const auto background = grid.tree().root().background(); + tree::LeafManager leafManager(grid.tree()); + leafManager.foreach([&background](auto& leaf, size_t) { if (leaf.buffer().empty()) { leaf.buffer().allocate(); leaf.buffer().fill(background); } - } - }); + }); + } return; } } diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index 7522e13075..260cbb14e7 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -1196,3 +1196,118 @@ TEST_F(TestCodec, testInactiveValuesAfterReadBuffers) std::remove(path.c_str()); } } + +// A MaskGrid has no ValueMask specialization of InternalNode or RootNode, so its +// tiles carry a value separate from the active state. TopologyOnly resets those +// values to the background, which must leave the active topology untouched. +TEST_F(TestCodec, testTopologyOnlyPreservesMaskTiles) +{ + using namespace openvdb; + using namespace openvdb::io; + + const std::string gridName = "mask_tiles"; + const std::string path = "testTopologyOnlyMaskTiles.vdb"; + + // Active tiles at both internal levels and at the root, plus a region of + // individual voxels so that leaf nodes are exercised too. + MaskGrid::Ptr src = MaskGrid::create(); + src->setName(gridName); + src->tree().addTile(/*level=*/1, Coord(0), true, /*active=*/true); + src->tree().addTile(/*level=*/2, Coord(4096), true, /*active=*/true); + src->tree().addTile(/*level=*/3, Coord(-8192), true, /*active=*/true); + src->fill(CoordBBox(Coord(1024), Coord(1030)), true, /*active=*/true); + + const Index64 srcActiveVoxels = src->activeVoxelCount(); + const Index64 srcActiveTiles = src->tree().activeTileCount(); + ASSERT_TRUE(srcActiveTiles > 0); + + { + io::File f(path); + f.write(GridPtrVec{src}); + } + + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + MaskGrid::Ptr readTopo, readOriginal; + { + io::File f(path); + f.open(); + readTopo = gridPtrCast(f.readGrid(gridName, topoOpts)); + readOriginal = gridPtrCast(f.readGrid(gridName, ReadOptions{})); + f.close(); + } + ASSERT_TRUE(readTopo); + ASSERT_TRUE(readOriginal); + + // TopologyOnly must not lose the active tiles. + EXPECT_EQ(readTopo->tree().activeTileCount(), srcActiveTiles); + EXPECT_EQ(readTopo->activeVoxelCount(), srcActiveVoxels); + EXPECT_TRUE(src->tree().hasSameTopology(readTopo->tree())); + EXPECT_TRUE(readOriginal->tree().hasSameTopology(readTopo->tree())); + + // Leaf voxels keep their values because value and active state share one bit. + auto topoAcc = readTopo->getConstAccessor(); + for (MaskGrid::ValueOnCIter it = readOriginal->cbeginValueOn(); it; ++it) { + if (it.isVoxelValue()) EXPECT_TRUE(topoAcc.isValueOn(it.getCoord())); + } + + // Tile values are reset to the background, matching a TopologyCopy, while the + // tiles stay active. + for (MaskGrid::ValueOnCIter it = readTopo->cbeginValueOn(); it; ++it) { + if (!it.isVoxelValue()) EXPECT_FALSE(*it); + } + + std::remove(path.c_str()); +} + +// A BoolGrid stores the value and the active state separately, so TopologyOnly is +// expected to discard values while keeping topology. +TEST_F(TestCodec, testTopologyOnlyClearsBoolTiles) +{ + using namespace openvdb; + using namespace openvdb::io; + + const std::string gridName = "bool_tiles"; + const std::string path = "testTopologyOnlyBoolTiles.vdb"; + + // Active tiles at both internal levels, plus a region of individual voxels. + BoolGrid::Ptr src = BoolGrid::create(/*background=*/false); + src->setName(gridName); + src->tree().addTile(/*level=*/1, Coord(0), true, /*active=*/true); + src->tree().addTile(/*level=*/2, Coord(4096), true, /*active=*/true); + src->fill(CoordBBox(Coord(1024), Coord(1030)), true, /*active=*/true); + + const Index64 srcActiveVoxels = src->activeVoxelCount(); + const Index64 srcActiveTiles = src->tree().activeTileCount(); + ASSERT_TRUE(srcActiveTiles > 0); + + { + io::File f(path); + f.write(GridPtrVec{src}); + } + + ReadOptions topoOpts; + topoOpts.readMode = ReadMode::TopologyOnly; + + BoolGrid::Ptr readTopo; + { + io::File f(path); + f.open(); + readTopo = gridPtrCast(f.readGrid(gridName, topoOpts)); + f.close(); + } + ASSERT_TRUE(readTopo); + + // Topology is preserved, including active tiles. + EXPECT_EQ(readTopo->tree().activeTileCount(), srcActiveTiles); + EXPECT_EQ(readTopo->activeVoxelCount(), srcActiveVoxels); + EXPECT_TRUE(src->tree().hasSameTopology(readTopo->tree())); + + // Values are discarded: tiles are reset to the background even where active. + for (BoolGrid::ValueAllCIter it = readTopo->cbeginValueAll(); it; ++it) { + if (!it.isVoxelValue()) EXPECT_FALSE(*it); + } + + std::remove(path.c_str()); +} From 3a86689535259919e93be9a7499f2f3585bf9eca Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Wed, 26 Aug 2026 22:29:57 -0700 Subject: [PATCH 74/80] Fix MaskGrid conversion for files with no offsets Signed-off-by: Dan Bailey --- openvdb/openvdb/io/File.cc | 100 ++++++++++++++------------ openvdb/openvdb/unittest/TestCodec.cc | 54 ++++++++++++++ 2 files changed, 109 insertions(+), 45 deletions(-) diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index 2f6cea1f14..6d543b9821 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -34,9 +34,10 @@ namespace { /// produced had it been read through @a codec (looked up by the caller via /// the protected @c Archive::findCodec(), since this is a free function). /// Returns null if no conversion is needed or possible, in which case the -/// caller keeps the original grid and @a diagnostics is set accordingly. +/// caller keeps the original grid. A conversion that was requested but cannot +/// be done is reported to @a diagnostics and logged against @a filename. GridBase::Ptr convertGridForReadMode(const GridBase& source, const ReadOptions& readOptions, - Codec* codec, ReadDiagnostics& diagnostics); + Codec* codec, ReadDiagnostics& diagnostics, const std::string& filename); /// @brief Return the name used in diagnostics and log messages for @a mode. std::string readModeName(ReadMode mode); @@ -677,25 +678,42 @@ namespace { namespace convert_grid_internal { /// @brief Convert @a source to the grid type @c ValueConverter::Type, -/// provided the registry-reported @a targetType agrees and the value -/// conversion is legal. Returns null otherwise. +/// provided the registry-reported @a targetType agrees and the value conversion +/// is legal. Sets @a alreadyTargetType when @a source has that build type +/// already. Returns null in both cases, so the caller keeps the original grid. template inline GridBase::Ptr -convertToBuildType(const GridBase& source, const std::string& targetType) +convertToTargetType(const GridBase& source, const std::string& targetType, + bool& alreadyTargetType) { + // A MaskGrid source is only visited when the target is itself a mask, where + // the build types match and the branch below attempts no conversion. No + // LeafNode conversion constructor exists from ValueMask to another build type. + using SourceGridTypes = std::conditional_t, + GridTypes, GridTypes::Remove>; + GridBase::Ptr result; - source.apply([&](const auto& typedSource) { + source.apply([&](const auto& typedSource) { using SourceGridT = std::decay_t; using TargetGridT = typename SourceGridT::template ValueConverter::Type; - // No LeafNode conversion constructor exists from ValueMask to any - // other build type, so exclude it here; nested if constexpr avoids - // instantiating the ambiguous CanConvertType. - if constexpr (!std::is_same_v) { - if constexpr (CanConvertType::value) { - if (TargetGridT::gridType() == targetType) { - result = typename TargetGridT::Ptr(new TargetGridT(typedSource)); - } + if constexpr (std::is_same_v) { + alreadyTargetType = true; + } else if constexpr (CanConvertType::value) { + if (TargetGridT::gridType() != targetType) return; + if constexpr (std::is_same_v) { + // A mask records active state, not values, so copy the topology + // instead of casting values. Casting would let a non-zero + // background or an inactive non-zero tile become true, which the + // codec path does not do. create() takes the GridBase overload to + // copy the metadata and transform without converting the values. + auto target = TargetGridT::create(static_cast(typedSource)); + target->setTree(typename TargetGridT::TreeType::Ptr( + new typename TargetGridT::TreeType(typedSource.constTree(), + /*inactiveValue=*/false, /*activeValue=*/true, TopologyCopy()))); + result = target; + } else { + result = typename TargetGridT::Ptr(new TargetGridT(typedSource)); } } }); @@ -720,7 +738,7 @@ readModeName(ReadMode mode) GridBase::Ptr convertGridForReadMode(const GridBase& source, const ReadOptions& readOptions, - Codec* codec, ReadDiagnostics& diagnostics) + Codec* codec, ReadDiagnostics& diagnostics, const std::string& filename) { if (readOptions.readMode != ReadMode::Half && readOptions.readMode != ReadMode::Bool && @@ -729,38 +747,38 @@ convertGridForReadMode(const GridBase& source, const ReadOptions& readOptions, return GridBase::Ptr(); } - const std::string modeStr = readModeName(readOptions.readMode); - + // Ask the codec that would have read this grid which type it produces. CodecData::Ptr codecData = codec ? codec->createData() : CodecData::Ptr(); const std::string targetType = (codecData && codecData->grid) ? codecData->grid->type() : std::string(); - // No conversion codec registered for this grid type (falls through to the - // plain gridType codec), or the registry agrees the type is unchanged. - if (targetType.empty() || targetType == source.type()) { - diagnostics.addWarning(source.getName(), - "ReadMode::" + modeStr + " conversion is not supported for grid type '" - + source.type() + "'; reading as original type"); - return GridBase::Ptr(); - } - GridBase::Ptr result; + bool alreadyTargetType = false; if (readOptions.readMode == ReadMode::Half) { - result = convert_grid_internal::convertToBuildType(source, targetType); + result = convert_grid_internal::convertToTargetType( + source, targetType, alreadyTargetType); } else if (readOptions.readMode == ReadMode::Bool) { - result = convert_grid_internal::convertToBuildType(source, targetType); + result = convert_grid_internal::convertToTargetType( + source, targetType, alreadyTargetType); } else { - result = convert_grid_internal::convertToBuildType(source, targetType); + result = convert_grid_internal::convertToTargetType( + source, targetType, alreadyTargetType); } - // The registry named a target type that this dispatch could not produce - // (e.g. a custom-registered grid type outside GridTypes, or the - // CanConvertType guard rejected the pair): fall through with a warning - // rather than silently diverging from what the registry reported. - if (!result) { + // Either no conversion codec is registered for this grid type (targetType is + // empty, or names the plain gridType codec), or the registry named a target + // type this dispatch cannot produce, such as a grid type outside GridTypes or + // a pair that CanConvertType rejects. A grid that already has the requested + // build type is not a failure, so it is not reported. + if (!result && !alreadyTargetType) { + const std::string modeStr = readModeName(readOptions.readMode); diagnostics.addWarning(source.getName(), "ReadMode::" + modeStr + " conversion is not supported for grid type '" + source.type() + "'; reading as original type"); + OPENVDB_LOG_WARN(filename << ": grid \"" << source.getName() + << "\" requested ReadMode::" << modeStr << ", but no conversion is " + "available for grid type \"" << source.type() + << "\"; returning the original type"); } return result; @@ -789,18 +807,10 @@ File::resolveCachedGrid(const GridBase::Ptr& cachedGrid, const io::ReadOptions& "with values intact"); } else { io::Codec* codec = Archive::findCodec(grid->type(), readOptions); - GridBase::Ptr converted = convertGridForReadMode(*grid, readOptions, codec, diagnostics); - if (converted) { - grid = converted; - } else if (readOptions.readMode == ReadMode::Half || - readOptions.readMode == ReadMode::Bool || - readOptions.readMode == ReadMode::Mask) + if (GridBase::Ptr converted = + convertGridForReadMode(*grid, readOptions, codec, diagnostics, mFilename)) { - const std::string modeStr = readModeName(readOptions.readMode); - OPENVDB_LOG_WARN(mFilename << ": grid \"" << grid->getName() - << "\" requested ReadMode::" << modeStr << ", but no conversion is " - "available for grid type \"" << grid->type() - << "\"; returning the original type"); + grid = converted; } } diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index 260cbb14e7..a952e21c2d 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -1311,3 +1311,57 @@ TEST_F(TestCodec, testTopologyOnlyClearsBoolTiles) std::remove(path.c_str()); } + +// ReadMode::Mask must record which voxels are active, not cast the source values. +// A non-zero background and an inactive non-zero tile would both become true under +// a value cast. Streamed files have no grid offsets, so this exercises the +// in-memory conversion in File::resolveCachedGrid rather than the codec path. +TEST_F(TestCodec, testCachedMaskConversionIgnoresValues) +{ + using namespace openvdb; + using namespace openvdb::io; + + io::internal::initialize(); + + const std::string gridName = "nonzero_background"; + const std::string path = "testCachedMaskConversion.vdb"; + + // A non-zero background, an inactive non-zero tile and an active region. + FloatGrid::Ptr src = FloatGrid::create(/*background=*/1.0f); + src->setName(gridName); + src->tree().addTile(/*level=*/2, Coord(4096), 0.1f, /*active=*/false); + src->fill(CoordBBox(Coord(0), Coord(6)), 5.0f, /*active=*/true); + + const Index64 srcActiveVoxels = src->activeVoxelCount(); + ASSERT_TRUE(srcActiveVoxels > 0); + + // Write via io::Stream so the file has no grid offsets and every grid is + // cached up front on open(). + { + std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{src}); + } + + ReadOptions maskOpts; + maskOpts.readMode = ReadMode::Mask; + + MaskGrid::Ptr readMask; + { + io::File f(path); + f.open(); + readMask = gridPtrCast(f.readGrid(gridName, maskOpts)); + f.close(); + } + ASSERT_TRUE(readMask); + + // Only the active voxels are on. A value cast would have activated nothing + // extra, but it would have set the background and the inactive tile to true. + EXPECT_EQ(readMask->activeVoxelCount(), srcActiveVoxels); + EXPECT_TRUE(src->tree().hasSameTopology(readMask->tree())); + EXPECT_FALSE(readMask->background()); + + // The inactive 0.1f tile must not have become an active or true tile. + EXPECT_FALSE(readMask->tree().getValue(Coord(4096))); + + std::remove(path.c_str()); +} From 863b4c6246e2e672212e310cd90913b929371382 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 31 Aug 2026 23:40:58 -0700 Subject: [PATCH 75/80] Add clang conversion warning pragmas Signed-off-by: Dan Bailey --- openvdb/openvdb/Platform.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openvdb/openvdb/Platform.h b/openvdb/openvdb/Platform.h index 322a7cae46..f947efc2b6 100644 --- a/openvdb/openvdb/Platform.h +++ b/openvdb/openvdb/Platform.h @@ -219,6 +219,14 @@ #if defined __INTEL_COMPILER #define OPENVDB_NO_TYPE_CONVERSION_WARNING_BEGIN #define OPENVDB_NO_TYPE_CONVERSION_WARNING_END +#elif defined __clang__ + #define OPENVDB_NO_TYPE_CONVERSION_WARNING_BEGIN \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wconversion\"") \ + _Pragma("clang diagnostic ignored \"-Wfloat-conversion\"") \ + _Pragma("clang diagnostic ignored \"-Wimplicit-float-conversion\"") + #define OPENVDB_NO_TYPE_CONVERSION_WARNING_END \ + _Pragma("clang diagnostic pop") #elif defined __GNUC__ // -Wfloat-conversion was only introduced in GCC 4.9 #define OPENVDB_NO_TYPE_CONVERSION_WARNING_BEGIN \ From 9acb15143bf4f7c159266e16a3439b6b5171e06f Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Mon, 31 Aug 2026 23:42:20 -0700 Subject: [PATCH 76/80] Add no type conversion warnings around Local::convertValue() and InternalNode::DeepCopy Signed-off-by: Dan Bailey --- openvdb/openvdb/tree/InternalNode.h | 2 ++ openvdb/openvdb/tree/LeafNode.h | 6 +++++- openvdb/openvdb/tree/RootNode.h | 6 +++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/openvdb/openvdb/tree/InternalNode.h b/openvdb/openvdb/tree/InternalNode.h index c61deedbfe..42558c4fcd 100644 --- a/openvdb/openvdb/tree/InternalNode.h +++ b/openvdb/openvdb/tree/InternalNode.h @@ -988,7 +988,9 @@ struct InternalNode::DeepCopy void operator()(const tbb::blocked_range &r) const { for (Index i = r.begin(), end=r.end(); i!=end; ++i) { if (s->mChildMask.isOff(i)) { + OPENVDB_NO_TYPE_CONVERSION_WARNING_BEGIN t->mNodes[i].setValue(ValueType(s->mNodes[i].getValue())); + OPENVDB_NO_TYPE_CONVERSION_WARNING_END } else { t->mNodes[i].setChild(new ChildNodeType(*(s->mNodes[i].getChild()))); } diff --git a/openvdb/openvdb/tree/LeafNode.h b/openvdb/openvdb/tree/LeafNode.h index 21cae3d7a2..79c8b3d9a7 100644 --- a/openvdb/openvdb/tree/LeafNode.h +++ b/openvdb/openvdb/tree/LeafNode.h @@ -1009,7 +1009,11 @@ LeafNode::LeafNode(const LeafNode& other) { struct Local { /// @todo Consider using a value conversion functor passed as an argument instead. - static inline ValueType convertValue(const OtherValueType& val) { return ValueType(val); } + static inline ValueType convertValue(const OtherValueType& val) { + OPENVDB_NO_TYPE_CONVERSION_WARNING_BEGIN + return ValueType(val); + OPENVDB_NO_TYPE_CONVERSION_WARNING_END + } }; for (Index i = 0; i < SIZE; ++i) { diff --git a/openvdb/openvdb/tree/RootNode.h b/openvdb/openvdb/tree/RootNode.h index ccf9bc6a24..3eec43a7f9 100644 --- a/openvdb/openvdb/tree/RootNode.h +++ b/openvdb/openvdb/tree/RootNode.h @@ -1178,7 +1178,11 @@ struct RootNodeCopyHelper struct Local { /// @todo Consider using a value conversion functor passed as an argument instead. - static inline ValueT convertValue(const OtherValueT& val) { return ValueT(val); } + static inline ValueT convertValue(const OtherValueT& val) { + OPENVDB_NO_TYPE_CONVERSION_WARNING_BEGIN + return ValueT(val); + OPENVDB_NO_TYPE_CONVERSION_WARNING_END + } }; self.mBackground = Local::convertValue(other.mBackground); From 829f6b75e71504773bdca897b593fbf321f8097b Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Tue, 1 Sep 2026 10:39:35 -0700 Subject: [PATCH 77/80] Fix a minor dangling else issue Signed-off-by: Dan Bailey --- openvdb/openvdb/unittest/TestCodec.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index a952e21c2d..7f02d14d72 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -1249,13 +1249,17 @@ TEST_F(TestCodec, testTopologyOnlyPreservesMaskTiles) // Leaf voxels keep their values because value and active state share one bit. auto topoAcc = readTopo->getConstAccessor(); for (MaskGrid::ValueOnCIter it = readOriginal->cbeginValueOn(); it; ++it) { - if (it.isVoxelValue()) EXPECT_TRUE(topoAcc.isValueOn(it.getCoord())); + if (it.isVoxelValue()) { + EXPECT_TRUE(topoAcc.isValueOn(it.getCoord())); + } } // Tile values are reset to the background, matching a TopologyCopy, while the // tiles stay active. for (MaskGrid::ValueOnCIter it = readTopo->cbeginValueOn(); it; ++it) { - if (!it.isVoxelValue()) EXPECT_FALSE(*it); + if (!it.isVoxelValue()) { + EXPECT_FALSE(*it); + } } std::remove(path.c_str()); @@ -1306,7 +1310,9 @@ TEST_F(TestCodec, testTopologyOnlyClearsBoolTiles) // Values are discarded: tiles are reset to the background even where active. for (BoolGrid::ValueAllCIter it = readTopo->cbeginValueAll(); it; ++it) { - if (!it.isVoxelValue()) EXPECT_FALSE(*it); + if (!it.isVoxelValue()) { + EXPECT_FALSE(*it); + } } std::remove(path.c_str()); From f5d807d33bfc52434ed628612785c327f6499526 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Fri, 4 Sep 2026 11:04:11 -0700 Subject: [PATCH 78/80] Address feedback Signed-off-by: Dan Bailey --- openvdb/openvdb/io/Codec.h | 18 ++++++++++++------ openvdb/openvdb/unittest/TestCodec.cc | 12 ++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/openvdb/openvdb/io/Codec.h b/openvdb/openvdb/io/Codec.h index 473102b57a..95210bbf5c 100644 --- a/openvdb/openvdb/io/Codec.h +++ b/openvdb/openvdb/io/Codec.h @@ -77,12 +77,18 @@ enum class ReadMode { /// then constructing a @c MaskGrid from the active-voxel set, but may /// be performed more efficiently inside the codec. Mask, - /// Deserialize topology only; value buffers are skipped. The resulting - /// grid has a valid tree structure (active/inactive state, node - /// hierarchy) and all leaf buffers are allocated and filled with the - /// grid's background value. Useful when only the active-voxel mask is - /// needed and avoiding the cost of reading large value buffers is - /// desirable. + /// Deserialize topology only; reading of value buffers may be skipped. + /// Useful when only the active-voxel mask is needed and avoiding the cost + /// of reading large value buffers is desirable. + /// + /// @warning Only the topology is guaranteed. The resulting grid has a + /// valid tree structure (node hierarchy, tiles and active/inactive state), + /// but the voxel and tile values are unspecified. They are typically the + /// grid's background value, however a grid read from a file without grid + /// offsets is fully cached up-front, so the returned grid keeps the values + /// that were read. Deep-copying such a grid to reset its values would cost + /// more than the read this mode is meant to avoid. Callers must not rely + /// on the values being the background value. TopologyOnly, /// Deserialize grid metadata and transform only; no topology, no value /// buffers. The codec is still used to construct the correct grid type, diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index 7f02d14d72..765e7c1ac4 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -245,6 +245,7 @@ void testIOImpl( EXPECT_EQ(readTopo->tree().leafCount(), srcGrid->tree().leafCount()); EXPECT_TRUE(readTopo->tree().leafCount() > 0); EXPECT_EQ(readTopo->activeVoxelCount(), srcGrid->activeVoxelCount()); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(readTopo->tree())); // verify leaf buffers are allocated (bool/mask buffers are always present, skip empty() check) if constexpr (!std::is_same_v) { for (auto leafIter = readTopo->tree().cbeginLeaf(); leafIter; ++leafIter) { @@ -761,6 +762,11 @@ TEST_F(TestCodec, testTopologyOnlyWarnsAndIgnoresNoGridOffsets) EXPECT_TRUE(grid->isType()); EXPECT_EQ(size_t(1), f.readDiagnostics().diagnostics().size()); + // Whatever happens to the values, the topology must match the source grid. + FloatGrid::Ptr floatGrid = gridPtrCast(grid); + ASSERT_TRUE(floatGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(floatGrid->tree())); + f.close(); std::remove(path.c_str()); } @@ -798,6 +804,9 @@ TEST_F(TestCodec, testGetGridsTopologyOnlyWarnsNoGridOffsets) EXPECT_TRUE(grid->isType()); ASSERT_EQ(size_t(1), f.readDiagnostics().diagnostics().size()); readGridMessage = f.readDiagnostics().diagnostics()[0].message; + FloatGrid::Ptr floatGrid = gridPtrCast(grid); + ASSERT_TRUE(floatGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(floatGrid->tree())); f.close(); } @@ -812,6 +821,9 @@ TEST_F(TestCodec, testGetGridsTopologyOnlyWarnsNoGridOffsets) EXPECT_TRUE((*grids)[0]->isType()); ASSERT_EQ(size_t(1), f.readDiagnostics().diagnostics().size()); EXPECT_EQ(f.readDiagnostics().diagnostics()[0].message, readGridMessage); + FloatGrid::Ptr floatGrid = gridPtrCast((*grids)[0]); + ASSERT_TRUE(floatGrid); + EXPECT_TRUE(srcGrid->tree().hasSameTopology(floatGrid->tree())); f.close(); } From ab58a58dd142adfb8f830e3b90adf3b298b7ffa0 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Sat, 5 Sep 2026 18:31:37 -0700 Subject: [PATCH 79/80] Fix ReadTopology for MaskGrids to always use active state Signed-off-by: Dan Bailey --- openvdb/openvdb/codecs/TopologyCodec.h | 33 +++- openvdb/openvdb/unittest/TestCodec.cc | 211 +++++++++++++++++++++---- 2 files changed, 209 insertions(+), 35 deletions(-) diff --git a/openvdb/openvdb/codecs/TopologyCodec.h b/openvdb/openvdb/codecs/TopologyCodec.h index 10816609ab..a7855fe37b 100644 --- a/openvdb/openvdb/codecs/TopologyCodec.h +++ b/openvdb/openvdb/codecs/TopologyCodec.h @@ -117,6 +117,11 @@ struct ReadTopologyOp using LeafT = typename TreeT::LeafNodeType; using StorageValueT = typename StorageTreeT::ValueType; + // A ValueMask target records active state, not cast storage values: value + // must equal active state at every level, so the background and tile/node + // values are forced from activity instead of being converted from storage. + static constexpr bool isMaskTarget = std::is_same_v; + ReadTopologyOp(std::istream& _is, bool _saveFloatAsHalf, io::ReadDiagnostics& _diagnostics, const std::string& _gridName) : is(_is) @@ -140,7 +145,11 @@ struct ReadTopologyOp // Read a RootNode that was stored in the current format. is.read(reinterpret_cast(&storageBackground), sizeof(StorageValueT)); - background = static_cast(storageBackground); + if constexpr (isMaskTarget) { + background = false; + } else { + background = static_cast(storageBackground); + } Index numTiles = 0, numChildren = 0; is.read(reinterpret_cast(&numTiles), sizeof(Index)); @@ -156,7 +165,9 @@ struct ReadTopologyOp is.read(reinterpret_cast(&value), sizeof(StorageValueT)); is.read(reinterpret_cast(&active), sizeof(bool)); Coord origin(vec); - if constexpr (std::is_same_v) { + if constexpr (isMaskTarget) { + root.addTile(origin, active, active); + } else if constexpr (std::is_same_v) { root.addTile(origin, value, active); } else { root.addTile(origin, static_cast(value), active); @@ -194,7 +205,10 @@ struct ReadTopologyOp StorageValueT* values = valuePtr.get(); io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf, &storageBackground); - // Copy values from the array into this node's table. + // Copy values from the array into this node's table. For a + // ValueMask target the decoded values array is only read to keep + // the stream position correct; the value comes from the value + // mask instead, so that value equals active state. if (oldVersion) { // The node's member child mask is still empty at this point // (PartialCreate; setChildUnsafe runs below), so iterate the @@ -202,12 +216,21 @@ struct ReadTopologyOp // avoid over-reading the countOff-sized values array. Index n = 0; for (auto iter = childMask.beginOff(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n++])); + if constexpr (isMaskTarget) { + node.setValueOnlyUnsafe(iter.pos(), valueMask.isOn(iter.pos())); + } else { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[n])); + } + ++n; } OPENVDB_ASSERT(n == numValues); } else { for (auto iter = node.beginValueAll(); iter; ++iter) { - node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); + if constexpr (isMaskTarget) { + node.setValueOnlyUnsafe(iter.pos(), valueMask.isOn(iter.pos())); + } else { + node.setValueOnlyUnsafe(iter.pos(), static_cast(values[iter.pos()])); + } } } } diff --git a/openvdb/openvdb/unittest/TestCodec.cc b/openvdb/openvdb/unittest/TestCodec.cc index 765e7c1ac4..ce163f0920 100644 --- a/openvdb/openvdb/unittest/TestCodec.cc +++ b/openvdb/openvdb/unittest/TestCodec.cc @@ -1330,56 +1330,207 @@ TEST_F(TestCodec, testTopologyOnlyClearsBoolTiles) std::remove(path.c_str()); } -// ReadMode::Mask must record which voxels are active, not cast the source values. -// A non-zero background and an inactive non-zero tile would both become true under -// a value cast. Streamed files have no grid offsets, so this exercises the -// in-memory conversion in File::resolveCachedGrid rather than the codec path. -TEST_F(TestCodec, testCachedMaskConversionIgnoresValues) +// Compare two grids of the same type for background, topology, values and active +// states, so that a conversion done on read can be checked against a reference. +template +void expectGridsMatch(const GridT& reference, const GridT& other, const std::string& context) +{ + using namespace openvdb; + + SCOPED_TRACE(context); + + EXPECT_EQ(reference.background(), other.background()); + EXPECT_TRUE(reference.constTree().hasSameTopology(other.constTree())); + EXPECT_EQ(reference.activeVoxelCount(), other.activeVoxelCount()); + EXPECT_EQ(reference.constTree().activeTileCount(), other.constTree().activeTileCount()); + + // Walk each grid against an accessor on the other so that a tile or voxel + // present in only one of them is still compared. + auto otherAccessor = other.getConstAccessor(); + for (typename GridT::ValueAllCIter it = reference.cbeginValueAll(); it; ++it) { + const Coord ijk = it.getCoord(); + EXPECT_EQ(*it, otherAccessor.getValue(ijk)); + EXPECT_EQ(it.isValueOn(), otherAccessor.isValueOn(ijk)); + } + auto referenceAccessor = reference.getConstAccessor(); + for (typename GridT::ValueAllCIter it = other.cbeginValueAll(); it; ++it) { + const Coord ijk = it.getCoord(); + EXPECT_EQ(*it, referenceAccessor.getValue(ijk)); + EXPECT_EQ(it.isValueOn(), referenceAccessor.isValueOn(ijk)); + } +} + +// A conversion readMode must produce the same grid as reading the grid in its +// original type and converting it in memory. That in-memory conversion is the +// reference for the conversion codec (files with grid offsets) and for the cached +// conversion in File::resolveCachedGrid (files without). This holds for Bool and +// Half, whose targets have genuine values distinct from their active state; Mask +// is the exception and is tested separately in testMaskConversionMatchesInMemory. +template +void testConversionMatchesInMemoryImpl(const std::string& label) { using namespace openvdb; using namespace openvdb::io; + CodecRegistry::clear(); io::internal::initialize(); - const std::string gridName = "nonzero_background"; - const std::string path = "testCachedMaskConversion.vdb"; + const std::string gridName = "conversion_parity"; - // A non-zero background, an inactive non-zero tile and an active region. + // A non-zero background, an inactive non-zero tile, an active zero tile and a + // region of active voxels. Casting values and copying activity disagree on + // the background and on both tiles, which is what makes this a useful case. + // The two tiles sit in different internal nodes so that neither node holds + // more than two distinct inactive values, keeping them exact under the + // default active mask compression. FloatGrid::Ptr src = FloatGrid::create(/*background=*/1.0f); src->setName(gridName); src->tree().addTile(/*level=*/2, Coord(4096), 0.1f, /*active=*/false); + src->tree().addTile(/*level=*/2, Coord(8192), 0.0f, /*active=*/true); src->fill(CoordBBox(Coord(0), Coord(6)), 5.0f, /*active=*/true); - const Index64 srcActiveVoxels = src->activeVoxelCount(); - ASSERT_TRUE(srcActiveVoxels > 0); - - // Write via io::Stream so the file has no grid offsets and every grid is - // cached up front on open(). + // io::File writes grid offsets, so a conversion readMode goes through the + // conversion codec. io::Stream writes none, so every grid is cached on open() + // and converted in memory instead. + const std::string offsetsPath = "test_conversion_parity_offsets_" + label + ".vdb"; + const std::string noOffsetsPath = "test_conversion_parity_no_offsets_" + label + ".vdb"; { - std::ofstream os(path, std::ios_base::out | std::ios_base::binary); + io::File f(offsetsPath); + f.write(GridPtrVec{src}); + } + { + std::ofstream os(noOffsetsPath, std::ios_base::out | std::ios_base::binary); io::Stream(os).write(GridPtrVec{src}); } - ReadOptions maskOpts; - maskOpts.readMode = ReadMode::Mask; + ReadOptions convertOptions; + convertOptions.readMode = mode; + + for (const std::string& path : {offsetsPath, noOffsetsPath}) { + FloatGrid::Ptr readOriginal; + typename DstGridT::Ptr readConverted; + { + io::File f(path); + f.open(); + readOriginal = gridPtrCast(f.readGrid(gridName, ReadOptions{})); + readConverted = gridPtrCast(f.readGrid(gridName, convertOptions)); + f.close(); + } + ASSERT_TRUE(readOriginal); + ASSERT_TRUE(readConverted); + + // Read as the original type, then convert in memory: the reference. + const DstGridT reference(*readOriginal); + + // Pin the reference semantics for the bool-valued targets, so that a change + // to the core grid conversion is caught here rather than silently redefining + // what the read paths are compared against. Values come from a cast, active + // state from the source topology, and the two disagree on both tiles. + if constexpr (std::is_same_v) { + auto accessor = reference.getConstAccessor(); + EXPECT_TRUE(reference.background()); // bool(1.0f) + EXPECT_TRUE(accessor.getValue(Coord(4096))); // bool(0.1f) + EXPECT_FALSE(accessor.isValueOn(Coord(4096))); + EXPECT_FALSE(accessor.getValue(Coord(8192))); // bool(0.0f) + EXPECT_TRUE(accessor.isValueOn(Coord(8192))); + EXPECT_TRUE(accessor.getValue(Coord(0))); // bool(5.0f) + EXPECT_TRUE(accessor.isValueOn(Coord(0))); + } + + expectGridsMatch(reference, *readConverted, path); + } + + std::remove(offsetsPath.c_str()); + std::remove(noOffsetsPath.c_str()); +} + +// ReadMode::Mask does not match MaskGrid(readOriginal): a mask records active +// state rather than a cast value, so the reference here is a topology copy of +// the source rather than the in-memory conversion used for Bool and Half. +TEST_F(TestCodec, testMaskConversionMatchesInMemory) +{ + using namespace openvdb; + using namespace openvdb::io; - MaskGrid::Ptr readMask; + CodecRegistry::clear(); + io::internal::initialize(); + + const std::string gridName = "conversion_parity"; + + // Same source grid as the generic parity test: a non-zero background, an + // inactive non-zero tile and an active zero tile disagree between a value + // cast and a topology copy, which is the case worth pinning here. + FloatGrid::Ptr src = FloatGrid::create(/*background=*/1.0f); + src->setName(gridName); + src->tree().addTile(/*level=*/2, Coord(4096), 0.1f, /*active=*/false); + src->tree().addTile(/*level=*/2, Coord(8192), 0.0f, /*active=*/true); + src->fill(CoordBBox(Coord(0), Coord(6)), 5.0f, /*active=*/true); + + const std::string offsetsPath = "test_conversion_parity_offsets_mask.vdb"; + const std::string noOffsetsPath = "test_conversion_parity_no_offsets_mask.vdb"; { - io::File f(path); - f.open(); - readMask = gridPtrCast(f.readGrid(gridName, maskOpts)); - f.close(); + io::File f(offsetsPath); + f.write(GridPtrVec{src}); + } + { + std::ofstream os(noOffsetsPath, std::ios_base::out | std::ios_base::binary); + io::Stream(os).write(GridPtrVec{src}); } - ASSERT_TRUE(readMask); - // Only the active voxels are on. A value cast would have activated nothing - // extra, but it would have set the background and the inactive tile to true. - EXPECT_EQ(readMask->activeVoxelCount(), srcActiveVoxels); - EXPECT_TRUE(src->tree().hasSameTopology(readMask->tree())); - EXPECT_FALSE(readMask->background()); + ReadOptions maskOptions; + maskOptions.readMode = ReadMode::Mask; - // The inactive 0.1f tile must not have become an active or true tile. - EXPECT_FALSE(readMask->tree().getValue(Coord(4096))); + for (const std::string& path : {offsetsPath, noOffsetsPath}) { + FloatGrid::Ptr readOriginal; + MaskGrid::Ptr readConverted; + { + io::File f(path); + f.open(); + readOriginal = gridPtrCast(f.readGrid(gridName, ReadOptions{})); + readConverted = gridPtrCast(f.readGrid(gridName, maskOptions)); + f.close(); + } + ASSERT_TRUE(readOriginal); + ASSERT_TRUE(readConverted); + + // The reference is a topology copy of the source, not MaskGrid(readOriginal): + // value equals active state everywhere, background is false, and inactive + // tiles are preserved rather than dropped. + MaskGrid::Ptr reference = MaskGrid::create(static_cast(*readOriginal)); + reference->setTree(MaskGrid::TreeType::Ptr( + new MaskGrid::TreeType(readOriginal->constTree(), + /*inactiveValue=*/false, /*activeValue=*/true, TopologyCopy()))); + + auto accessor = reference->getConstAccessor(); + EXPECT_FALSE(reference->background()); + EXPECT_FALSE(accessor.getValue(Coord(4096))); + EXPECT_FALSE(accessor.isValueOn(Coord(4096))); + EXPECT_TRUE(accessor.getValue(Coord(8192))); + EXPECT_TRUE(accessor.isValueOn(Coord(8192))); + EXPECT_TRUE(accessor.getValue(Coord(0))); + EXPECT_TRUE(accessor.isValueOn(Coord(0))); + EXPECT_EQ(reference->activeVoxelCount(), readOriginal->activeVoxelCount()); + EXPECT_EQ(reference->constTree().activeTileCount(), readOriginal->constTree().activeTileCount()); + + // Value equals active state at every position for a mask, which is what + // the topology-copy reference buys over a plain value cast. + for (auto it = reference->cbeginValueAll(); it; ++it) { + EXPECT_EQ(*it, it.isValueOn()); + } - std::remove(path.c_str()); + expectGridsMatch(*reference, *readConverted, path); + } + + std::remove(offsetsPath.c_str()); + std::remove(noOffsetsPath.c_str()); +} + +TEST_F(TestCodec, testBoolConversionMatchesInMemory) +{ + testConversionMatchesInMemoryImpl("bool"); +} + +TEST_F(TestCodec, testHalfConversionMatchesInMemory) +{ + testConversionMatchesInMemoryImpl("half"); } From 9616e62c7f9e0f129ef97ea44a287c7d77feb9d1 Mon Sep 17 00:00:00 2001 From: Dan Bailey Date: Sat, 5 Sep 2026 19:16:12 -0700 Subject: [PATCH 80/80] Remove all reference to Boost in CI and docs Signed-off-by: Dan Bailey --- .github/workflows/weekly.yml | 4 ++-- README.md | 4 ---- ci/install_windows.ps1 | 3 --- doc/build.txt | 13 +++---------- doc/dependencies.txt | 3 --- doc/nanovdb/FAQ.md | 2 +- doc/nanovdb/HowToBuild.md | 4 ++-- openvdb/openvdb/io/File.cc | 4 ---- openvdb/openvdb/tree/Tree.h | 1 - 9 files changed, 8 insertions(+), 30 deletions(-) diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index 7b805ba35b..08ae467821 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -201,7 +201,7 @@ jobs: - name: install_deps run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo apt-get -q install -y libboost-dev libboost-iostreams-dev libtbb-dev libblosc-dev libgtest-dev libgmock-dev python3-numpy + sudo apt-get -q install -y libtbb-dev libblosc-dev libgtest-dev libgmock-dev python3-numpy ./ci/install_nanobind.sh 2.5.0 elif [ "$RUNNER_OS" == "macOS" ]; then ./ci/install_macos.sh 20 @@ -560,7 +560,7 @@ jobs: echo "Using major version: ${VDB_MAJOR_VERSION}" echo "VDB_MAJOR_VERSION=${VDB_MAJOR_VERSION}" >> "$GITHUB_ENV" - name: install_deps - run: sudo apt-get -q install -y libboost-iostreams-dev libtbb-dev libblosc-dev elfutils + run: sudo apt-get -q install -y libtbb-dev libblosc-dev elfutils - name: install_abi_checker run: sudo apt-get -q install -y abi-dumper abi-compliance-checker - name: build_new diff --git a/README.md b/README.md index f86d9491b3..8297e59bf8 100644 --- a/README.md +++ b/README.md @@ -59,14 +59,12 @@ for help with installations. # Linux # @note If your distribution does not have required versions, consider using # apt pinning. See the dependency documentation for more details. -apt-get install -y libboost-iostreams-dev apt-get install -y libtbb-dev apt-get install -y libblosc-dev # MacOS # @note We are using homebrew in this example to install requried dependencies # https://brew.sh/ -brew install boost brew install tbb brew install c-blosc ``` @@ -94,8 +92,6 @@ dependencies. vcpkg install zlib:x64-windows vcpkg install blosc:x64-windows vcpkg install tbb:x64-windows -vcpkg install boost-iostreams:x64-windows -vcpkg install boost-interprocess:x64-windows ``` ```bash diff --git a/ci/install_windows.ps1 b/ci/install_windows.ps1 index a4d44da353..683fdd9066 100644 --- a/ci/install_windows.ps1 +++ b/ci/install_windows.ps1 @@ -15,9 +15,6 @@ $vcpkgPackages = @( "glew", "python3", "jemalloc", - "boost-iostreams", - "boost-interprocess", - "boost-algorithm", "nanobind" ) diff --git a/doc/build.txt b/doc/build.txt index 7356f5eb64..53ba249c18 100644 --- a/doc/build.txt +++ b/doc/build.txt @@ -299,15 +299,12 @@ Optional: - @b USE_DEFAULT_HOUDINI_INSTALL =`ON` # Use the above default locations if @b OPENVDB_HOUDINI_INSTALL_PREFIX is not specified. If `OFF`, uses the value of @b CMAKE_INSTALL_PREFIX. - - @b BOOST_ROOT =`/path/to/boost/install` # Path to boost. May not be required, - CMake may find it automatically For example on MacOS and where the build folder has been created inside the OpenVDB source root: @code -cmake -D BOOST_ROOT=/path/to/boost/install \ - -D Houdini_ROOT=/Applications/Houdini/Houdini17.0.506/ \ +cmake -D Houdini_ROOT=/Applications/Houdini/Houdini17.0.506/ \ -D USE_HOUDINI=ON \ ../ @endcode @@ -343,7 +340,6 @@ Package | Description -------------- | --------------------------------------------------------------- | -------------------------------- | CMake | Cross-platform family of tools designed to help build software | All | C++17 Compiler | Matching Houdini compiler and version | All | -Boost | Components: iostreams | All | ZLIB | Compression library for disk serialization compression | All | Blosc | Recommended dependency for improved disk compression | All* | GoogleTest | A unit testing framework module for C++ | Unit Tests | @@ -358,7 +354,7 @@ LLVM | Target-independent code generation * See [blosc support](@ref buildBloscSupport) -At a minimum, boost, a matching C++17 compiler, ZLIB, blosc and CMake will be +At a minimum, a matching C++17 compiler, ZLIB, blosc and CMake will be required. See the full [dependency list](@ref dependencies) for help with downloading and installing the above software. @@ -386,15 +382,12 @@ Optional: the link for default values. - @b OPENVDB_MAYA_INSTALL_PREFIX =`/path/to/install/dir` # The location to install the Maya plugin to. Defaults to the value of `${CMAKE_INSTALL_PREFIX}/maya${Maya_VERSION}` - - @b BOOST_ROOT =`/path/to/boost/install` # Path to boost. May not be required, - CMake may find it automatically For example on MacOS and where the build folder has been created inside the OpenVDB source root: @code -cmake -D BOOST_ROOT=/path/to/boost/install \ - -D Maya_ROOT=/Applications/Autodesk/maya2019/ \ +cmake -D Maya_ROOT=/Applications/Autodesk/maya2019/ \ -D USE_MAYA=ON \ ../ @endcode diff --git a/doc/dependencies.txt b/doc/dependencies.txt index 9c391f23db..7a45338d5b 100644 --- a/doc/dependencies.txt +++ b/doc/dependencies.txt @@ -73,7 +73,6 @@ Imath | 3.2 | >= 3.3 | Half precision floating points OpenEXR | 3.2 | >= 3.3 | EXR serialization support | Y | Y | http://www.openexr.com TBB | 2020.3 | >= 2021 | Threading Building Blocks - template library for task parallelism | Y | Y | https://www.threadingbuildingblocks.org ZLIB | 1.2.7 | Latest | Compression library for disk serialization compression | Y | Y | https://www.zlib.net -Boost | 1.82 | >= 1.85 | Components: iostreams | Y | Y | https://www.boost.org VCL | 2.02.0 | >= 2.02.0 | x86 intrinsics and SIMD class wrappers and | Y | Y | https://github.com/vectorclass/version2 LLVM | 15.0.0* | 18.0.0 | Target-independent code generation | Y | Y | https://llvm.org/ Bison | 3.7.0 | 3.7.0 | General-purpose parser generator | Y | Y | https://www.gnu.org/software/gcc @@ -153,7 +152,6 @@ other package managers. apt-get install cmake # CMake apt-get install libtbb-dev # TBB apt-get install zlibc # zlib -apt-get install libboost-iostreams-dev # Boost::iostream apt-get install libblosc-dev # Blosc # AX apt-get install llvm-15-dev # LLVM @@ -184,7 +182,6 @@ an alternative to apt-get or manual installation from source. brew install cmake # CMake brew install tbb # TBB brew install zlib # zlib -brew install boost # Boost brew install c-blosc # blosc # AX brew install llvm # LLVM diff --git a/doc/nanovdb/FAQ.md b/doc/nanovdb/FAQ.md index 6c06eaaffa..7d2cbe2feb 100644 --- a/doc/nanovdb/FAQ.md +++ b/doc/nanovdb/FAQ.md @@ -30,7 +30,7 @@ * **How does the data structure of NanoVDB compare to that of OpenVDB?** - Conceptually they are essentially identical (NanoVDB uses the exact same tree configuration), but there are some implementation differences to minimize dependencies on external libraries in NanoVDB (e.g. no boost and std containers), and more importantly, NanoVDB has additional meta-data encoded in tree nodes specifically to accelerate ray-tracing (e.g. min/max values and AABB of active values). Also, NanoVDB is pointer-less and occupies a contiguous block of memory, i.e. de-fragmentized relative to OpenVDB. Effectively, NanoVDB is a linearized version of the well-known OpenVDB data structure with a few additional information baked into the tree nodes! + Conceptually they are essentially identical (NanoVDB uses the exact same tree configuration), but there are some implementation differences to minimize dependencies on external libraries in NanoVDB (e.g. no std containers), and more importantly, NanoVDB has additional meta-data encoded in tree nodes specifically to accelerate ray-tracing (e.g. min/max values and AABB of active values). Also, NanoVDB is pointer-less and occupies a contiguous block of memory, i.e. de-fragmentized relative to OpenVDB. Effectively, NanoVDB is a linearized version of the well-known OpenVDB data structure with a few additional information baked into the tree nodes! * **Are there known issues with NanoVDB?** diff --git a/doc/nanovdb/HowToBuild.md b/doc/nanovdb/HowToBuild.md index ab3638c098..003199bc37 100644 --- a/doc/nanovdb/HowToBuild.md +++ b/doc/nanovdb/HowToBuild.md @@ -16,7 +16,7 @@ NanoVDB is now a module of the larger OpenVDB project. A user can build both lib ```console foo@bar:~$ mkdir build foo@bar:~$ cd build - foo@bar:~$ cmake .. -DUSE_NANOVDB=ON -DTBB_ROOT=/path/to/tbb -DBOOST_ROOT=/path/to/boost -DBLOSC_ROOT=/path/to/blosc -DCMAKE_INSTALL_PREFIX=/install/path + foo@bar:~$ cmake .. -DUSE_NANOVDB=ON -DTBB_ROOT=/path/to/tbb -DBLOSC_ROOT=/path/to/blosc -DCMAKE_INSTALL_PREFIX=/install/path foo@bar:~$ make -j 4 && make install ``` Note that the default value of `NANOVDB_USE_OPENVDB` is linked to `OPENVDB_BUILD_CORE` option and can be overriden by passing on `-DNANOVDB_USE_OPENVDB=OFF`. The `Boost` library is included because it is a requirement for building OpenVDB. @@ -49,7 +49,7 @@ From the root OpenVDB directory: ```console foo@bar:~$ mkdir build foo@bar:~$ cd build - foo@bar:~$ cmake .. -DUSE_NANOVDB=ON -DNANOVDB_BUILD_UNITTESTS=ON -DNANOVDB_BUILD_EXAMPLES=ON -DNANOVDB_BUILD_BENCHMARK=ON -DNANOVDB_USE_INTRINSICS=ON -DNANOVDB_USE_CUDA=ON -DNANOVDB_CUDA_KEEP_PTX=ON -DTBB_ROOT=/path/to/tbb -DBOOST_ROOT=/path/to/boost -DBLOSC_ROOT=/path/to/blosc -DGTEST_ROOT=/path/to/gtest -DCMAKE_INSTALL_PREFIX=/install/path + foo@bar:~$ cmake .. -DUSE_NANOVDB=ON -DNANOVDB_BUILD_UNITTESTS=ON -DNANOVDB_BUILD_EXAMPLES=ON -DNANOVDB_BUILD_BENCHMARK=ON -DNANOVDB_USE_INTRINSICS=ON -DNANOVDB_USE_CUDA=ON -DNANOVDB_CUDA_KEEP_PTX=ON -DTBB_ROOT=/path/to/tbb -DBLOSC_ROOT=/path/to/blosc -DGTEST_ROOT=/path/to/gtest -DCMAKE_INSTALL_PREFIX=/install/path foo@bar:~$ make -j 4 && make install ``` Note that if you already have the correct version of OpenVDB pre-installed, you can configure CMake to link against that library by passing the arguments `-DOPENVDB_BUILD_CORE=OFF -DOPENVDB_BUILD_BINARIES=OFF -DOPENVDB_ROOT=/path/to/openvdb` when invoking `cmake`. diff --git a/openvdb/openvdb/io/File.cc b/openvdb/openvdb/io/File.cc index b783008986..3e94928f4f 100644 --- a/openvdb/openvdb/io/File.cc +++ b/openvdb/openvdb/io/File.cc @@ -121,10 +121,6 @@ File::inputStream() const Index64 File::getSize() const { - /// @internal boost::filesystem::file_size() would be a more portable alternative, - /// but as of 9/2014, Houdini ships without the Boost.Filesystem library, - /// which makes it much less convenient to use that library. - Index64 result = std::numeric_limits::max(); std::string mesg = "could not get size of file " + mFilename; diff --git a/openvdb/openvdb/tree/Tree.h b/openvdb/openvdb/tree/Tree.h index 07ed98e5db..1ca127c290 100644 --- a/openvdb/openvdb/tree/Tree.h +++ b/openvdb/openvdb/tree/Tree.h @@ -1990,7 +1990,6 @@ Tree::print(std::ostream& os, int verboseLevel) const { if (verboseLevel <= 0) return; - /// @todo Consider using boost::io::ios_precision_saver instead. struct OnExit { std::ostream& os; std::streamsize savedPrecision;