fix: handle std::bad_alloc and invalid formats on RAW file loading

YUViewApp/src/yuviewapp.cpp

@@ -31,6 +31,9 @@
 */
 
 #include <QCoreApplication>
+#include <QMessageBox>
+
+#include <iostream>
 
 #include <common/Typedef.h>
 #include <ui/YUViewApplication.h>
@@ -46,7 +49,22 @@ int main(int argc, char *argv[])
 
   qRegisterMetaType<recacheIndicator>("recacheIndicator");
 
-  YUViewApplication app(argc, argv);
-
-  return app.returnCode;
+  try
+  {
+    YUViewApplication app(argc, argv);
+    return app.returnCode;
+  }
+  catch (const std::bad_alloc &)
+  {
+    try
+    {
+      if (QCoreApplication::instance())
+        QMessageBox::critical(nullptr, "YUView", "Out of memory.");
+    }
+    catch (...)
+    {
+      std::cerr << "YUView: Out of memory." << std::endl;
+    }
+    return 1;
+  }
 }

YUViewLib/src/common/Typedef.h

@@ -184,6 +184,8 @@ struct Ratio
 
 struct Size
 {
+  static constexpr unsigned MAX_DIMENSION = 16384;
+
   constexpr Size(unsigned width, unsigned height) : width(width), height(height) {}
   constexpr Size() = default;
 
@@ -196,7 +198,11 @@ struct Size
     return this->width != other.width || this->height != other.height;
   }
   explicit       operator bool() const { return this->isValid(); }
-  constexpr bool isValid() const { return this->width > 0 && this->height > 0; }
+  constexpr bool isValid() const
+  {
+    return this->width > 0 && this->width <= MAX_DIMENSION &&
+           this->height > 0 && this->height <= MAX_DIMENSION;
+  }
   unsigned       width{};
   unsigned       height{};
 };

YUViewLib/src/filesource/FileSource.cpp

@@ -83,6 +83,9 @@ int64_t FileSource::readBytes(QByteArray &targetBuffer, int64_t startPos, int64_
   if (!this->isOk())
     return 0;
 
+  if (nrBytes <= 0)
+    return 0;
+
   if (targetBuffer.size() < nrBytes)
     targetBuffer.resize(nrBytes);
 

YUViewLib/src/playlistitem/playlistItemRawFile.cpp

@@ -32,6 +32,8 @@
 
 #include "playlistItemRawFile.h"
 
+#include <new>
+
 #include <QPainter>
 #include <QUrl>
 #include <QVBoxLayout>
@@ -134,10 +136,17 @@ playlistItemRawFile::playlistItemRawFile(const QString &rawFilePath,
 
     if (!this->video->isFormatValid())
     {
-      // Load 24883200 bytes from the input and try to get the format from the correlation.
-      QByteArray rawData;
-      this->dataSource.readBytes(rawData, 0, 24883200);
-      this->video->setFormatFromCorrelation(rawData, this->dataSource.getFileSize().value_or(-1));
+      try
+      {
+        QByteArray rawData;
+        this->dataSource.readBytes(rawData, 0, 24883200);
+        this->video->setFormatFromCorrelation(rawData,
+                                              this->dataSource.getFileSize().value_or(-1));
+      }
+      catch (const std::bad_alloc &)
+      {
+        qWarning() << "Out of memory while guessing format from correlation.";
+      }
     }
   }
   else
@@ -226,6 +235,13 @@ InfoData playlistItemRawFile::getInfo() const
     else
       info.items.append(InfoItem("Warning"sv, "Could not obtain file size from input."));
   }
+  else if (this->dataSource.isOk() && !this->video->isFormatValid())
+  {
+    info.items.append(InfoItem(
+      "Warning"sv,
+      "Could not determine the video format. Please set the width, height and pixel format "
+      "manually in the properties panel."));
+  }
 
   return info;
 }
@@ -542,6 +558,8 @@ void playlistItemRawFile::loadRawData(int frameIdx)
     return;
 
   auto nrBytes = this->video->getBytesPerFrame();
+  if (nrBytes <= 0)
+    return;
 
   // Load the raw data for the given frameIdx from file and set it in the video
   int64_t fileStartPos;
@@ -550,10 +568,21 @@ void playlistItemRawFile::loadRawData(int frameIdx)
   else
     fileStartPos = frameIdx * nrBytes;
 
+  if (fileStartPos < 0)
+    return;
+
   DEBUG_RAWFILE("playlistItemRawFile::loadRawData Start loading frame " << frameIdx << " bytes "
                                                                         << int(nrBytes));
-  if (this->dataSource.readBytes(this->video->rawData, fileStartPos, nrBytes) < nrBytes)
-    return; // Error
+  try
+  {
+    if (this->dataSource.readBytes(this->video->rawData, fileStartPos, nrBytes) < nrBytes)
+      return; // Error
+  }
+  catch (const std::bad_alloc &)
+  {
+    qWarning() << "Out of memory while loading RAW frame data.";
+    return;
+  }
   this->video->rawData_frameIndex = frameIdx;
 
   DEBUG_RAWFILE("playlistItemRawFile::loadRawData Frame " << frameIdx << " loaded");

YUViewLib/src/video/FrameHandler.cpp

@@ -124,10 +124,10 @@ QLayout *FrameHandler::createFrameHandlerControls(bool isSizeFixed)
   ui.setupUi();
 
   // Set default values
-  ui.widthSpinBox->setMaximum(100000);
+  ui.widthSpinBox->setMaximum(Size::MAX_DIMENSION);
   ui.widthSpinBox->setValue(frameSize.width);
   ui.widthSpinBox->setEnabled(!isSizeFixed);
-  ui.heightSpinBox->setMaximum(100000);
+  ui.heightSpinBox->setMaximum(Size::MAX_DIMENSION);
   ui.heightSpinBox->setValue(frameSize.height);
   ui.heightSpinBox->setEnabled(!isSizeFixed);
   ui.frameSizeComboBox->addItems(presetFrameSizes.getFormattedNames());

YUViewLib/src/video/rgb/ConversionDifferenceRGB.cpp

@@ -76,9 +76,12 @@ RenderValue convertDeltaToRenderValue(const rgba_t &delta,
   }
   else
   {
-    const auto r = functions::clip(128 + delta.r * amplificationFactor, 0, 255);
-    const auto g = functions::clip(128 + delta.g * amplificationFactor, 0, 255);
-    const auto b = functions::clip(128 + delta.b * amplificationFactor, 0, 255);
+    const auto r = static_cast<unsigned char>(
+      functions::clip(128 + static_cast<int64_t>(delta.r) * amplificationFactor, 0LL, 255LL));
+    const auto g = static_cast<unsigned char>(
+      functions::clip(128 + static_cast<int64_t>(delta.g) * amplificationFactor, 0LL, 255LL));
+    const auto b = static_cast<unsigned char>(
+      functions::clip(128 + static_cast<int64_t>(delta.b) * amplificationFactor, 0LL, 255LL));
     return {r, g, b};
   }
 }

YUViewLib/src/video/rgb/ConversionDifferenceRGB.h

@@ -65,10 +65,10 @@ class SSE
 public:
   void addSample(const rgba_t &delta)
   {
-    this->r += delta.r * delta.r;
-    this->g += delta.g * delta.g;
-    this->b += delta.b * delta.b;
-    this->a += delta.a * delta.a;
+    this->r += static_cast<int64_t>(delta.r) * delta.r;
+    this->g += static_cast<int64_t>(delta.g) * delta.g;
+    this->b += static_cast<int64_t>(delta.b) * delta.b;
+    this->a += static_cast<int64_t>(delta.a) * delta.a;
     ++this->nrSamples;
   }
 
@@ -83,11 +83,11 @@ class SSE
   }
 
 private:
-  int64_t r{};
-  int64_t g{};
-  int64_t b{};
-  int64_t a{};
-  int64_t nrSamples{};
+  uint64_t r{};
+  uint64_t g{};
+  uint64_t b{};
+  uint64_t a{};
+  uint64_t nrSamples{};
 };
 
 std::pair<QImage, MSE> calculateDifferenceAndMSE(const InputFrameParameters &frame1,

YUViewLib/src/video/rgb/videoHandlerRGB.cpp

@@ -35,6 +35,8 @@
 #include "video/PixelFormat.h"
 #include "video/rgb/PixelFormatRGB.h"
 
+#include <new>
+
 #include <common/EnumMapper.h>
 #include <common/Formatting.h>
 #include <common/Functions.h>
@@ -482,14 +484,30 @@ void videoHandlerRGB::loadFrame(int frameIndex, bool loadToDoubleBuffer)
   if (loadToDoubleBuffer)
   {
     QImage newImage;
-    convertRGBToImage(currentFrameRawData, newImage);
+    try
+    {
+      convertRGBToImage(currentFrameRawData, newImage);
+    }
+    catch (const std::bad_alloc &)
+    {
+      qWarning() << "Out of memory in loadFrame convertRGBToImage (double buffer).";
+      return;
+    }
     doubleBufferImage           = newImage;
     doubleBufferImageFrameIndex = frameIndex;
   }
   else if (currentImageIndex != frameIndex)
   {
     QImage newImage;
-    convertRGBToImage(currentFrameRawData, newImage);
+    try
+    {
+      convertRGBToImage(currentFrameRawData, newImage);
+    }
+    catch (const std::bad_alloc &)
+    {
+      qWarning() << "Out of memory in loadFrame convertRGBToImage.";
+      return;
+    }
     QMutexLocker writeLock(&currentImageSetMutex);
     currentImage      = newImage;
     currentImageIndex = frameIndex;
@@ -551,6 +569,9 @@ void videoHandlerRGB::loadPlaylist(const YUViewDomElement &element)
 
 void videoHandlerRGB::loadFrameForCaching(int frameIndex, QImage &frameToCache)
 {
+  if (!isFormatValid())
+    return;
+
   DEBUG_RGB("videoHandlerRGB::loadFrameForCaching %d", frameIndex);
 
   // Lock the mutex for the rgbFormat. The main thread has to wait until caching is done
@@ -571,7 +592,15 @@ void videoHandlerRGB::loadFrameForCaching(int frameIndex, QImage &frameToCache)
   }
 
   // Convert RGB to image. This can then be cached.
-  convertRGBToImage(tmpBufferRawRGBDataCaching, frameToCache);
+  try
+  {
+    convertRGBToImage(tmpBufferRawRGBDataCaching, frameToCache);
+  }
+  catch (const std::bad_alloc &)
+  {
+    qWarning() << "Out of memory in loadFrameForCaching convertRGBToImage.";
+    frameToCache = QImage();
+  }
 
   rgbFormatMutex.unlock();
 }
@@ -730,6 +759,8 @@ void videoHandlerRGB::convertSourceToRGBA32Bit(const QByteArray &sourceBuffer,
 
 rgba_t videoHandlerRGB::getPixelValue(const QPoint &pixelPos) const
 {
+  if (!isFormatValid())
+    return {};
   return getPixelValueFromBuffer(
     this->currentFrameRawData, this->srcPixelFormat, this->frameSize, pixelPos);
 }
@@ -753,6 +784,9 @@ void videoHandlerRGB::drawPixelValues(QPainter     *painter,
                                       const bool    markDifference,
                                       const int     frameIdxItem1)
 {
+  if (!isFormatValid())
+    return;
+
   // First determine which pixels from this item are actually visible, because we only have to draw
   // the pixel values of the pixels that are actually visible
   auto viewport       = painter->viewport();
@@ -869,6 +903,9 @@ QImage videoHandlerRGB::calculateDifference(FrameHandler    *item2,
                                             const int        amplificationFactor,
                                             const bool       markDifference)
 {
+  if (!isFormatValid())
+    return QImage();
+
   videoHandlerRGB *rgbItem2 = dynamic_cast<videoHandlerRGB *>(item2);
   if (rgbItem2 == nullptr)
     // The given item is not a RGB source. We cannot compare raw RGB values to non raw RGB values.