fix map3d on windows

.github/workflows/windows_build.yml

@@ -33,7 +33,7 @@ jobs:
         run: ./installer.exe /verysilent /allusers /dir=inst
       - name: Build MAVProxy
         run: |
-          python3 -m pip install .[recommended] --user
+          python3 -m pip install .[recommended,map3d] --user
           python3 -m pip list
       - name: Prepare installer
         run: |

MAVProxy/mavproxy.py

@@ -377,6 +377,7 @@ def load_module(self, modname, quiet=False, **kwargs):
                 # don't report an error
                 return True
         ex = None
+        ex_modpath = None
         for modpath in modpaths:
             try:
                 m = import_package(modpath)
@@ -394,9 +395,16 @@ def load_module(self, modname, quiet=False, **kwargs):
                     ex = "%s.init did not return a MPModule instance" % modname
                     break
             except ImportError as msg:
-                ex = msg
+                # keep the first real failure, so a fallback modpath's "No
+                # module named X" cannot hide it. A ModuleNotFoundError naming
+                # the modpath we just tried only says that path does not exist,
+                # so let a later attempt replace it with its own reason
+                if (ex is None or
+                        (isinstance(ex, ModuleNotFoundError) and ex.name == ex_modpath)):
+                    ex = msg
+                    ex_modpath = modpath
                 if mpstate.settings.moddebug > 1:
-                    print(get_exception_stacktrace(ex))
+                    print(get_exception_stacktrace(msg))
         help_traceback = ""
         if mpstate.settings.moddebug < 3:
             help_traceback = " Use 'set moddebug 3' in the MAVProxy console to enable traceback"

MAVProxy/modules/lib/mp_util.py

@@ -595,3 +595,16 @@ def natural_sort_key(s):
 def sorted_natural(lst):
     '''sort using a 'natural' sort order'''
     return sorted(lst, key=natural_sort_key)
+
+def import_cv2():
+    '''import cv2, explaining the usual cause of failure. A distro opencv is
+    built against one numpy major version, so pulling in a newer numpy (some
+    packages require numpy>=2) leaves cv2 unimportable'''
+    try:
+        import cv2
+        return cv2
+    except ImportError as ex:
+        raise ImportError(
+            "opencv (cv2) is not usable: %s. If numpy was upgraded then the "
+            "installed opencv may be built for the previous numpy; try "
+            "'pip install -U opencv-python'" % ex)

MAVProxy/modules/mavproxy_console.py

@@ -116,6 +116,7 @@ def __init__(self, mpstate):
             self.add_menu(MPMenuSubMenu('MAVProxy',
                                         items=[MPMenuItem('Settings', 'Settings', 'menuSettings'),
                                                MPMenuItem('Show Map', 'Load Map', '# module load map'),
+                                               MPMenuItem('Show Map3d', 'Load 3D Map', '# module load map3d'),
                                                MPMenuItem('Show HUD', 'Load HUD', '# module load horizon'),
                                                MPMenuItem('Show Checklist', 'Load Checklist', '# module load checklist')]))
             self.vehicle_menu = MPMenuSubMenu('Vehicle', items=[])

MAVProxy/modules/mavproxy_map/mp_slipmap.py

@@ -7,14 +7,15 @@
 '''
 
 import time
-import cv2
 
 from MAVProxy.modules.mavproxy_map import mp_tile
 from MAVProxy.modules.lib import mp_util
 from MAVProxy.modules.lib import win_layout
 from MAVProxy.modules.lib import multiproc
 from MAVProxy.modules.mavproxy_map.mp_slipmap_util import *
 
+cv2 = mp_util.import_cv2()
+
 
 class MPSlipMap():
     '''

MAVProxy/modules/mavproxy_map/mp_tile.py

@@ -26,7 +26,6 @@
 import pathlib
 import string
 import time
-import cv2
 import numpy as np
 
 from math import log, tan, radians, degrees, sin, cos, exp, pi, asin, atan
@@ -44,6 +43,8 @@
 
 from MAVProxy.modules.lib import mp_util
 
+cv2 = mp_util.import_cv2()
+
 
 class TileException(Exception):
     '''tile error class'''
@@ -218,21 +219,25 @@ def __init__(self, tile, zoom, scale, src, dst, service):
         (self.dstx, self.dsty) = dst
 
 
+def default_cache_path():
+    '''default root of the on-disk tile cache. HOME is not set on native
+    Windows, where the cwd may well be unwritable (eg. Program Files)'''
+    if 'HOME' in os.environ:
+        return os.path.join(os.environ['HOME'], '.tilecache')
+    if 'LOCALAPPDATA' in os.environ:
+        return os.path.join(os.environ['LOCALAPPDATA'], '.tilecache')
+    import tempfile
+    return os.path.join(tempfile.gettempdir(), '.tilecache')
+
+
 class MPTile:
     '''map tile object'''
     def __init__(self, cache_path=None, download=True, cache_size=500,
                  service="MicrosoftSat", tile_delay=0.3, debug=False,
                  max_zoom=19, refresh_age=30*24*60*60):
 
         if cache_path is None:
-            try:
-                cache_path = os.path.join(os.environ['HOME'], '.tilecache')
-            except Exception:
-                if 'LOCALAPPDATA' in os.environ:
-                    cache_path = os.path.join(os.environ['LOCALAPPDATA'], '.tilecache')
-                else:
-                    import tempfile
-                    cache_path = os.path.join(tempfile.gettempdir(), '.tilecache')
+            cache_path = default_cache_path()
 
         if not os.path.exists(cache_path):
             mp_util.mkdir_p(cache_path)
@@ -696,7 +701,7 @@ def mp_icon(filename):
         with importlib.resources.open_binary(package, filename) as stream:
             raw = np.frombuffer(stream.read(), dtype=np.uint8)
     except Exception:
-        with open(os.path.join(os.path.dirname(__file__), 'data', filename)).read() as stream:
+        with open(os.path.join(os.path.dirname(__file__), 'data', filename), 'rb') as stream:
             raw = np.frombuffer(stream.read(), dtype=np.uint8)
     img = cv2.imdecode(raw, cv2.IMREAD_COLOR)
     return img

MAVProxy/modules/mavproxy_map3d/__init__.py

@@ -13,7 +13,8 @@
 
 from MAVProxy.modules.lib import mp_module
 from MAVProxy.modules.lib import mp_settings
-from MAVProxy.modules.mavproxy_map3d.map3d import Map3D
+from MAVProxy.modules.mavproxy_map3d.map3d import (
+    Map3D, missing_packages, missing_packages_message)
 
 # fence colours as the 2D map's PolyFence layer uses them (OpenCV BGR)
 FENCE_INCLUSION_BGR = (0, 255, 0)
@@ -107,6 +108,10 @@ def start_map(self):
         if self.map is not None and self.map.is_alive():
             print("map3d already running")
             return
+        missing = missing_packages()
+        if missing:
+            print(missing_packages_message(missing))
+            return
         self.map = Map3D(title="MAVProxy 3D Map",
                          service=self.map3d_settings.service,
                          zexag=self.map3d_settings.zexag,
@@ -366,18 +371,23 @@ def send_kml(self, kml_mod=None):
     def idle_task(self):
         if self.map is None:
             return
-        if not self.map.is_alive():
-            self.map = None
-            return
+        # drain events before dropping a dead child, so we don't lose the
+        # reason it failed to start
+        alive = self.map.is_alive()
         for event in self.map.check_events():
-            if event[0] == 'render_settings':
+            if event[0] == 'startup_error':
+                print("map3d: the 3D view failed to start:\n%s" % event[1])
+            elif event[0] == 'render_settings':
                 (_, brightness, shading, wireframe, fpvfov) = event
                 self.map3d_settings.terrainbrightness = brightness
                 self.map3d_settings.terrainshading = shading
                 self.map3d_settings.terrainwireframe = wireframe
                 self.map3d_settings.fpvfov = fpvfov
             elif event[0] == 'follow':
                 self.follow = bool(event[1])
+        if not alive:
+            self.map = None
+            return
         kml_mod = self.module('kmlread')
         if self._kml_state(kml_mod) != self.kml_change_state:
             self.send_kml(kml_mod)

MAVProxy/modules/mavproxy_map3d/map3d.py

@@ -3,11 +3,36 @@
 (mirrors mp_slipmap) and pushes element/camera updates over a queue.
 '''
 
+import importlib.util
 import time
 import queue
+import traceback
 
 from MAVProxy.modules.lib import multiproc
 
+PACKAGES = ('vtk', 'quantized_mesh_tile')
+
+
+def missing_packages():
+    '''optional 3D map packages that are not installed. The viewer imports them
+    in a child process, where an ImportError would go unseen, so callers check
+    before starting a viewer. find_spec avoids importing VTK into the parent.'''
+    missing = []
+    for name in PACKAGES:
+        try:
+            if importlib.util.find_spec(name) is None:
+                missing.append(name)
+        except Exception:
+            missing.append(name)
+    return missing
+
+
+def missing_packages_message(missing):
+    # install via the extra, not 'pip install vtk quantized-mesh-tile': the
+    # latter pulls numpy>=2 without the opencv/matplotlib builds to match
+    return ("map3d needs extra packages: pip install 'MAVProxy[map3d]' "
+            "(missing %s)" % ', '.join(missing))
+
 
 class Map3D:
     def __init__(self, title="3D Map", service="MicrosoftSat", zexag=1.0,
@@ -38,14 +63,22 @@ def __init__(self, title="3D Map", service="MicrosoftSat", zexag=1.0,
     def child_task(self):
         from MAVProxy.modules.lib import mp_util
         mp_util.child_close_fds()
-        from MAVProxy.modules.lib import wx_processguard  # noqa: F401
-        from MAVProxy.modules.lib.wx_loader import wx
-        from MAVProxy.modules.mavproxy_map3d.map3d_ui import Map3DFrame
-
-        app = wx.App(False)
-        app.SetExitOnFrameDelete(True)
-        frame = Map3DFrame(self)
-        frame.Show()
+        try:
+            from MAVProxy.modules.lib import wx_processguard  # noqa: F401
+            from MAVProxy.modules.lib.wx_loader import wx
+            from MAVProxy.modules.mavproxy_map3d.map3d_ui import Map3DFrame
+
+            app = wx.App(False)
+            app.SetExitOnFrameDelete(True)
+            frame = Map3DFrame(self)
+            frame.Show()
+        except (Exception, SystemExit):
+            # our stderr goes nowhere when MAVProxy is started from a GUI, so
+            # hand the failure to the parent rather than dying unexplained.
+            # wx exits rather than raising when it cannot open the display,
+            # hence SystemExit
+            self.event_queue.put(('startup_error', traceback.format_exc()))
+            return
         self.app_ready.set()
         app.MainLoop()
 

MAVProxy/modules/mavproxy_map3d/map3d_ui.py

@@ -9,7 +9,7 @@
 import time
 
 from MAVProxy.modules.lib.wx_loader import wx
-from vtk.wx.wxVTKRenderWindowInteractor import wxVTKRenderWindowInteractor
+from vtkmodules.wx.wxVTKRenderWindowInteractor import wxVTKRenderWindowInteractor
 
 from MAVProxy.modules.mavproxy_map import mp_tile
 from MAVProxy.modules.mavproxy_map3d.camera import TerrainCamera, TerrainStyle

MAVProxy/modules/mavproxy_map3d/terrain.py

@@ -16,7 +16,9 @@
 
 import numpy as np
 import vtk
-from vtk.util import numpy_support
+# import via vtkmodules, not the vtk.* aliases: vtk is a plain module that fakes
+# a package with __path__, which pyinstaller cannot follow into a frozen build
+from vtkmodules.util import numpy_support
 
 from quantized_mesh_tile import decode as qmt_decode
 from quantized_mesh_tile.global_geodetic import GlobalGeodetic
@@ -25,7 +27,7 @@
 
 R = 6378137.0
 QUANTIZED_BASE = "https://plot.ardupilot.org/quantized"
-CACHE_DIR = os.path.join(os.environ.get("HOME", "."), ".tilecache", "quantized")
+CACHE_DIR = os.path.join(mp_tile.default_cache_path(), "quantized")
 
 # ArduPilot tiles include the optional lighting extension. This decoder warns
 # when it skips that extension, but map3d computes its own VTK normals anyway.

MAVProxy/tools/MAVExplorer.py

@@ -661,9 +661,15 @@ def cmd_map(args):
 def cmd_map3d(args):
     '''show a 3D map view: draped satellite imagery over terrain'''
     try:
-        from MAVProxy.modules.mavproxy_map3d.map3d import Map3D
+        from MAVProxy.modules.mavproxy_map3d.map3d import (
+            Map3D, missing_packages, missing_packages_message)
     except ImportError as ex:
-        print("map3d needs extra packages: pip install vtk quantized-mesh-tile (%s)" % ex)
+        print("map3d needs extra packages: pip install 'MAVProxy[map3d]' (%s)" % ex)
+        return
+    # map3d.py itself does not import VTK; the viewer child process does
+    missing = missing_packages()
+    if missing:
+        print(missing_packages_message(missing))
         return
 
     mlog = mestate.mlog

setup.py

@@ -98,8 +98,11 @@ def package_files(directory):
       install_requires=requirements,
       extras_require={
         'cesium': ['tornado'],
-        # map3d module (native 3D terrain map)
-        'map3d': ['vtk', 'quantized-mesh-tile'],
+        # map3d module (native 3D terrain map). quantized-mesh-tile needs
+        # numpy>=2, which leaves a distro opencv/matplotlib built against
+        # numpy 1 unimportable, so pull in builds that match
+        'map3d': ['vtk', 'quantized-mesh-tile',
+                  'opencv-python>=4.10', 'matplotlib>=3.9'],
         # restserver module
         'server': ['flask'],
         'recommended': ['flask', 'PyYAML', 'lxml', 'wxpython',

windows/mavproxy.spec

@@ -33,7 +33,10 @@ MAVExpAny = Analysis(['.\\tools\\MAVExplorer.py'],
                             'wx.lib.embeddedimage', 'wx.lib.imageutils', 'wx.lib.agw.aquabutton', 
                             'wx.lib.agw.gradientbutton', 'FileDialog', 'Dialog',
                             ] + collect_submodules('pymavlink'),
-             datas= [ ('tools\\graphs\\*.*', 'MAVProxy\\tools\\graphs' ) ],
+             # mavproxy_map\data holds the map icons, needed by both the 2D map
+             # and the 3D map that MAVExplorer can open
+             datas= [ ('tools\\graphs\\*.*', 'MAVProxy\\tools\\graphs' ),
+                      ('modules\\mavproxy_map\\data\\*.*', 'MAVProxy\\modules\\mavproxy_map\\data' )],
              hookspath=None,
              runtime_hooks=None,
              excludes= ['sphinx', 'docutils', 'alabaster', 'FixTk', 'tcl', 'tk', 'Tkinter'])