diff --git a/.github/workflows/windows_build.yml b/.github/workflows/windows_build.yml index 7a7c0191f1..8e17d2d896 100644 --- a/.github/workflows/windows_build.yml +++ b/.github/workflows/windows_build.yml @@ -33,7 +33,7 @@ jobs: run: ./installer.exe /verysilent /allusers /dir=inst - name: Build MAVProxy run: | - python3 -m pip install .[recommended] --user + python3 -m pip install .[recommended,map3d] --user python3 -m pip list - name: Prepare installer run: | diff --git a/MAVProxy/mavproxy.py b/MAVProxy/mavproxy.py index e0a97c2371..d30fcdb25e 100755 --- a/MAVProxy/mavproxy.py +++ b/MAVProxy/mavproxy.py @@ -377,6 +377,7 @@ def load_module(self, modname, quiet=False, **kwargs): # don't report an error return True ex = None + ex_modpath = None for modpath in modpaths: try: m = import_package(modpath) @@ -394,9 +395,16 @@ def load_module(self, modname, quiet=False, **kwargs): ex = "%s.init did not return a MPModule instance" % modname break except ImportError as msg: - ex = msg + # keep the first real failure, so a fallback modpath's "No + # module named X" cannot hide it. A ModuleNotFoundError naming + # the modpath we just tried only says that path does not exist, + # so let a later attempt replace it with its own reason + if (ex is None or + (isinstance(ex, ModuleNotFoundError) and ex.name == ex_modpath)): + ex = msg + ex_modpath = modpath if mpstate.settings.moddebug > 1: - print(get_exception_stacktrace(ex)) + print(get_exception_stacktrace(msg)) help_traceback = "" if mpstate.settings.moddebug < 3: help_traceback = " Use 'set moddebug 3' in the MAVProxy console to enable traceback" diff --git a/MAVProxy/modules/lib/mp_util.py b/MAVProxy/modules/lib/mp_util.py index 0e5aa80410..b7368bc4cf 100644 --- a/MAVProxy/modules/lib/mp_util.py +++ b/MAVProxy/modules/lib/mp_util.py @@ -595,3 +595,16 @@ def natural_sort_key(s): def sorted_natural(lst): '''sort using a 'natural' sort order''' return sorted(lst, key=natural_sort_key) + +def import_cv2(): + '''import cv2, explaining the usual cause of failure. A distro opencv is + built against one numpy major version, so pulling in a newer numpy (some + packages require numpy>=2) leaves cv2 unimportable''' + try: + import cv2 + return cv2 + except ImportError as ex: + raise ImportError( + "opencv (cv2) is not usable: %s. If numpy was upgraded then the " + "installed opencv may be built for the previous numpy; try " + "'pip install -U opencv-python'" % ex) diff --git a/MAVProxy/modules/mavproxy_console.py b/MAVProxy/modules/mavproxy_console.py index a6d9e6c065..be4f05fa94 100644 --- a/MAVProxy/modules/mavproxy_console.py +++ b/MAVProxy/modules/mavproxy_console.py @@ -116,6 +116,7 @@ def __init__(self, mpstate): self.add_menu(MPMenuSubMenu('MAVProxy', items=[MPMenuItem('Settings', 'Settings', 'menuSettings'), MPMenuItem('Show Map', 'Load Map', '# module load map'), + MPMenuItem('Show Map3d', 'Load 3D Map', '# module load map3d'), MPMenuItem('Show HUD', 'Load HUD', '# module load horizon'), MPMenuItem('Show Checklist', 'Load Checklist', '# module load checklist')])) self.vehicle_menu = MPMenuSubMenu('Vehicle', items=[]) diff --git a/MAVProxy/modules/mavproxy_map/mp_slipmap.py b/MAVProxy/modules/mavproxy_map/mp_slipmap.py index 9329626a06..96e7a09fee 100755 --- a/MAVProxy/modules/mavproxy_map/mp_slipmap.py +++ b/MAVProxy/modules/mavproxy_map/mp_slipmap.py @@ -7,7 +7,6 @@ ''' import time -import cv2 from MAVProxy.modules.mavproxy_map import mp_tile from MAVProxy.modules.lib import mp_util @@ -15,6 +14,8 @@ from MAVProxy.modules.lib import multiproc from MAVProxy.modules.mavproxy_map.mp_slipmap_util import * +cv2 = mp_util.import_cv2() + class MPSlipMap(): ''' diff --git a/MAVProxy/modules/mavproxy_map/mp_tile.py b/MAVProxy/modules/mavproxy_map/mp_tile.py index cfe864c270..b59cce82d3 100755 --- a/MAVProxy/modules/mavproxy_map/mp_tile.py +++ b/MAVProxy/modules/mavproxy_map/mp_tile.py @@ -26,7 +26,6 @@ import pathlib import string import time -import cv2 import numpy as np from math import log, tan, radians, degrees, sin, cos, exp, pi, asin, atan @@ -44,6 +43,8 @@ from MAVProxy.modules.lib import mp_util +cv2 = mp_util.import_cv2() + class TileException(Exception): '''tile error class''' @@ -218,6 +219,17 @@ def __init__(self, tile, zoom, scale, src, dst, service): (self.dstx, self.dsty) = dst +def default_cache_path(): + '''default root of the on-disk tile cache. HOME is not set on native + Windows, where the cwd may well be unwritable (eg. Program Files)''' + if 'HOME' in os.environ: + return os.path.join(os.environ['HOME'], '.tilecache') + if 'LOCALAPPDATA' in os.environ: + return os.path.join(os.environ['LOCALAPPDATA'], '.tilecache') + import tempfile + return os.path.join(tempfile.gettempdir(), '.tilecache') + + class MPTile: '''map tile object''' def __init__(self, cache_path=None, download=True, cache_size=500, @@ -225,14 +237,7 @@ def __init__(self, cache_path=None, download=True, cache_size=500, max_zoom=19, refresh_age=30*24*60*60): if cache_path is None: - try: - cache_path = os.path.join(os.environ['HOME'], '.tilecache') - except Exception: - if 'LOCALAPPDATA' in os.environ: - cache_path = os.path.join(os.environ['LOCALAPPDATA'], '.tilecache') - else: - import tempfile - cache_path = os.path.join(tempfile.gettempdir(), '.tilecache') + cache_path = default_cache_path() if not os.path.exists(cache_path): mp_util.mkdir_p(cache_path) @@ -696,7 +701,7 @@ def mp_icon(filename): with importlib.resources.open_binary(package, filename) as stream: raw = np.frombuffer(stream.read(), dtype=np.uint8) except Exception: - with open(os.path.join(os.path.dirname(__file__), 'data', filename)).read() as stream: + with open(os.path.join(os.path.dirname(__file__), 'data', filename), 'rb') as stream: raw = np.frombuffer(stream.read(), dtype=np.uint8) img = cv2.imdecode(raw, cv2.IMREAD_COLOR) return img diff --git a/MAVProxy/modules/mavproxy_map3d/__init__.py b/MAVProxy/modules/mavproxy_map3d/__init__.py index c5e71db50b..2f9e7c9377 100644 --- a/MAVProxy/modules/mavproxy_map3d/__init__.py +++ b/MAVProxy/modules/mavproxy_map3d/__init__.py @@ -13,7 +13,8 @@ from MAVProxy.modules.lib import mp_module from MAVProxy.modules.lib import mp_settings -from MAVProxy.modules.mavproxy_map3d.map3d import Map3D +from MAVProxy.modules.mavproxy_map3d.map3d import ( + Map3D, missing_packages, missing_packages_message) # fence colours as the 2D map's PolyFence layer uses them (OpenCV BGR) FENCE_INCLUSION_BGR = (0, 255, 0) @@ -107,6 +108,10 @@ def start_map(self): if self.map is not None and self.map.is_alive(): print("map3d already running") return + missing = missing_packages() + if missing: + print(missing_packages_message(missing)) + return self.map = Map3D(title="MAVProxy 3D Map", service=self.map3d_settings.service, zexag=self.map3d_settings.zexag, @@ -366,11 +371,13 @@ def send_kml(self, kml_mod=None): def idle_task(self): if self.map is None: return - if not self.map.is_alive(): - self.map = None - return + # drain events before dropping a dead child, so we don't lose the + # reason it failed to start + alive = self.map.is_alive() for event in self.map.check_events(): - if event[0] == 'render_settings': + if event[0] == 'startup_error': + print("map3d: the 3D view failed to start:\n%s" % event[1]) + elif event[0] == 'render_settings': (_, brightness, shading, wireframe, fpvfov) = event self.map3d_settings.terrainbrightness = brightness self.map3d_settings.terrainshading = shading @@ -378,6 +385,9 @@ def idle_task(self): self.map3d_settings.fpvfov = fpvfov elif event[0] == 'follow': self.follow = bool(event[1]) + if not alive: + self.map = None + return kml_mod = self.module('kmlread') if self._kml_state(kml_mod) != self.kml_change_state: self.send_kml(kml_mod) diff --git a/MAVProxy/modules/mavproxy_map3d/map3d.py b/MAVProxy/modules/mavproxy_map3d/map3d.py index b42f6373f9..7ec27d363e 100644 --- a/MAVProxy/modules/mavproxy_map3d/map3d.py +++ b/MAVProxy/modules/mavproxy_map3d/map3d.py @@ -3,11 +3,36 @@ (mirrors mp_slipmap) and pushes element/camera updates over a queue. ''' +import importlib.util import time import queue +import traceback from MAVProxy.modules.lib import multiproc +PACKAGES = ('vtk', 'quantized_mesh_tile') + + +def missing_packages(): + '''optional 3D map packages that are not installed. The viewer imports them + in a child process, where an ImportError would go unseen, so callers check + before starting a viewer. find_spec avoids importing VTK into the parent.''' + missing = [] + for name in PACKAGES: + try: + if importlib.util.find_spec(name) is None: + missing.append(name) + except Exception: + missing.append(name) + return missing + + +def missing_packages_message(missing): + # install via the extra, not 'pip install vtk quantized-mesh-tile': the + # latter pulls numpy>=2 without the opencv/matplotlib builds to match + return ("map3d needs extra packages: pip install 'MAVProxy[map3d]' " + "(missing %s)" % ', '.join(missing)) + class Map3D: def __init__(self, title="3D Map", service="MicrosoftSat", zexag=1.0, @@ -38,14 +63,22 @@ def __init__(self, title="3D Map", service="MicrosoftSat", zexag=1.0, def child_task(self): from MAVProxy.modules.lib import mp_util mp_util.child_close_fds() - from MAVProxy.modules.lib import wx_processguard # noqa: F401 - from MAVProxy.modules.lib.wx_loader import wx - from MAVProxy.modules.mavproxy_map3d.map3d_ui import Map3DFrame - - app = wx.App(False) - app.SetExitOnFrameDelete(True) - frame = Map3DFrame(self) - frame.Show() + try: + from MAVProxy.modules.lib import wx_processguard # noqa: F401 + from MAVProxy.modules.lib.wx_loader import wx + from MAVProxy.modules.mavproxy_map3d.map3d_ui import Map3DFrame + + app = wx.App(False) + app.SetExitOnFrameDelete(True) + frame = Map3DFrame(self) + frame.Show() + except (Exception, SystemExit): + # our stderr goes nowhere when MAVProxy is started from a GUI, so + # hand the failure to the parent rather than dying unexplained. + # wx exits rather than raising when it cannot open the display, + # hence SystemExit + self.event_queue.put(('startup_error', traceback.format_exc())) + return self.app_ready.set() app.MainLoop() diff --git a/MAVProxy/modules/mavproxy_map3d/map3d_ui.py b/MAVProxy/modules/mavproxy_map3d/map3d_ui.py index eed8432cb9..f7192fd142 100644 --- a/MAVProxy/modules/mavproxy_map3d/map3d_ui.py +++ b/MAVProxy/modules/mavproxy_map3d/map3d_ui.py @@ -9,7 +9,7 @@ import time from MAVProxy.modules.lib.wx_loader import wx -from vtk.wx.wxVTKRenderWindowInteractor import wxVTKRenderWindowInteractor +from vtkmodules.wx.wxVTKRenderWindowInteractor import wxVTKRenderWindowInteractor from MAVProxy.modules.mavproxy_map import mp_tile from MAVProxy.modules.mavproxy_map3d.camera import TerrainCamera, TerrainStyle diff --git a/MAVProxy/modules/mavproxy_map3d/terrain.py b/MAVProxy/modules/mavproxy_map3d/terrain.py index f404df0432..e6cb5ccd5b 100644 --- a/MAVProxy/modules/mavproxy_map3d/terrain.py +++ b/MAVProxy/modules/mavproxy_map3d/terrain.py @@ -16,7 +16,9 @@ import numpy as np import vtk -from vtk.util import numpy_support +# import via vtkmodules, not the vtk.* aliases: vtk is a plain module that fakes +# a package with __path__, which pyinstaller cannot follow into a frozen build +from vtkmodules.util import numpy_support from quantized_mesh_tile import decode as qmt_decode from quantized_mesh_tile.global_geodetic import GlobalGeodetic @@ -25,7 +27,7 @@ R = 6378137.0 QUANTIZED_BASE = "https://plot.ardupilot.org/quantized" -CACHE_DIR = os.path.join(os.environ.get("HOME", "."), ".tilecache", "quantized") +CACHE_DIR = os.path.join(mp_tile.default_cache_path(), "quantized") # ArduPilot tiles include the optional lighting extension. This decoder warns # when it skips that extension, but map3d computes its own VTK normals anyway. diff --git a/MAVProxy/tools/MAVExplorer.py b/MAVProxy/tools/MAVExplorer.py index 86b97661a2..69fd79d728 100755 --- a/MAVProxy/tools/MAVExplorer.py +++ b/MAVProxy/tools/MAVExplorer.py @@ -661,9 +661,15 @@ def cmd_map(args): def cmd_map3d(args): '''show a 3D map view: draped satellite imagery over terrain''' try: - from MAVProxy.modules.mavproxy_map3d.map3d import Map3D + from MAVProxy.modules.mavproxy_map3d.map3d import ( + Map3D, missing_packages, missing_packages_message) except ImportError as ex: - print("map3d needs extra packages: pip install vtk quantized-mesh-tile (%s)" % ex) + print("map3d needs extra packages: pip install 'MAVProxy[map3d]' (%s)" % ex) + return + # map3d.py itself does not import VTK; the viewer child process does + missing = missing_packages() + if missing: + print(missing_packages_message(missing)) return mlog = mestate.mlog diff --git a/setup.py b/setup.py index 168351a904..56b627a7c2 100755 --- a/setup.py +++ b/setup.py @@ -98,8 +98,11 @@ def package_files(directory): install_requires=requirements, extras_require={ 'cesium': ['tornado'], - # map3d module (native 3D terrain map) - 'map3d': ['vtk', 'quantized-mesh-tile'], + # map3d module (native 3D terrain map). quantized-mesh-tile needs + # numpy>=2, which leaves a distro opencv/matplotlib built against + # numpy 1 unimportable, so pull in builds that match + 'map3d': ['vtk', 'quantized-mesh-tile', + 'opencv-python>=4.10', 'matplotlib>=3.9'], # restserver module 'server': ['flask'], 'recommended': ['flask', 'PyYAML', 'lxml', 'wxpython', diff --git a/windows/mavproxy.spec b/windows/mavproxy.spec index d564a3c838..243134065e 100755 --- a/windows/mavproxy.spec +++ b/windows/mavproxy.spec @@ -33,7 +33,10 @@ MAVExpAny = Analysis(['.\\tools\\MAVExplorer.py'], 'wx.lib.embeddedimage', 'wx.lib.imageutils', 'wx.lib.agw.aquabutton', 'wx.lib.agw.gradientbutton', 'FileDialog', 'Dialog', ] + collect_submodules('pymavlink'), - datas= [ ('tools\\graphs\\*.*', 'MAVProxy\\tools\\graphs' ) ], + # mavproxy_map\data holds the map icons, needed by both the 2D map + # and the 3D map that MAVExplorer can open + datas= [ ('tools\\graphs\\*.*', 'MAVProxy\\tools\\graphs' ), + ('modules\\mavproxy_map\\data\\*.*', 'MAVProxy\\modules\\mavproxy_map\\data' )], hookspath=None, runtime_hooks=None, excludes= ['sphinx', 'docutils', 'alabaster', 'FixTk', 'tcl', 'tk', 'Tkinter'])