support for displaying images in webview

extensions/vi-mode/commands.lisp

@@ -198,16 +198,19 @@
 
 (define-command vi-forward-char (&optional (n 1)) (:universal)
   (let* ((p (current-point))
-         (max-offset (- (length (line-string p))
-                        (point-charpos p))))
+         (max-offset (with-point ((e p))
+                       (visual-line-end e)
+                       (count-characters p e))))
     (character-offset p (min n max-offset))))
 
 (define-command vi-backward-char (&optional (n 1)) (:universal)
   (let ((p (current-point)))
-    (dotimes (_ n)
-      (if (bolp p)
-          (return)
-          (character-offset p -1)))))
+    (with-point ((bol p))
+      (visual-line-beginning bol)
+      (dotimes (_ n)
+        (if (point<= p bol)
+            (return)
+            (character-offset p -1))))))
 
 (define-motion vi-next-line (&optional (n 1)) (:universal)
     (:type :line)
@@ -309,7 +312,7 @@
 
 (define-command vi-move-to-beginning-of-line () ()
   (with-point ((start (current-point)))
-    (line-start start)
+    (visual-line-beginning start)
     (or (text-property-at (current-point) :field -1)
         (previous-single-property-change (current-point)
                                          :field
@@ -318,7 +321,7 @@
 
 (define-command vi-move-to-end-of-line (&optional (n 1)) (:universal)
   (vi-line n)
-  (line-end (current-point)))
+  (visual-line-end (current-point)))
 
 (define-command vi-move-to-last-nonblank () ()
   (vi-move-to-end-of-line)

extensions/vi-mode/commands/utils.lisp

@@ -61,7 +61,10 @@
             (1- len)))))
 
 (defun fall-within-line (point)
-  (when (eolp point)
+  "clamps the given point to the character before the newline at the end of the line its in."
+  ;; a buffer-line end whose newline is hidden by a fold is mid-visual-line, so we dont clamp.
+  (when (and (eolp point)
+             (not (invisible-overlay-covering point)))
     (line-end point)
     (unless (bolp point)
       (character-offset point *cursor-offset*))))

frontends/server/frontend/editor.js

@@ -346,6 +346,10 @@ class BaseSurface {
 
   drawBlock(x, y, width, height, color) { }
   drawText(x, y, text, textWidth, attribute) { }
+  drawImage(x, y, widthCells, heightCells, pixelWidth, pixelHeight, url) { }
+
+  clearImages(yStart, yEnd) { }
+  clearAllImages() { }
 
   touch() {
     return;
@@ -388,6 +392,18 @@ class CanvasSurface extends BaseSurface {
     ctx.scale(ratio, ratio);
   }
 
+  move(x, y, pixelX, pixelY) {
+    super.move(x, y, pixelX, pixelY);
+    if (this.imageEls) {
+      for (const [, entry] of this.imageEls) this.positionImage(entry);
+    }
+  }
+
+  delete() {
+    this.clearAllImages();
+    super.delete();
+  }
+
   drawBlock(x, y, width, height, color) {
     const option = this.editor.option;
     this.drawingQueue.push(function(ctx) {
@@ -475,6 +491,69 @@ class CanvasSurface extends BaseSurface {
     });
   }
 
+  // images are rendered as DOM <img> elements on a layer above the canvas rather than into it.
+  imageBaseLeft() { return parseFloat(this.mainDOM.style.left) || 0; }
+  imageBaseTop() { return parseFloat(this.mainDOM.style.top) || 0; }
+
+  drawImage(x, y, widthCells, heightCells, pixelWidth, pixelHeight, url) {
+    if (!this.imageEls)
+      // mapping "x,y" to { el, url, x, y, widthCells, heightCells, pixelWidth, pixelHeight }
+      this.imageEls = new Map();
+    const key = x + ',' + y;
+    let entry = this.imageEls.get(key);
+    if (entry && entry.url !== url) {
+      entry.el.remove();
+      this.imageEls.delete(key);
+      entry = null;
+    }
+    if (!entry) {
+      const el = document.createElement('img');
+      el.style.position = 'absolute';
+      el.style.pointerEvents = 'none';
+      // above the surface's own canvas but below the modeline/floating windows.
+      el.style.zIndex = '1';
+      el.src = url;
+      this.mainDOM.parentNode.appendChild(el);
+      entry = { el, url };
+      this.imageEls.set(key, entry);
+    }
+    entry.x = x;
+    entry.y = y;
+    entry.widthCells = widthCells;
+    entry.heightCells = heightCells;
+    entry.pixelWidth = pixelWidth;
+    entry.pixelHeight = pixelHeight;
+    this.positionImage(entry);
+  }
+
+  positionImage(entry) {
+    const option = this.editor.option;
+    // we reserved widthCells x heightCells cells for this image.
+    const boxWidth = entry.widthCells * option.fontWidth;
+    const boxHeight = entry.heightCells * option.fontHeight;
+    entry.el.style.left = (this.imageBaseLeft() + entry.x * option.fontWidth) + 'px';
+    entry.el.style.top = (this.imageBaseTop() + entry.y * option.fontHeight) + 'px';
+    entry.el.style.width = (entry.pixelWidth != null ? Math.min(entry.pixelWidth, boxWidth) : boxWidth) + 'px';
+    entry.el.style.height = (entry.pixelHeight != null ? Math.min(entry.pixelHeight, boxHeight) : boxHeight) + 'px';
+  }
+
+  // remove image elements whose row span intersects [yStart, yEnd).
+  clearImages(yStart, yEnd) {
+    if (!this.imageEls) return;
+    for (const [key, entry] of this.imageEls) {
+      if (entry.y < yEnd && (entry.y + entry.heightCells) > yStart) {
+        entry.el.remove();
+        this.imageEls.delete(key);
+      }
+    }
+  }
+
+  clearAllImages() {
+    if (!this.imageEls) return;
+    for (const [, entry] of this.imageEls) entry.el.remove();
+    this.imageEls.clear();
+  }
+
   touch() {
     const ctx = this.mainDOM.getContext('2d');
     for (let fn of this.drawingQueue) {
@@ -778,14 +857,15 @@ class View {
     );
   }
 
-  clearEol(x, y) {
+  clearEol(x, y, height=1) {
     this.mainSurface.drawBlock(
       x,
       y,
       this.width - x,
-      1,
+      height,
       this.option.background,
     );
+    this.mainSurface.clearImages(y, y + height);
   }
 
   clearEob(x, y) {
@@ -796,6 +876,7 @@ class View {
       this.height - y,
       this.option.background,
     );
+    this.mainSurface.clearImages(y, this.height);
   }
 
   print(x, y, text, textWidth, attribute, font) {
@@ -809,6 +890,10 @@ class View {
     );
   }
 
+  printImage(x, y, width, height, pixelWidth, pixelHeight, url) {
+    this.mainSurface.drawImage(x, y, width, height, pixelWidth, pixelHeight, url);
+  }
+
   printToModeline(x, y, text, textWidth, attribute) {
     if (this.modelineSurface) {
       this.modelineSurface.drawText(
@@ -1166,6 +1251,7 @@ export class Editor {
       'clear-eol': this.clearEol.bind(this),
       'clear-eob': this.clearEob.bind(this),
       'put': this.put.bind(this),
+      'put-image': this.putImage.bind(this),
       'modeline-put': this.modelinePut.bind(this),
       'update-display': this.updateDisplay.bind(this),
       'move-cursor': this.moveCursor.bind(this),
@@ -1280,6 +1366,8 @@ export class Editor {
       size: this.getDisplaySize(),
       foreground: this.option.foreground,
       background: this.option.background,
+      fontWidth: this.option.fontWidth,
+      fontHeight: this.option.fontHeight,
     }, (response) => {
       this.updateForeground(response.foreground);
       this.updateBackground(response.background);
@@ -1364,9 +1452,9 @@ export class Editor {
     view.clear();
   }
 
-  clearEol({ viewInfo: { id }, x, y }) {
+  clearEol({ viewInfo: { id }, x, y, height }) {
     const view = this.findViewById(id);
-    view.clearEol(x, y);
+    view.clearEol(x, y, height);
   }
 
   clearEob({ viewInfo: { id }, x, y }) {
@@ -1379,6 +1467,11 @@ export class Editor {
     view.print(x, y, text, textWidth, attribute, font);
   }
 
+  putImage({ viewInfo: { id }, x, y, width, height, pixelWidth, pixelHeight, url }) {
+    const view = this.findViewById(id);
+    view.printImage(x, y, width, height, pixelWidth, pixelHeight, url);
+  }
+
   modelinePut({ viewInfo: { id }, x, y, text, textWidth, attribute }) {
     const view = this.findViewById(id);
     view.printToModeline(x, y, text, textWidth, attribute);

frontends/server/main.lisp

@@ -101,7 +101,12 @@
    (message-queue :initform (queue:make-queue)
                   :reader jsonrpc-message-queue)
    (editor-thread :initform nil
-                  :accessor jsonrpc-editor-thread))
+                  :accessor jsonrpc-editor-thread)
+   ;; pixel size of one character cell, reported by the client.
+   (cell-width :initform nil
+               :accessor jsonrpc-cell-width)
+   (cell-height :initform nil
+                :accessor jsonrpc-cell-height))
   (:default-initargs
    :name :jsonrpc
    :redraw-after-modifying-floating-window t
@@ -162,6 +167,10 @@ the same immutable instance for every subsequent message."
         (let ((width (gethash "width" size))
               (height (gethash "height" size)))
           (resize-display jsonrpc width height)))
+      (alexandria:when-let ((fw (gethash "fontWidth" params)))
+        (when (plusp fw) (setf (jsonrpc-cell-width jsonrpc) fw)))
+      (alexandria:when-let ((fh (gethash "fontHeight" params)))
+        (when (plusp fh) (setf (jsonrpc-cell-height jsonrpc) fh)))
       (when background
         (alexandria:when-let (color (lem:parse-color background))
           (setf (jsonrpc-background-color jsonrpc) color)))
@@ -584,7 +593,34 @@ the same immutable instance for every subsequent message."
   (lem-core:string-width (lem-core/display:text-object-string drawing-object)))
 
 (defmethod object-width ((drawing-object display:image-object))
-  0)
+  ;; width in character cells. when :pixel-width is given (and the client's cell size is known),
+  ;; round it up to whole cells so the column reserves enough grid space. otherwise use :width
+  ;; (a cell count) from the attribute.
+  (let ((pw (image-pixel-dimension drawing-object :pixel-width))
+        (cw (jsonrpc-cell-width (lem-core:implementation))))
+    (if (and pw cw)
+        (ceiling pw cw)
+        (or (display:image-object-width drawing-object) 1))))
+
+(defgeneric object-height (drawing-object)
+  (:documentation "height of DRAWING-OBJECT in character cells.
+we advance the vertical position of the next line by the tallest object's height (see
+`max-height-of-objects'), so returning more than 1 for an image makes its line grow to fit."))
+
+(defmethod object-height (drawing-object)
+  1)
+
+(defmethod object-height ((drawing-object display:image-object))
+  (let ((ph (image-pixel-dimension drawing-object :pixel-height))
+        (ch (jsonrpc-cell-height (lem-core:implementation))))
+    (if (and ph ch)
+        (ceiling ph ch)
+        (or (display:image-object-height drawing-object) 1))))
+
+(defun image-pixel-dimension (object key)
+  "pixel value of KEY (:pixel-width / :pixel-height) on OBJECT's attribute, or NIL."
+  (let ((attribute (display:image-object-attribute object)))
+    (and attribute (lem:attribute-value attribute key))))
 
 (defgeneric draw-object (jsonrpc object x y view))
 
@@ -703,8 +739,42 @@ same hash."
          attribute
          :text-width width)))
 
+(defun image-object-url (object)
+  "return a URL the JS client can load for OBJECT's image, or NIL.
+a pathname or plain-string path is served through the existing /local static route.
+a string already carrying a data:/https: URL is passed through unchanged."
+  (let ((image (display:image-object-image object)))
+    (typecase image
+      (pathname (format nil "/local~A" (namestring image)))
+      (string (if (or (alexandria:starts-with-subseq "data:" image)
+                      (alexandria:starts-with-subseq "http:" image)
+                      (alexandria:starts-with-subseq "https:" image))
+                  image
+                  (format nil "/local~A" image)))
+      (t nil))))
+
 (defmethod draw-object (jsonrpc (object display:image-object) x y view)
-  (values))
+  (let ((url (image-object-url object)))
+    (when url
+      (with-error-handler ()
+        ;; use the attribute's :pixel-width/:pixel-height if given, else the reserved cell box
+        ;; (cells * cell pixel size) when the cell size is known.
+        (let ((pw (or (image-pixel-dimension object :pixel-width)
+                      (alexandria:when-let ((cw (jsonrpc-cell-width jsonrpc)))
+                        (* (object-width object) cw))))
+              (ph (or (image-pixel-dimension object :pixel-height)
+                      (alexandria:when-let ((ch (jsonrpc-cell-height jsonrpc)))
+                        (* (object-height object) ch)))))
+          (notify* jsonrpc
+                   "put-image"
+                   (hash "viewInfo" (view-id-hash view)
+                         "x" x
+                         "y" y
+                         "width" (object-width object)
+                         "height" (object-height object)
+                         "pixelWidth" pw
+                         "pixelHeight" ph
+                         "url" url)))))))
 
 (defun render-line (jsonrpc view x y objects)
   (loop :for object :in objects
@@ -719,11 +789,14 @@ same hash."
 
 (defmethod lem-if:render-line ((jsonrpc jsonrpc) view x y objects height)
   (with-error-handler ()
+    ;; clear the line's full height (not just one row) since a tall object such as an image may
+    ;; occupy several rows.
     (notify* jsonrpc
              "clear-eol"
              (hash "viewInfo" (view-id-hash view)
                    "x" x
-                   "y" y))
+                   "y" y
+                   "height" height))
     (render-line jsonrpc view x y objects)))
 
 (defmethod lem-if:render-line-on-modeline ((jsonrpc jsonrpc) view left-objects right-objects
@@ -745,7 +818,7 @@ same hash."
   (object-width drawing-object))
 
 (defmethod lem-if:object-height ((jsonrpc jsonrpc) drawing-object)
-  1)
+  (object-height drawing-object))
 
 (defmethod lem-if:clear-to-end-of-window ((jsonrpc jsonrpc) view y)
   (notify* jsonrpc

lem-tests.asd

@@ -79,6 +79,7 @@
                (:file "filer")
                (:file "listener-mode")
                (:file "interface")
-               (:file "display-cache"))
+               (:file "display-cache")
+               (:file "visual-line"))
   :perform (test-op (o c)
                     (symbol-call :rove :run c)))