Re: [Emacs-diffs] master 51e7e46: Font-lock elisp macros/special forms dynamically

[Tassilo Horn]

Stefan Monnier <address@hidden> writes:

>> > Any suggestions how/where to update existing buffers?
>
> My suggestion would be to do it from after-load-functions, i.e. using
> the exact same code as the code used for the alternative
> implementation ;-)

Yes, that's what I'm doing now.  In addition, `defmacro' does update as
well (in case of a new macro or a redefinition with different
'no-font-lock-keyword declaration).  So with that, you get instant
fontification updates no matter if you C-x C-e a macro definition or
load a file containing macro definitions.

There's at most one update per C-x C-e on a defmacro form, and exactly
one update per file-load.  And updates are cheaper since no regexp has
to be computed.  I've measured it, and compared to `font-lock-flush'
(with `jit-lock-mode' enabled) collecting all macros and builting a
regexp using `regexp-opt' is about 80 times more expensive.

>> Isn't font-lock-flush supposed to be cheap?
>
> If you use jit-lock-mode, yes.  If not, no.

Using it is the default.  Is there a good reason a user might have
disabled it for elisp buffers?

If no one objects, I'm going to install the patch below anytime soon
when the master branch bootstraps again without

  Eager macro-expansion failure: (void-function cl-every)

Of course, then we still need to handle the function-like macros.
Daniel's suggestion of using the `debug' declaration for that doesn't
work (or I don't get it).  At least, having a debug declaration doesn't
imply being "non-function-like", and neither does having no debug
declaration imply being "function-like".

Bye,
Tassilo

>From 2724f2899a04291e3b4ce5b6f8ddd186165ae40d Mon Sep 17 00:00:00 2001
From: Tassilo Horn <address@hidden>
Date: Mon, 16 Mar 2015 10:25:14 +0100
Subject: [PATCH] Improve dynamic elisp keyword font-locking

* emacs-lisp/byte-run.el (macro-declarations-alist): New
declaration no-font-lock-keyword.
(defmacro): Flush font-lock in existing elisp buffers.

* emacs-lisp/lisp-mode.el (lisp--el-update-after-load)
(lisp--el-update-macro-regexp, lisp--el-macro-regexp): Delete
functions and defconst.
(lisp--el-match-keyword): Rename from lisp--el-match-macro.
(lisp--el-font-lock-flush-elisp-buffers): New function.
(lisp-mode-variables): Remove code for updating
lisp--el-macro-regexp, and add
lisp--el-font-lock-flush-elisp-buffers to after-load-functions.
---
 lisp/ChangeLog               | 15 ++++++++++++++
 lisp/emacs-lisp/byte-run.el  | 28 ++++++++++++++++++++-----
 lisp/emacs-lisp/lisp-mode.el | 49 ++++++++++++++++++--------------------------
 3 files changed, 58 insertions(+), 34 deletions(-)

diff --git a/lisp/ChangeLog b/lisp/ChangeLog
index cbd1bce..88c3c01 100644
--- a/lisp/ChangeLog
+++ b/lisp/ChangeLog
@@ -1,3 +1,18 @@
+2015-03-17  Tassilo Horn  <address@hidden>
+
+       * emacs-lisp/byte-run.el (macro-declarations-alist): New
+       declaration no-font-lock-keyword.
+       (defmacro): Flush font-lock in existing elisp buffers.
+
+       * emacs-lisp/lisp-mode.el (lisp--el-update-after-load)
+       (lisp--el-update-macro-regexp, lisp--el-macro-regexp): Delete
+       functions and defconst.
+       (lisp--el-match-keyword): Rename from lisp--el-match-macro.
+       (lisp--el-font-lock-flush-elisp-buffers): New function.
+       (lisp-mode-variables): Remove code for updating
+       lisp--el-macro-regexp, and add
+       lisp--el-font-lock-flush-elisp-buffers to after-load-functions.
+
 2015-03-16  Alan Mackenzie  <address@hidden>
 
        Edebug: Allow "S" to work during trace mode.  Fixes debbugs #20074.
diff --git a/lisp/emacs-lisp/byte-run.el b/lisp/emacs-lisp/byte-run.el
index caa7e3d..01c1af7 100644
--- a/lisp/emacs-lisp/byte-run.el
+++ b/lisp/emacs-lisp/byte-run.el
@@ -147,11 +147,16 @@ This is used by `declare'.")
 (defvar macro-declarations-alist
   (cons
    (list 'debug
-         #'(lambda (name _args spec)
-             (list 'progn :autoload-end
-                   (list 'put (list 'quote name)
-                         ''edebug-form-spec (list 'quote spec)))))
-   defun-declarations-alist)
+        #'(lambda (name _args spec)
+            (list 'progn :autoload-end
+                  (list 'put (list 'quote name)
+                        ''edebug-form-spec (list 'quote spec)))))
+   (cons
+    (list 'no-font-lock-keyword
+         #'(lambda (name _args val)
+             (list 'function-put (list 'quote name)
+                   ''no-font-lock-keyword (list 'quote val))))
+    defun-declarations-alist))
   "List associating properties of macros to their macro expansion.
 Each element of the list takes the form (PROP FUN) where FUN is a function.
 For each (PROP . VALUES) in a macro's declaration, the FUN corresponding
@@ -201,6 +206,19 @@ The return value is undefined.
                          (message "Warning: Unknown macro property %S in %S"
                                   (car x) name))))
                  decls)))
+          ;; Refresh font-lock if this is a new macro, or it is an
+          ;; existing macro whose 'no-font-lock-keyword declaration
+          ;; has changed.
+          (if (and
+               ;; During bootstrap, subr.el isn't loaded, but then we
+               ;; don't need to refresh anyway.
+               (fboundp 'macrop)
+               (fboundp 'lisp--el-font-lock-flush-elisp-buffers)
+               (macrop name)
+               (member `(function-put ',name 'no-font-lock-keyword
+                                      ',(get name 'no-font-lock-keyword))
+                       declarations))
+              (lisp--el-font-lock-flush-elisp-buffers))
           (if declarations
               (cons 'prog1 (cons def declarations))
             def))))))
diff --git a/lisp/emacs-lisp/lisp-mode.el b/lisp/emacs-lisp/lisp-mode.el
index b4f87fd..6b30773 100644
--- a/lisp/emacs-lisp/lisp-mode.el
+++ b/lisp/emacs-lisp/lisp-mode.el
@@ -181,32 +181,25 @@
             nil)))
     res))
 
-(defconst lisp--el-macro-regexp nil
-  "A regular expression matching all loaded elisp macros.
-Can be updated using `lisp--el-update-macro-regexp' after new
-macros were defined.")
-
-(defun lisp--el-update-macro-regexp ()
-  "Update `lisp--el-update-macro-regexp' from `obarray'.
-Return non-nil only if the old and new value are different."
-  (let ((old-regex lisp--el-macro-regexp)
-       (elisp-macros nil))
-    (mapatoms (lambda (a)
-               (when (or (macrop a) (special-form-p a))
-                 (push (symbol-name a) elisp-macros))))
-    (setq lisp--el-macro-regexp
-         (concat "(" (regexp-opt elisp-macros t) "\\_>"))
-    (not (string= old-regex lisp--el-macro-regexp))))
-
-(defun lisp--el-update-after-load (_file)
-  "Update `lisp--el-macro-regexp' and adjust font-lock in existing buffers."
-  (when (lisp--el-update-macro-regexp)
+(defun lisp--el-match-keyword (limit)
+  (catch 'found
+    (while (re-search-forward "(\\(\\(?:\\sw\\|\\s_\\)+\\)\\_>" limit t)
+      (let ((sym (intern-soft (match-string 1))))
+       (when (or (special-form-p sym)
+                 (and (macrop sym)
+                      (not (get sym 'no-font-lock-keyword))))
+         (throw 'found t))))))
+
+(defun lisp--el-font-lock-flush-elisp-buffers (&optional file)
+  ;; Don't flush during load unless called from after-load-functions.
+  ;; In that case, FILE is non-nil.  It's somehow strange that
+  ;; load-in-progress is t when an after-load-function is called since
+  ;; that should run *after* the load...
+  (when (or (not load-in-progress) file)
     (dolist (buf (buffer-list))
-      (when (derived-mode-p 'emacs-lisp-mode)
-       (font-lock-flush)))))
-
-(defun lisp--el-match-macro (limit)
-  (re-search-forward lisp--el-macro-regexp limit t))
+      (with-current-buffer buf
+       (when (derived-mode-p 'emacs-lisp-mode)
+         (font-lock-flush))))))
 
 (pcase-let
     ((`(,vdefs ,tdefs
@@ -362,7 +355,7 @@ Return non-nil only if the old and new value are different."
      `( ;; Regexp negated char group.
        ("\\[\\(\\^\\)" 1 font-lock-negation-char-face prepend)
        ;; Control structures.  Common Lisp forms.
-       (lisp--el-match-macro . 1)
+       (lisp--el-match-keyword . 1)
        ;; Exit/Feature symbols as constants.
        (,(concat "(\\(catch\\|throw\\|featurep\\|provide\\|require\\)\\_>"
                  "[ \t']*\\(\\(?:\\sw\\|\\s_\\)+\\)?")
@@ -543,9 +536,7 @@ font-lock keywords will not be case sensitive."
           . lisp-font-lock-syntactic-face-function)))
   (setq-local prettify-symbols-alist lisp--prettify-symbols-alist)
   (when elisp
-    (unless lisp--el-macro-regexp
-      (lisp--el-update-macro-regexp))
-    (add-hook 'after-load-functions #'lisp--el-update-after-load)
+    (add-hook 'after-load-functions #'lisp--el-font-lock-flush-elisp-buffers)
     (setq-local electric-pair-text-pairs
                 (cons '(?\` . ?\') electric-pair-text-pairs)))
   (setq-local electric-pair-skip-whitespace 'chomp)
-- 
2.3.3