Unifying "foo-mode"s and "foo-ts-mode"s

[Philip Kaludercic]

Eli Zaretskii <eliz@gnu.org> writes:

> You can try.  I would like to start a full feature freeze in a day or
> two, so I'm not sure you will have enough time.  And it isn't like we
> didn't try various approaches during the past two months, so frankly I
> don't think that a better way even exists.  But if you come up with
> some very bright idea, who knows?

I have attached a sketch of my proposal with support for Python.
Instead of a separate python-ts-mode, we regulate tree-sitter support
using a user option `treesit-enabled-modes'.  It can either be a list

--8<---------------cut here---------------start------------->8---
  (setq treesit-enabled-modes '(python-mode c-mode))
--8<---------------cut here---------------end--------------->8---

or generally enable tree-sitter

--8<---------------cut here---------------start------------->8---
  (setq treesit-enabled-modes t)
--8<---------------cut here---------------end--------------->8---

All a major modes has to do is pass a parser configuration

--8<---------------cut here---------------start------------->8---
  (define-derived-mode python-mode prog-mode "Python"
    "Major mode for editing Python files.

  \\{python-mode-map}"
    :syntax-table python-mode-syntax-table
    :parser-conf python-mode--treesit-conf
   ...
--8<---------------cut here---------------end--------------->8---

that expands to

--8<---------------cut here---------------start------------->8---
  (when-let
      ((conf python-mode--treesit-conf)
       ((cond
         ((listp treesit-enabled-modes)
          (memq 'python-mode treesit-enabled-modes))
         ((eq treesit-enabled-modes t))))
       ((treesit-ready-p
         (nth 0 conf)))
       (parser
        (treesit-parser-create
         (nth 0 conf))))
    (setq-local treesit-font-lock-feature-list
                (nth 1 conf)
                treesit-font-lock-settings
                (nth 2 conf)
                treesit-defun-name-function
                (nth 3 conf)
                treesit-defun-type-regexp
                (nth 4 conf)
                imenu-create-index-function
                (nth 5 conf))
    (treesit-major-mode-setup))
--8<---------------cut here---------------end--------------->8---

at *the end* of the major mode definition.  Note that if no parser
configuration was parsed, the entire expression is byte-compiled away,
so there is no run-time overhead for other modes.

The parser configuration is currently a list but if might as well be a
vector, a structure or anything else.  This is just a rushed proposal to
meet the deadline.  How does it look like?

diff --git a/lisp/emacs-lisp/derived.el b/lisp/emacs-lisp/derived.el
index 260fc3bf47..d6a1f4af00 100644
--- a/lisp/emacs-lisp/derived.el
+++ b/lisp/emacs-lisp/derived.el
@@ -143,6 +143,8 @@ define-derived-mode
            :interactive BOOLEAN
                    Whether the derived mode should be `interactive' or not.
                    The default is t.
+           :parser-conf CONF
+                   A tree-sitter parser configuration.
 
 BODY:      forms to execute just before running the
            hooks for the new mode.  Do not use `interactive' here.
@@ -192,6 +194,7 @@ define-derived-mode
        (hook (derived-mode-hook-name child))
        (group nil)
         (interactive t)
+        (parser-conf nil)
         (after-hook nil))
 
     ;; Process the keyword args.
@@ -202,6 +205,7 @@ define-derived-mode
        (:syntax-table (setq syntax (pop body)) (setq declare-syntax nil))
         (:after-hook (setq after-hook (pop body)))
         (:interactive (setq interactive (pop body)))
+        (:parser-conf (setq parser-conf (pop body)))
        (_ (pop body))))
 
     (setq docstring (derived-mode-make-docstring
@@ -285,7 +289,22 @@ define-derived-mode
          ,(when abbrev `(setq local-abbrev-table ,abbrev))
                                        ; Splice in the body (if any).
          ,@body
-         )
+
+          ;; Activate tree-sitter if requested and available.
+          (when-let ((conf ,parser-conf)
+                      ((cond
+                        ((listp treesit-enabled-modes)
+                         (memq ',child treesit-enabled-modes))
+                        ((eq treesit-enabled-modes t))))
+                      ((treesit-ready-p (nth 0 conf)))
+                      (parser (treesit-parser-create (nth 0 conf))))
+             (setq-local
+              treesit-font-lock-feature-list (nth 1 conf)
+              treesit-font-lock-settings     (nth 2 conf)
+              treesit-defun-name-function    (nth 3 conf)
+              treesit-defun-type-regexp      (nth 4 conf)
+              imenu-create-index-function    (nth 5 conf))
+             (treesit-major-mode-setup)))
         ,@(when after-hook
             `((push (lambda () ,after-hook) delayed-after-hook-functions)))
         ;; Run the hooks (and delayed-after-hook-functions), if any.
diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index 9a6f807f4f..5ad90e6d71 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -405,9 +405,6 @@ python-mode-map
     map)
   "Keymap for `python-mode'.")
 
-(defvar python-ts-mode-map (copy-keymap python-mode-map)
-  "Keymap for `(copy-keymap python-mode-map)'.")
-
 
 ;;; Python specialized rx
 
@@ -1072,124 +1069,6 @@ python--treesit-fontify-string
     (treesit-fontify-with-override
      string-beg string-end face override start end)))
 
-(defvar python--treesit-settings
-  (treesit-font-lock-rules
-   :feature 'comment
-   :language 'python
-   '((comment) @font-lock-comment-face)
-
-   :feature 'string
-   :language 'python
-   '((string) @python--treesit-fontify-string)
-
-   :feature 'string-interpolation
-   :language 'python
-   :override t
-   '((interpolation (identifier) @font-lock-variable-name-face))
-
-   :feature 'definition
-   :language 'python
-   '((function_definition
-      name: (identifier) @font-lock-function-name-face)
-     (class_definition
-      name: (identifier) @font-lock-type-face))
-
-   :feature 'function
-   :language 'python
-   '((function_definition
-      name: (identifier) @font-lock-function-name-face)
-     (call function: (identifier) @font-lock-function-name-face)
-     (call function: (attribute
-                      attribute: (identifier) @font-lock-function-name-face)))
-
-   :feature 'keyword
-   :language 'python
-   `([,@python--treesit-keywords] @font-lock-keyword-face
-     ((identifier) @font-lock-keyword-face
-      (:match "^self$" @font-lock-keyword-face)))
-
-   :feature 'builtin
-   :language 'python
-   `(((identifier) @font-lock-builtin-face
-      (:match ,(rx-to-string
-                `(seq bol
-                      (or ,@python--treesit-builtins
-                          ,@python--treesit-special-attributes)
-                      eol))
-              @font-lock-builtin-face)))
-
-   :feature 'constant
-   :language 'python
-   '([(true) (false) (none)] @font-lock-constant-face)
-
-   :feature 'assignment
-   :language 'python
-   `(;; Variable names and LHS.
-     (assignment left: (identifier)
-                 @font-lock-variable-name-face)
-     (assignment left: (attribute
-                        attribute: (identifier)
-                        @font-lock-property-face))
-     (pattern_list (identifier)
-                   @font-lock-variable-name-face)
-     (tuple_pattern (identifier)
-                    @font-lock-variable-name-face)
-     (list_pattern (identifier)
-                   @font-lock-variable-name-face)
-     (list_splat_pattern (identifier)
-                         @font-lock-variable-name-face))
-
-   :feature 'decorator
-   :language 'python
-   '((decorator "@" @font-lock-type-face)
-     (decorator (call function: (identifier) @font-lock-type-face))
-     (decorator (identifier) @font-lock-type-face))
-
-   :feature 'type
-   :language 'python
-   `(((identifier) @font-lock-type-face
-      (:match ,(rx-to-string
-                `(seq bol (or ,@python--treesit-exceptions)
-                      eol))
-              @font-lock-type-face))
-     (type (identifier) @font-lock-type-face))
-
-   :feature 'escape-sequence
-   :language 'python
-   :override t
-   '((escape_sequence) @font-lock-escape-face)
-
-   :feature 'number
-   :language 'python
-   '([(integer) (float)] @font-lock-number-face)
-
-   :feature 'property
-   :language 'python
-   '((attribute
-      attribute: (identifier) @font-lock-property-face)
-     (class_definition
-      body: (block
-             (expression_statement
-              (assignment left:
-                          (identifier) @font-lock-property-face)))))
-
-   :feature 'operator
-   :language 'python
-   `([,@python--treesit-operators] @font-lock-operator-face)
-
-   :feature 'bracket
-   :language 'python
-   '(["(" ")" "[" "]" "{" "}"] @font-lock-bracket-face)
-
-   :feature 'delimiter
-   :language 'python
-   '(["," "." ":" ";" (ellipsis)] @font-lock-delimiter-face)
-
-   :feature 'variable
-   :language 'python
-   '((identifier) @python--treesit-fontify-variable))
-  "Tree-sitter font-lock settings.")
-
 (defun python--treesit-variable-p (node)
   "Check whether NODE is a variable.
 NODE's type should be \"identifier\"."
@@ -6541,13 +6420,145 @@ python-electric-pair-string-delimiter
 (defvar electric-indent-inhibit)
 (defvar prettify-symbols-alist)
 
+(defvar python-mode--treesit-conf
+  (list
+   'python
+   ;; font-lock feature list
+   '(( comment definition)
+     ( keyword string type)
+     ( assignment builtin constant decorator
+       escape-sequence number property string-interpolation )
+     ( bracket delimiter function operator variable))
+   ;; font-lock settings
+   (treesit-font-lock-rules
+    :feature 'comment
+    :language 'python
+    '((comment) @font-lock-comment-face)
+
+    :feature 'string
+    :language 'python
+    '((string) @python--treesit-fontify-string)
+
+    :feature 'string-interpolation
+    :language 'python
+    :override t
+    '((interpolation (identifier) @font-lock-variable-name-face))
+
+    :feature 'definition
+    :language 'python
+    '((function_definition
+       name: (identifier) @font-lock-function-name-face)
+      (class_definition
+       name: (identifier) @font-lock-type-face))
+
+    :feature 'function
+    :language 'python
+    '((function_definition
+       name: (identifier) @font-lock-function-name-face)
+      (call function: (identifier) @font-lock-function-name-face)
+      (call function: (attribute
+                       attribute: (identifier) @font-lock-function-name-face)))
+
+    :feature 'keyword
+    :language 'python
+    `([,@python--treesit-keywords] @font-lock-keyword-face
+      ((identifier) @font-lock-keyword-face
+       (:match "^self$" @font-lock-keyword-face)))
+
+    :feature 'builtin
+    :language 'python
+    `(((identifier) @font-lock-builtin-face
+       (:match ,(rx-to-string
+                 `(seq bol
+                       (or ,@python--treesit-builtins
+                           ,@python--treesit-special-attributes)
+                       eol))
+               @font-lock-builtin-face)))
+
+    :feature 'constant
+    :language 'python
+    '([(true) (false) (none)] @font-lock-constant-face)
+
+    :feature 'assignment
+    :language 'python
+    `(;; Variable names and LHS.
+      (assignment left: (identifier)
+                  @font-lock-variable-name-face)
+      (assignment left: (attribute
+                         attribute: (identifier)
+                         @font-lock-property-face))
+      (pattern_list (identifier)
+                    @font-lock-variable-name-face)
+      (tuple_pattern (identifier)
+                     @font-lock-variable-name-face)
+      (list_pattern (identifier)
+                    @font-lock-variable-name-face)
+      (list_splat_pattern (identifier)
+                          @font-lock-variable-name-face))
+
+    :feature 'decorator
+    :language 'python
+    '((decorator "@" @font-lock-type-face)
+      (decorator (call function: (identifier) @font-lock-type-face))
+      (decorator (identifier) @font-lock-type-face))
+
+    :feature 'type
+    :language 'python
+    `(((identifier) @font-lock-type-face
+       (:match ,(rx-to-string
+                 `(seq bol (or ,@python--treesit-exceptions)
+                       eol))
+               @font-lock-type-face))
+      (type (identifier) @font-lock-type-face))
+
+    :feature 'escape-sequence
+    :language 'python
+    :override t
+    '((escape_sequence) @font-lock-escape-face)
+
+    :feature 'number
+    :language 'python
+    '([(integer) (float)] @font-lock-number-face)
+
+    :feature 'property
+    :language 'python
+    '((attribute
+       attribute: (identifier) @font-lock-property-face)
+      (class_definition
+       body: (block
+              (expression_statement
+               (assignment left:
+                           (identifier) @font-lock-property-face)))))
+
+    :feature 'operator
+    :language 'python
+    `([,@python--treesit-operators] @font-lock-operator-face)
+
+    :feature 'bracket
+    :language 'python
+    '(["(" ")" "[" "]" "{" "}"] @font-lock-bracket-face)
+
+    :feature 'delimiter
+    :language 'python
+    '(["," "." ":" ";" (ellipsis)] @font-lock-delimiter-face)
+
+    :feature 'variable
+    :language 'python
+    '((identifier) @python--treesit-fontify-variable))
+   ;; defun name function
+   #'python--treesit-defun-name
+   ;; defun regexp
+   (rx (or "function" "class") "_definition")
+   ;; imenu function
+   #'python-imenu-create-index))
+
 ;;;###autoload
-(define-derived-mode python-base-mode prog-mode "Python"
-  "Generic major mode for editing Python files.
+(define-derived-mode python-mode prog-mode "Python"
+  "Major mode for editing Python files.
 
-This is a generic major mode intended to be inherited by
-concrete implementations.  Currently there are two concrete
-implementations: `python-mode' and `python-ts-mode'."
+\\{python-mode-map}"
+  :syntax-table python-mode-syntax-table
+  :parser-conf python-mode--treesit-conf
   (setq-local tab-width 8)
   (setq-local indent-tabs-mode nil)
 
@@ -6603,20 +6614,19 @@ python-base-mode
                       #'python-eldoc-function))))
 
   ;; TODO: Use tree-sitter to figure out the block in `python-ts-mode'.
-  (dolist (mode '(python-mode python-ts-mode))
-    (add-to-list
-     'hs-special-modes-alist
-     `(,mode
-       ,python-nav-beginning-of-block-regexp
-       ;; Use the empty string as end regexp so it doesn't default to
-       ;; "\\s)".  This way parens at end of defun are properly hidden.
-       ""
-       "#"
-       python-hideshow-forward-sexp-function
-       nil
-       python-nav-beginning-of-block
-       python-hideshow-find-next-block
-       python-info-looking-at-beginning-of-block)))
+  (add-to-list
+   'hs-special-modes-alist
+   `(python-mode
+     ,python-nav-beginning-of-block-regexp
+     ;; Use the empty string as end regexp so it doesn't default to
+     ;; "\\s)".  This way parens at end of defun are properly hidden.
+     ""
+     "#"
+     python-hideshow-forward-sexp-function
+     nil
+     python-nav-beginning-of-block
+     python-hideshow-find-next-block
+     python-info-looking-at-beginning-of-block))
 
   (setq-local outline-regexp (python-rx (* space) block-start))
   (setq-local outline-level
@@ -6630,13 +6640,8 @@ python-base-mode
 
   (make-local-variable 'python-shell-internal-buffer)
 
-  (add-hook 'flymake-diagnostic-functions #'python-flymake nil t))
+  (add-hook 'flymake-diagnostic-functions #'python-flymake nil t)
 
-;;;###autoload
-(define-derived-mode python-mode python-base-mode "Python"
-  "Major mode for editing Python files.
-
-\\{python-mode-map}"
   (setq-local font-lock-defaults
               `(,python-font-lock-keywords
                 nil nil nil nil
@@ -6654,32 +6659,6 @@ python-mode
   (when python-indent-guess-indent-offset
     (python-indent-guess-indent-offset)))
 
-;;;###autoload
-(define-derived-mode python-ts-mode python-base-mode "Python"
-  "Major mode for editing Python files, using tree-sitter library.
-
-\\{python-ts-mode-map}"
-  :syntax-table python-mode-syntax-table
-  (when (treesit-ready-p 'python)
-    (treesit-parser-create 'python)
-    (setq-local treesit-font-lock-feature-list
-                '(( comment definition)
-                  ( keyword string type)
-                  ( assignment builtin constant decorator
-                    escape-sequence number property string-interpolation )
-                  ( bracket delimiter function operator variable)))
-    (setq-local treesit-font-lock-settings python--treesit-settings)
-    (setq-local imenu-create-index-function
-                #'python-imenu-treesit-create-index)
-    (setq-local treesit-defun-type-regexp (rx (or "function" "class")
-                                              "_definition"))
-    (setq-local treesit-defun-name-function
-                #'python--treesit-defun-name)
-    (treesit-major-mode-setup)
-
-    (when python-indent-guess-indent-offset
-      (python-indent-guess-indent-offset))))
-
 ;;; Completion predicates for M-x
 ;; Commands that only make sense when editing Python code
 (dolist (sym '(python-add-import
diff --git a/lisp/treesit.el b/lisp/treesit.el
index 2130cd0061..ba38a7d9b2 100644
--- a/lisp/treesit.el
+++ b/lisp/treesit.el
@@ -99,6 +99,15 @@ treesit
   :group 'tools
   :version "29.1")
 
+(defcustom treesit-enabled-modes nil
+  "List of modes to enable tree-sitter support if available.
+When initialising a major mode with potential tree-sitter
+support, this variable is consulted.  The special value t will
+enable tree-sitter support whenever possible."
+  :type '(choice (const :tag "Whenever possible" t)
+                 (repeat :tag "Specific modes" function))
+  :version "29.1")
+
 (defcustom treesit-max-buffer-size
   (let ((mb (* 1024 1024)))
     ;; 40MB for 64-bit systems, 15 for 32-bit.