bug#49822: [RFC PATCH] python-shell-send functions show no output

[Augusto Stoffel]

On Thu, 26 Aug 2021 at 16:09, Lars Ingebrigtsen <larsi@gnus.org> wrote:

> If you could fix this (i.e., not use temporary files), that would be
> very nice.

All right, I'm attaching a proposal for 'python-shell-send-string'
(affecting also all other 'python-shell-send-' functions except
'python-shell-send-buffer').

The main user-visible change is that now a result is printed in the
shell in all possible cases, i.e.:

1. If the string being evaluated is a one-line expression.
2. If the string being evaluated is a multiline expression such as
   "(1+\n1)".
3. The string being evaluated is contains multiple statements, the last
   of which is an expression.  E.g., "x=1\nx".

Currently, only case 1 prints a result (this is the actual topic of this
bug).

By avoiding to use 'python-shell-send-file', the following additional
issues are fixed:

1. The plumbing code used to execute the user's code could crop up in a
   stack trace (Bug#32042).  This is now almost completely fixed (the
   only exception I see is in the plain python3 interpreter, where a
   not-too-annoying mention to __PYTHON_EL_eval still appears.)
2. Sending temporary files over Tramp has a noticeable delay (and lots
   of echo area messages).  In particular, completion is now faster on
   remote Python shells.
3. Some global symbols (os, codecs, etc.) used to be redefined; some
   others (compile, etc.) used to be assumed to not have been redefined
   by the user.  We now only assume this is the case for 'exec' (because
   it's a keyword in Python 2).

Note, however, that 'python-shell-send-file' remains unchanged.

This is a reasonably sensitive change and it would be good to collect
feedback from others before merging it (a properly formatted patch will
then follow).  In particular, since I'm not an expert in Python's
internals, there might be some possible refinements to the
__PYTHON_EL_eval function.  Is this perhaps a better topic for the
emacs-devel list?

>From 10b474c9f82f13b3ee9c451a6aea0d3660041095 Mon Sep 17 00:00:00 2001
From: Augusto Stoffel <arstoffel@gmail.com>
Date: Sat, 28 Aug 2021 11:16:40 +0200
Subject: [PATCH] Improve python-shell-send functions

---
 lisp/progmodes/python.el | 64 +++++++++++++++++++++++++++++++---------
 1 file changed, 50 insertions(+), 14 deletions(-)

diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index d5209d8d2f..bed709bb65 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -3081,6 +3081,45 @@ python-shell--save-temp-file
       (delete-trailing-whitespace))
     temp-file-name))
 
+(defvar python-shell-eval-setup-code
+  "\
+def __PYTHON_EL_eval(source, filename):
+    import ast, sys
+    if sys.version_info[0] == 2:
+        from __builtin__ import compile, eval, globals
+    else:
+        from builtins import compile, eval, globals
+    sys.stdout.write('\\n')
+    try:
+        p, e = ast.parse(source, filename), None
+    except SyntaxError:
+        t, v, tb = sys.exc_info()
+        sys.excepthook(t, v, tb.tb_next)
+        return
+    if p.body and isinstance(p.body[-1], ast.Expr):
+        e = p.body.pop()
+    try:
+        g = globals()
+        exec(compile(p, filename, 'exec'), g, g)
+        if e:
+            return eval(compile(ast.Expression(e.value), filename, 'eval'), g, 
g)
+    except Exception:
+        t, v, tb = sys.exc_info()
+        sys.excepthook(t, v, tb.tb_next)"
+  "Code used to evaluate statements in inferior Python processes.")
+
+(defalias 'python-shell--encode-string
+  (let ((fun (if (and (fboundp 'json-serialize)
+                      (>= emacs-major-version 28))
+                 'json-serialize
+               (require 'json)
+               'json-encode-string)))
+    (lambda (text)
+      (if (stringp text)
+          (funcall fun text)
+        (signal 'wrong-type-argument (list 'stringp text)))))
+  "Encode TEXT as a valid Python string.")
+
 (defun python-shell-send-string (string &optional process msg)
   "Send STRING to inferior Python PROCESS.
 When optional argument MSG is non-nil, forces display of a
@@ -3088,16 +3127,12 @@ python-shell-send-string
 t when called interactively."
   (interactive
    (list (read-string "Python command: ") nil t))
-  (let ((process (or process (python-shell-get-process-or-error msg))))
-    (if (string-match ".\n+." string)   ;Multiline.
-        (let* ((temp-file-name (with-current-buffer (process-buffer process)
-                                 (python-shell--save-temp-file string)))
-               (file-name (or (buffer-file-name) temp-file-name)))
-          (python-shell-send-file file-name process temp-file-name t))
-      (comint-send-string process string)
-      (when (or (not (string-match "\n\\'" string))
-                (string-match "\n[ \t].*\n?\\'" string))
-        (comint-send-string process "\n")))))
+  (comint-send-string
+   (or process (python-shell-get-process-or-error msg))
+   (format "exec(%s);__PYTHON_EL_eval(%s, %s)\n"
+           (python-shell--encode-string python-shell-eval-setup-code)
+           (python-shell--encode-string string)
+           (python-shell--encode-string (or (buffer-file-name) "<string>")))))
 
 (defvar python-shell-output-filter-in-progress nil)
 (defvar python-shell-output-filter-buffer nil)
@@ -3139,7 +3174,8 @@ python-shell-send-string-no-output
         (inhibit-quit t))
     (or
      (with-local-quit
-       (python-shell-send-string string process)
+       (comint-send-string process (format "exec(%s)\n"
+                                           (python-shell--encode-string 
string)))
        (while python-shell-output-filter-in-progress
          ;; `python-shell-output-filter' takes care of setting
          ;; `python-shell-output-filter-in-progress' to NIL after it
@@ -3362,7 +3398,8 @@ python-shell-send-file
          (temp-file-name (when temp-file-name
                            (file-local-name (expand-file-name
                                              temp-file-name)))))
-    (python-shell-send-string
+    (comint-send-string
+     process
      (format
       (concat
        "import codecs, os;"
@@ -3372,8 +3409,7 @@ python-shell-send-file
        (when (and delete temp-file-name)
          (format "os.remove('''%s''');" temp-file-name))
        "exec(compile(__code, '''%s''', 'exec'));")
-      (or temp-file-name file-name) encoding encoding file-name)
-     process)))
+      (or temp-file-name file-name) encoding encoding file-name))))
 
 (defun python-shell-switch-to-shell (&optional msg)
   "Switch to inferior Python process buffer.
-- 
2.31.1