problem of `tramp-handle-file-local-copy' and inline transfer compressing

[Toru TSUNEYOSHI]

Hello, everyone.

I am one of tramp users. Tramp is very, very useful. Thanks, developers.

Well, while I use tramp, I met a problem. It is that tramp cannot deal
with file whose name extension is "gz" (maybe "bz2", "zip", and so on)
in the case of using inline transfer. I found that
`tramp-handle-file-local-copy' had the problem.
So I made the patch for it. Please check
"tramp.el.jka-compr-inhibit.diff". If it is valid, please apply it.
(tramp-version => "2.1.15" on Emacs 23.1)

And, I use `plink' usually. As you know, inline transfer of `plink' is
very, very slow. (Of course, I sometimes use `pscp' if file size is
large.)
So I made `tramp-handle-file-local-copy' and `tramp-handle-write-region'
enable inline transfer compressing. The speed is approximately 1.5 - 4
times than the original. (In some cases, approximately 10 times.)
Please check "tramp.el.jka-compr-inhibit+compress.diff".
If it is valid, please apply it.
(Sorry, please correct variable/function name if it is not proper.)

--- tramp.el.original   2009-07-08 22:22:51.000000000 +0900
+++ tramp.el    2010-04-22 11:24:19.918277500 +0900
@@ -4095,13 +4095,15 @@
                     v 5 "Decoding remote file %s with function %s..."
                     filename loc-dec)
                    (funcall loc-dec (point-min) (point-max))
-                   (let ((coding-system-for-write 'binary))
+                   (let ((coding-system-for-write 'binary)
+                         (jka-compr-inhibit t))
                      (write-region (point-min) (point-max) tmpfile)))
 
                ;; If tramp-decoding-function is not defined for this
                ;; method, we invoke tramp-decoding-command instead.
                (let ((tmpfile2 (tramp-compat-make-temp-file filename)))
-                 (let ((coding-system-for-write 'binary))
+                 (let ((coding-system-for-write 'binary)
+                       (jka-compr-inhibit t))
                    (write-region (point-min) (point-max) tmpfile2))
                  (tramp-message
                   v 5 "Decoding remote file %s with command %s..."

--- tramp.el.original   2009-07-08 22:22:51.000000000 +0900
+++ tramp.el    2010-04-22 11:38:30.907395300 +0900
@@ -263,6 +263,13 @@
   :group 'tramp
   :type 'string)
 
+(defcustom tramp-inline-transfer-compress-start-size 4096
+  "*The minimum size of compressing where inline transfer.
+When inline transfer, compress transfered data of file
+whose size is this value or above (up to `tramp-copy-size-limit')."
+  :group 'tramp
+  :type 'integer)
+
 (defcustom tramp-copy-size-limit 10240
   "*The maximum file size where inline copying is preferred over an 
out-of-the-band copy."
   :group 'tramp
@@ -4062,24 +4069,31 @@
 
     (let ((rem-enc (tramp-get-remote-coding v "remote-encoding"))
          (loc-dec (tramp-get-local-coding v "local-decoding"))
-         (tmpfile (tramp-compat-make-temp-file filename)))
+         (tmpfile (tramp-compat-make-temp-file filename))
+         (filesize (nth 7 (file-attributes filename)))
+         compress decompress)
 
       (condition-case err
          (cond
           ;; `copy-file' handles direct copy and out-of-band methods.
           ((or (tramp-local-host-p v)
                (and (tramp-method-out-of-band-p v)
-                    (> (nth 7 (file-attributes filename))
+                    (> filesize
                        tramp-copy-size-limit)))
            (copy-file filename tmpfile t t))
 
           ;; Use inline encoding for file transfer.
           (rem-enc
+           (if (<= tramp-inline-transfer-compress-start-size filesize)
+               (setq compress (tramp-get-inline-transfer-compress v 
"inline-transfer-compress")
+                     decompress (tramp-get-inline-transfer-compress v 
"inline-transfer-decompress")))
            (save-excursion
              (tramp-message v 5 "Encoding remote file %s..." filename)
              (tramp-barf-unless-okay
               v
-              (format "%s < %s" rem-enc (tramp-shell-quote-argument localname))
+              (if compress
+                  (format "%s < %s | %s" compress (tramp-shell-quote-argument 
localname) rem-enc)
+                (format "%s < %s" rem-enc (tramp-shell-quote-argument 
localname)))
               "Encoding remote file failed")
              (tramp-message v 5 "Encoding remote file %s...done" filename)
 
@@ -4095,19 +4109,46 @@
                     v 5 "Decoding remote file %s with function %s..."
                     filename loc-dec)
                    (funcall loc-dec (point-min) (point-max))
-                   (let ((coding-system-for-write 'binary))
+                   (let ((coding-system-for-write 'binary)
+                         (jka-compr-inhibit t))
+                     (when decompress
+                       (tramp-message
+                        v 5 "Decompress remote file %s with command `%s'..."
+                        filename decompress)
+                       (call-process-region (point-min) (point-max)
+                                            tramp-encoding-shell
+                                            t t nil
+                                            tramp-encoding-command-switch
+                                            decompress))
                      (write-region (point-min) (point-max) tmpfile)))
 
                ;; If tramp-decoding-function is not defined for this
                ;; method, we invoke tramp-decoding-command instead.
                (let ((tmpfile2 (tramp-compat-make-temp-file filename)))
-                 (let ((coding-system-for-write 'binary))
+                 (let ((coding-system-for-write 'binary)
+                       (jka-compr-inhibit t))
                    (write-region (point-min) (point-max) tmpfile2))
                  (tramp-message
                   v 5 "Decoding remote file %s with command %s..."
                   filename loc-dec)
                  (unwind-protect
-                     (tramp-call-local-coding-command loc-dec tmpfile2 tmpfile)
+                     (progn
+                       (tramp-call-local-coding-command loc-dec tmpfile2 
tmpfile)
+                       (if decompress
+                           (with-temp-buffer
+                             (set-buffer-multibyte nil)
+                             (insert-file-contents-literally tmpfile)
+                             (let ((coding-system-for-write 'binary)
+                                   (jka-compr-inhibit t))
+                               (tramp-message
+                                v 5 "Decompress remote file %s with command 
`%s'..."
+                                filename decompress)
+                               (call-process-region (point-min) (point-max)
+                                                    tramp-encoding-shell
+                                                    t t nil
+                                                    
tramp-encoding-command-switch
+                                                    decompress)
+                               (write-region (point-min) (point-max) 
tmpfile)))))
                    (delete-file tmpfile2))))
 
              (tramp-message v 5 "Decoding remote file %s...done" filename)
@@ -4389,7 +4430,9 @@
              ;; needed if we use an encoding function, but currently
              ;; we use it always because this makes the logic
              ;; simpler.
-             (tmpfile (tramp-compat-make-temp-file filename)))
+             (tmpfile (tramp-compat-make-temp-file filename))
+             filesize
+             compress decompress)
 
          ;; We say `no-message' here because we don't want the
          ;; visited file modtime data to be clobbered from the temp
@@ -4424,11 +4467,12 @@
          ;; specified.  However, if the method _also_ specifies an
          ;; encoding function, then that is used for encoding the
          ;; contents of the tmp file.
+         (setq filesize (nth 7 (file-attributes tmpfile)))
          (cond
           ;; `rename-file' handles direct copy and out-of-band methods.
           ((or (tramp-local-host-p v)
                (and (tramp-method-out-of-band-p v)
-                    (> (- (or end (point-max)) (or start (point-min)))
+                    (> filesize
                        tramp-copy-size-limit)))
            (condition-case err
                (rename-file tmpfile filename t)
@@ -4438,6 +4482,9 @@
 
           ;; Use inline file transfer.
           (rem-dec
+           (if (<= tramp-inline-transfer-compress-start-size filesize)
+               (setq compress (tramp-get-inline-transfer-compress v 
"inline-transfer-compress")
+                     decompress (tramp-get-inline-transfer-compress v 
"inline-transfer-decompress")))
            ;; Encode tmpfile.
            (tramp-message v 5 "Encoding region...")
            (unwind-protect
@@ -4461,6 +4508,15 @@
                        ;; a tmp file anyway.
                        (let ((default-directory
                                (tramp-compat-temporary-file-directory)))
+                         (when compress
+                           (tramp-message
+                            v 5 "Compress local file %s with command `%s'..."
+                            filename compress)
+                           (call-process-region (point-min) (point-max)
+                                                tramp-encoding-shell
+                                                t t nil
+                                                tramp-encoding-command-switch
+                                                compress))
                          (funcall loc-enc (point-min) (point-max))))
 
                    (tramp-message
@@ -4481,11 +4537,18 @@
                   v 5 "Decoding region into remote file %s..." filename)
                  (tramp-send-command
                   v
-                  (format
-                   "%s >%s <<'EOF'\n%sEOF"
-                   rem-dec
-                   (tramp-shell-quote-argument localname)
-                   (buffer-string)))
+                  (if decompress
+                      (format
+                       "(%s | %s >%s) <<'EOF'\n%sEOF"
+                       rem-dec
+                       decompress
+                       (tramp-shell-quote-argument localname)
+                       (buffer-string))
+                    (format
+                     "%s >%s <<'EOF'\n%sEOF"
+                     rem-dec
+                     (tramp-shell-quote-argument localname)
+                     (buffer-string))))
                  (tramp-barf-unless-okay
                   v nil
                   "Couldn't write region to `%s', decode using `%s' failed"
@@ -4510,11 +4573,17 @@
                       (buffer-string)
                       (with-current-buffer (tramp-get-buffer v)
                         (buffer-string))))
-                    (tramp-error
-                     v 'file-error
-                     (concat "Couldn't write region to `%s',"
-                             " decode using `%s' failed")
-                     filename rem-dec)))
+                    (if decompress
+                        (tramp-error
+                         v 'file-error
+                         (concat "Couldn't write region to `%s',"
+                                 " decode using `%s' and `%s' failed")
+                         filename rem-dec decompress)
+                      (tramp-error
+                       v 'file-error
+                       (concat "Couldn't write region to `%s',"
+                               " decode using `%s' failed")
+                       filename rem-dec))))
                  (tramp-message
                   v 5 "Decoding region into remote file %s...done" filename)
                  (tramp-flush-file-property v localname))
@@ -6312,6 +6381,62 @@
       (tramp-message vec 5 "Using remote decoding `%s'" rem-dec)
       (tramp-set-connection-property vec "remote-decoding" rem-dec))))
 
+(defconst tramp-inline-transfer-compress-commands
+  '(("gzip" "gzip -d")
+    ("bzip2" "bzip2 -d")
+    ("compress" "compress -d"))
+ "List of compress commands for inline transfer.
+Each item is a list that looks like this:
+
+\(ENCODING DECODING).
+
+ENCODING or DECODING is a list that looks like this:
+
+\(\"executable program\")
+or
+\(\"executable program\" \"option\" ...) ")
+
+(defun tramp-find-inline-transfer-compress (vec)
+  "Find an inline transfer compress command that works.
+Goes through the list `tramp-inline-transfer-compress-commands'."
+  (save-excursion
+    (let ((commands tramp-inline-transfer-compress-commands)
+         (magic "xyzzy")
+         item compress decompress
+         found)
+      (while (and commands (not found))
+       (catch 'next
+         (setq item (pop commands)
+               compress (nth 0 item)
+               decompress (nth 1 item))
+         (tramp-message
+          vec 5 "Checking local inline transfer compress command `%s', `%s' 
for sanity" compress decompress)
+         (unless (zerop (tramp-call-local-coding-command
+                         (format "echo %s | %s | %s"
+                                 magic compress decompress) nil nil))
+           (throw 'next nil))
+         (tramp-message
+          vec 5 "Checking remote inline transfer compress command `%s', `%s' 
for sanity" compress decompress)
+         (unless (zerop (tramp-send-command-and-check
+                         vec
+                         (format "echo %s | %s | %s"
+                                 magic compress decompress) t))
+           (throw 'next nil))
+         (setq found t)))
+
+      ;; Did we find something?
+      (if found
+         (progn
+           ;; Set connection properties.
+           (tramp-message vec 5 "Using inline transfer compress command `%s'" 
compress)
+           (tramp-set-connection-property vec "inline-transfer-compress" 
compress)
+           (tramp-message vec 5 "Using inline transfer decompress command 
`%s'" decompress)
+           (tramp-set-connection-property vec "inline-transfer-decompress" 
decompress))
+       ;; Set connection properties as checked (t).
+       (tramp-set-connection-property vec "inline-transfer-compress" t)
+       (tramp-set-connection-property vec "inline-transfer-decompress" t)
+       (tramp-message vec 2 "Couldn't find an inline transfer compress 
command")))))
+
 (defun tramp-call-local-coding-command (cmd input output)
   "Call the local encoding or decoding command.
 If CMD contains \"%s\", provide input file INPUT there in command.
@@ -7388,6 +7513,19 @@
      (tramp-find-inline-encoding vec)
      (tramp-get-connection-property vec prop nil))))
 
+(defun tramp-get-inline-transfer-compress (vec prop)
+  (let ((val (tramp-get-connection-property vec prop nil)))
+    (if (eq val t)     ; already checked, and compress command not found
+       nil
+      (or
+       val
+       (progn
+        (tramp-find-inline-transfer-compress vec)
+        (setq val (tramp-get-inline-transfer-compress vec prop))
+        (if (eq val t)                 ; compress command not found
+            nil
+          val))))))
+
 (defun tramp-get-method-parameter (method param)
   "Return the method parameter PARAM.
 If the `tramp-methods' entry does not exist, return NIL."