Re: org-persist cache for remote files

[Ihor Radchenko]

Lele Gaifax <lele@metapensiero.it> writes:

>> Maybe we should better make this a user option?
>
> Yes, that's what I had in mind: when (say) org-persist-do-not-auto-gc-remotes
> is enabled, the cleanup procedure would ignore remote Org documents and a new
> explicit (interactive) org-persist-gc-remotes would take care of them.
>
>> Or we can go a different route and just not garbage collect remote files
>> until cache gets too large.
>
> Yes, a threshold would work too.

Can you test the attached 2 patches? They do not seem to break local
files, but I do not have remote file caches to test on.

Best,
Ihor

>From 592608d3c48cb1e63f9b4427576f0698ca190e9a Mon Sep 17 00:00:00 2001
Message-Id: 
<592608d3c48cb1e63f9b4427576f0698ca190e9a.1639837163.git.yantar92@gmail.com>
From: Ihor Radchenko <yantar92@gmail.com>
Date: Sat, 18 Dec 2021 22:13:24 +0800
Subject: [PATCH] org-persist.el: Make org-persist-directory a custom variable

* lisp/org-persist.el (org-persist): Create org-persist customisation
group.
(org-persist-directory): Convert into defcustom.
---
 lisp/org-persist.el | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/lisp/org-persist.el b/lisp/org-persist.el
index f06d4fc61..90f9a3bc9 100644
--- a/lisp/org-persist.el
+++ b/lisp/org-persist.el
@@ -35,19 +35,27 @@ (declare-function org-back-to-heading "org" (&optional 
invisible-ok))
 (declare-function org-next-visible-heading "org" (arg))
 (declare-function org-at-heading-p "org" (&optional invisible-not-ok))
 
-(defvar org-persist-directory (expand-file-name
-                    (org-file-name-concat
-                     (let ((cache-dir (when (fboundp 'xdg-cache-home)
-                                        (xdg-cache-home))))
-                       (if (or (seq-empty-p cache-dir)
-                               (not (file-exists-p cache-dir))
-                               (file-exists-p (org-file-name-concat
-                                               user-emacs-directory
-                                               "org-persist")))
-                           user-emacs-directory
-                         cache-dir))
-                     "org-persist/"))
-  "Directory where the data is stored.")
+
+(defgroup org-persist nil
+  "Persistent cache for Org mode."
+  :tag "Org persist"
+  :group 'org)
+
+(defcustom org-persist-directory (expand-file-name
+                       (org-file-name-concat
+                        (let ((cache-dir (when (fboundp 'xdg-cache-home)
+                                           (xdg-cache-home))))
+                          (if (or (seq-empty-p cache-dir)
+                                  (not (file-exists-p cache-dir))
+                                  (file-exists-p (org-file-name-concat
+                                                  user-emacs-directory
+                                                  "org-persist")))
+                              user-emacs-directory
+                            cache-dir))
+                        "org-persist/"))
+  "Directory where the data is stored."
+  :group 'org-persist
+  :type 'directory)
 
 (defvar org-persist-index-file "index"
   "File name used to store the data index.")
-- 
2.32.0

>From de89eca758ca1972d75c8830aee224376a1e364e Mon Sep 17 00:00:00 2001
Message-Id: 
<de89eca758ca1972d75c8830aee224376a1e364e.1639837255.git.yantar92@gmail.com>
From: Ihor Radchenko <yantar92@gmail.com>
Date: Sat, 18 Dec 2021 22:14:24 +0800
Subject: [PATCH] org-persist.el: Customise persistence of data for remote
 files

* lisp/org-persist.el (org-persist-remote-files): New defcustom
defining strategy for persisting data associated with remote files.
Separate strategy is necessary to avoid unexpected password promtps
while garbage collecting the cache: remote files may require server
login.
(org-persist-gc): Decide whether to store remote file data according
to `org-persist-remote-files'.

Reported in https://list.orgmode.org/87bl1eilt6.fsf@localhost/T/#u
---
 lisp/org-persist.el | 41 ++++++++++++++++++++++++++++++++++-------
 1 file changed, 34 insertions(+), 7 deletions(-)

diff --git a/lisp/org-persist.el b/lisp/org-persist.el
index 90f9a3bc9..613290afc 100644
--- a/lisp/org-persist.el
+++ b/lisp/org-persist.el
@@ -57,6 +57,22 @@ (defcustom org-persist-directory (expand-file-name
   :group 'org-persist
   :type 'directory)
 
+(defcustom org-persist-remote-files 100
+  "Whether to keep persistent data for remote files.
+
+When this variable is nil, never save persitent data associated with
+remote files.  When `t', always keep the data.  When
+`check-existance', contact remote server containing the file and only
+keep the data when the file exists on the server.
+
+Note that the last option `check-existance' may cause Emacs to show
+password prompts to log in."
+  :group 'org-persist
+  :type '(choice (const :tag "Never" nil)
+                 (const :tag "Always" t)
+                 (number :tag "Keep note more than X files")
+                 (const :tag "Check if exist on remote" 'check-existence)))
+
 (defvar org-persist-index-file "index"
   "File name used to store the data index.")
 
@@ -287,15 +303,26 @@ (defun org-persist-read-all-buffer ()
 
 (defun org-persist-gc ()
   "Remove stored data for not existing files or unregistered variables."
-  (let (new-index)
+  (let (new-index (remote-files-num 0))
     (dolist (index org-persist--index)
-      (let ((file (plist-get index :path))
-            (persist-file (when (plist-get index :persist-file)
-                            (org-file-name-concat
-                             org-persist-directory
-                             (plist-get index :persist-file)))))
+      (let* ((file (plist-get index :path))
+             (file-remote (when file (file-remote-p file)))
+             (persist-file (when (plist-get index :persist-file)
+                             (org-file-name-concat
+                              org-persist-directory
+                              (plist-get index :persist-file)))))
         (when (and file persist-file)
-          (if (file-exists-p file)
+          (when (and file-remote persist-file)
+            (cl-incf remote-files-num))
+          (if (if (not file-remote)
+                  (file-exists-p file)
+                (pcase org-persist-remote-files
+                  ('t t)
+                  ('check-existence
+                   (file-exists-p file))
+                  ((pred #'numberp)
+                   (<= org-persist-remote-files remote-files-num))
+                  (_ nil)))
               (push index new-index)
             (when (file-exists-p persist-file)
               (delete-file persist-file)
-- 
2.32.0