[elpa] externals/vlf e8bb4a9 248/310: Apply batch size tuning on adjacent moves, search, save adjusting and

[Stefan Monnier]

branch: externals/vlf
commit e8bb4a91daf7db8c7ae636650ac5eb1c55c1f4e2
Author: Andrey Kotlarski <m00naticus@gmail.com>
Commit: Andrey Kotlarski <m00naticus@gmail.com>

    Apply batch size tuning on adjacent moves, search, save adjusting and
    report total times.
---
 vlf-search.el | 76 +++++++++++++++++++++++++++++++++++------------------------
 vlf-write.el  | 20 ++++++++++------
 vlf.el        | 12 ++++++++++
 3 files changed, 70 insertions(+), 38 deletions(-)

diff --git a/vlf-search.el b/vlf-search.el
index 0fe88fc..443276c 100644
--- a/vlf-search.el
+++ b/vlf-search.el
@@ -29,12 +29,21 @@
 
 (require 'vlf)
 
-(defun vlf-re-search (regexp count backward batch-step)
+(defun vlf-re-search (regexp count backward batch-step
+                             &optional reporter time)
   "Search for REGEXP COUNT number of times forward or BACKWARD.
-BATCH-STEP is amount of overlap between successive chunks."
+BATCH-STEP is amount of overlap between successive chunks.
+Use existing REPORTER and start TIME if given."
   (if (<= count 0)
       (error "Count must be positive"))
   (run-hook-with-args 'vlf-before-batch-functions 'search)
+  (or reporter (setq reporter (make-progress-reporter
+                               (concat "Searching for " regexp "...")
+                               (if backward
+                                   (- vlf-file-size vlf-end-pos)
+                                 vlf-start-pos)
+                               vlf-file-size)))
+  (or time (setq time (float-time)))
   (let* ((tramp-verbose (if (boundp 'tramp-verbose)
                             (min tramp-verbose 2)))
          (case-fold-search t)
@@ -44,13 +53,9 @@ BATCH-STEP is amount of overlap between successive chunks."
          (match-end-pos match-start-pos)
          (to-find count)
          (is-hexl (derived-mode-p 'hexl-mode))
-         (font-lock font-lock-mode)
-         (reporter (make-progress-reporter
-                    (concat "Searching for " regexp "...")
-                    (if backward
-                        (- vlf-file-size vlf-end-pos)
-                      vlf-start-pos)
-                    vlf-file-size)))
+         (tune-types (if is-hexl '(:hexl :dehexlify :insert :encode)
+                       '(:insert :encode)))
+         (font-lock font-lock-mode))
     (font-lock-mode 0)
     (vlf-with-undo-disabled
      (unwind-protect
@@ -69,7 +74,8 @@ BATCH-STEP is amount of overlap between successive chunks."
                                                 (match-end 0)))))
                        ((zerop vlf-start-pos)
                         (throw 'end-of-file nil))
-                       (t (let ((batch-move (- vlf-start-pos
+                       (t (vlf-tune-optimal tune-types)
+                          (let ((batch-move (- vlf-start-pos
                                                (- vlf-batch-size
                                                   batch-step))))
                             (vlf-move-to-batch
@@ -82,9 +88,9 @@ BATCH-STEP is amount of overlap between successive chunks."
                                                  match-start-pos))
                                          (point-max)
                                        (or (byte-to-position
-                                              (- match-start-pos
-                                                 vlf-start-pos))
-                                             (point-max))))
+                                            (- match-start-pos
+                                               vlf-start-pos))
+                                           (point-max))))
                           (progress-reporter-update
                            reporter (- vlf-file-size
                                        vlf-start-pos)))))
@@ -101,7 +107,8 @@ BATCH-STEP is amount of overlap between successive chunks."
                                               (match-end 0)))))
                      ((= vlf-end-pos vlf-file-size)
                       (throw 'end-of-file nil))
-                     (t (let ((batch-move (- vlf-end-pos batch-step)))
+                     (t (vlf-tune-optimal tune-types)
+                        (let ((batch-move (- vlf-end-pos batch-step)))
                           (vlf-move-to-batch
                            (if (or is-hexl
                                    (< match-end-pos batch-move))
@@ -111,9 +118,9 @@ BATCH-STEP is amount of overlap between successive chunks."
                                            (<= match-end-pos vlf-start-pos))
                                        (point-min)
                                      (or (byte-to-position
-                                            (- match-end-pos
-                                               vlf-start-pos))
-                                           (point-min))))
+                                          (- match-end-pos
+                                             vlf-start-pos))
+                                         (point-min))))
                         (progress-reporter-update reporter
                                                   vlf-end-pos)))))
            (progress-reporter-done reporter))
@@ -122,26 +129,26 @@ BATCH-STEP is amount of overlap between successive 
chunks."
        (if backward
            (vlf-goto-match match-chunk-start match-chunk-end
                            match-end-pos match-start-pos
-                           count to-find)
+                           count to-find time)
          (vlf-goto-match match-chunk-start match-chunk-end
                          match-start-pos match-end-pos
-                         count to-find))
+                         count to-find time))
        (run-hook-with-args 'vlf-after-batch-functions 'search)))))
 
 (defun vlf-goto-match (match-chunk-start match-chunk-end
-                                         match-pos-start
-                                         match-pos-end
-                                         count to-find)
+                                         match-pos-start match-pos-end
+                                         count to-find time)
   "Move to MATCH-CHUNK-START MATCH-CHUNK-END surrounding\
 MATCH-POS-START and MATCH-POS-END.
 According to COUNT and left TO-FIND, show if search has been
-successful.  Return nil if nothing found."
+successful.  Use start TIME to report how much it took.
+Return nil if nothing found."
   (if (= count to-find)
       (progn (vlf-move-to-chunk match-chunk-start match-chunk-end)
              (goto-char (or (byte-to-position (- match-pos-start
                                                  vlf-start-pos))
                             (point-max)))
-             (message "Not found")
+             (message "Not found (%f secs)" (- (float-time) time))
              nil)
     (let ((success (zerop to-find)))
       (if success
@@ -155,10 +162,11 @@ successful.  Return nil if nothing found."
                                         vlf-start-pos))
                                     match-end)))
         (overlay-put overlay 'face 'match)
-        (unless success
+        (if success
+            (message "Match found (%f secs)" (- (float-time) time))
           (goto-char match-end)
-          (message "Moved to the %d match which is last"
-                   (- count to-find)))
+          (message "Moved to the %d match which is last (%f secs)"
+                   (- count to-find) (- (float-time) time)))
         (unwind-protect (sit-for 3)
           (delete-overlay overlay))
         t))))
@@ -171,7 +179,7 @@ Search is performed chunk by chunk in `vlf-batch-size' 
memory."
                                       (if regexp-history
                                           (car regexp-history)))
                          (or current-prefix-arg 1))))
-  (vlf-re-search regexp count nil (/ vlf-batch-size 8)))
+  (vlf-re-search regexp count nil (min 1024 (/ vlf-batch-size 8))))
 
 (defun vlf-re-search-backward (regexp count)
   "Search backward for REGEXP prefix COUNT number of times.
@@ -181,7 +189,7 @@ Search is performed chunk by chunk in `vlf-batch-size' 
memory."
                                       (if regexp-history
                                           (car regexp-history)))
                          (or current-prefix-arg 1))))
-  (vlf-re-search regexp count t (/ vlf-batch-size 8)))
+  (vlf-re-search regexp count t (min 1024 (/ vlf-batch-size 8))))
 
 (defun vlf-goto-line (n)
   "Go to line N.  If N is negative, count from the end of file."
@@ -196,8 +204,10 @@ Search is performed chunk by chunk in `vlf-batch-size' 
memory."
         (pos (point))
         (is-hexl (derived-mode-p 'hexl-mode))
         (font-lock font-lock-mode)
+        (time (float-time))
         (success nil))
     (font-lock-mode 0)
+    (vlf-tune-optimal '(:raw))
     (unwind-protect
         (if (< 0 n)
             (let ((start 0)
@@ -218,6 +228,7 @@ Search is performed chunk by chunk in `vlf-batch-size' 
memory."
                      (while (re-search-forward "[\n\C-m]" nil t)
                        (setq n (1- n)))
                      (vlf-verify-size)
+                     (vlf-tune-optimal '(:raw))
                      (setq start end
                            end (min vlf-file-size
                                     (+ start vlf-batch-size)))
@@ -225,7 +236,8 @@ Search is performed chunk by chunk in `vlf-batch-size' 
memory."
                (when (< n (- vlf-file-size end))
                  (vlf-move-to-chunk-2 start end)
                  (goto-char (point-min))
-                 (setq success (vlf-re-search "[\n\C-m]" n nil 0)))))
+                 (setq success (vlf-re-search "[\n\C-m]" n nil 0
+                                              reporter time)))))
           (let ((start (max 0 (- vlf-file-size vlf-batch-size)))
                 (end vlf-file-size)
                 (reporter (make-progress-reporter
@@ -242,6 +254,7 @@ Search is performed chunk by chunk in `vlf-batch-size' 
memory."
                    (goto-char (point-max))
                    (while (re-search-backward "[\n\C-m]" nil t)
                      (setq n (1- n)))
+                   (vlf-tune-optimal '(:raw))
                    (setq end start
                          start (max 0 (- end vlf-batch-size)))
                    (progress-reporter-update reporter
@@ -249,7 +262,8 @@ Search is performed chunk by chunk in `vlf-batch-size' 
memory."
              (when (< n end)
                (vlf-move-to-chunk-2 start end)
                (goto-char (point-max))
-               (setq success (vlf-re-search "[\n\C-m]" n t 0))))))
+               (setq success (vlf-re-search "[\n\C-m]" n t 0
+                                            reporter time))))))
       (if font-lock (font-lock-mode 1))
       (unless success
         (vlf-with-undo-disabled
diff --git a/vlf-write.el b/vlf-write.el
index 969203e..5ef1d56 100644
--- a/vlf-write.el
+++ b/vlf-write.el
@@ -64,9 +64,11 @@ If changing size of chunk, shift remaining file content."
                   (pos (point))
                   (font-lock font-lock-mode))
               (font-lock-mode 0)
-              (if (< 0 size-change)
-                  (vlf-file-shift-back size-change region-length)
-                (vlf-file-shift-forward (- size-change) region-length))
+              (let ((time (float-time)))
+                (if (< 0 size-change)
+                    (vlf-file-shift-back size-change region-length)
+                  (vlf-file-shift-forward (- size-change) region-length))
+                (message "Save took %f seconds" (- (float-time) time)))
               (if font-lock (font-lock-mode 1))
               (vlf-move-to-chunk-2 vlf-start-pos
                                    (if (< (- vlf-end-pos vlf-start-pos)
@@ -76,7 +78,7 @@ If changing size of chunk, shift remaining file content."
               (vlf-update-buffer-name)
               (goto-char pos)))))
       (if hexl (vlf-tune-hexlify)))
-    (run-hook-with-args 'vlf-after-batch-functions 'write))
+     (run-hook-with-args 'vlf-after-batch-functions 'write))
   t)
 
 (defun vlf-file-shift-back (size-change write-size)
@@ -106,6 +108,7 @@ WRITE-SIZE is byte length of saved chunk."
 back at WRITE-POS.  Return nil if EOF is reached, t otherwise."
   (erase-buffer)
   (vlf-verify-size t)
+  (vlf-tune-optimal '(:raw :write))
   (let ((read-end (min (+ read-pos vlf-batch-size) vlf-file-size)))
     (vlf-tune-insert-file-contents-literally read-pos read-end)
     (vlf-tune-write nil nil write-pos 0 (- read-end read-pos))
@@ -115,7 +118,8 @@ back at WRITE-POS.  Return nil if EOF is reached, t 
otherwise."
   "Shift file contents SIZE-CHANGE bytes forward.
 WRITE-SIZE is byte length of saved chunk.
 Done by saving content up front and then writing previous batch."
-  (let ((read-size (max (/ vlf-batch-size 2) size-change))
+  (vlf-tune-optimal '(:raw :write))
+  (let ((read-size (max vlf-batch-size size-change))
         (read-pos vlf-end-pos)
         (write-pos vlf-start-pos)
         (reporter (make-progress-reporter "Adjusting file content..."
@@ -124,18 +128,20 @@ Done by saving content up front and then writing previous 
batch."
     (vlf-with-undo-disabled
      (when (vlf-shift-batches read-size read-pos write-pos
                               write-size t)
+       (vlf-tune-optimal '(:raw :write))
        (setq write-pos (+ read-pos size-change)
              read-pos (+ read-pos read-size)
              write-size read-size
-             read-size (max (/ vlf-batch-size 2) size-change))
+             read-size (max vlf-batch-size size-change))
        (progress-reporter-update reporter write-pos)
        (let ((coding-system-for-write 'no-conversion))
          (while (vlf-shift-batches read-size read-pos write-pos
                                    write-size nil)
+           (vlf-tune-optimal '(:raw :write))
            (setq write-pos (+ read-pos size-change)
                  read-pos (+ read-pos read-size)
                  write-size read-size
-                 read-size (max (/ vlf-batch-size 2) size-change))
+                 read-size (max vlf-batch-size size-change))
            (progress-reporter-update reporter write-pos)))))
     (progress-reporter-done reporter)))
 
diff --git a/vlf.el b/vlf.el
index 8acd54e..0aae3ab 100644
--- a/vlf.el
+++ b/vlf.el
@@ -172,6 +172,9 @@ When prefix argument is negative
  append next APPEND number of batches to the existing buffer."
   (interactive "p")
   (vlf-verify-size)
+  (if (derived-mode-p 'hexl-mode)
+      (vlf-tune-conservative '(:hexl :dehexlify :insert :encode))
+    (vlf-tune-conservative '(:insert :encode)))
   (let* ((end (min (+ vlf-end-pos (* vlf-batch-size (abs append)))
                    vlf-file-size))
          (start (if (< append 0)
@@ -188,6 +191,9 @@ When prefix argument is negative
   (interactive "p")
   (if (zerop vlf-start-pos)
       (error "Already at BOF"))
+  (if (derived-mode-p 'hexl-mode)
+      (vlf-tune-conservative '(:hexl :dehexlify :insert :encode))
+    (vlf-tune-conservative '(:insert :encode)))
   (let* ((start (max 0 (- vlf-start-pos (* vlf-batch-size (abs prepend)))))
          (end (if (< prepend 0)
                   vlf-end-pos
@@ -260,12 +266,18 @@ with the prefix argument DECREASE it is halved."
 (defun vlf-beginning-of-file ()
   "Jump to beginning of file content."
   (interactive)
+  (if (derived-mode-p 'hexl-mode)
+      (vlf-tune-conservative '(:hexl :dehexlify :insert :encode))
+    (vlf-tune-conservative '(:insert :encode)))
   (vlf-move-to-batch 0))
 
 (defun vlf-end-of-file ()
   "Jump to end of file content."
   (interactive)
   (vlf-verify-size)
+  (if (derived-mode-p 'hexl-mode)
+      (vlf-tune-conservative '(:hexl :dehexlify :insert :encode))
+    (vlf-tune-conservative '(:insert :encode)))
   (vlf-move-to-batch vlf-file-size))
 
 (defun vlf-revert (&optional _auto noconfirm)