bug#6286: General delimited literals in ruby-mode patch

[Stefan Monnier]

I've installed your 2 patches, plus the patch below.
Could you take a look at my patch?  Here is what it does:
- Fix up commenting conventions at random places.
- Split large regexp into more manageable chunks.
- During the split I saw that gsub/sub/split/scan were matched (for
  regexp) without regards to what precedes them, so "asub / a + bsub / b"
  was taken for a regexp.
- I found a problem in your approach to handling Cucumber code.
I don't know Ruby and don't use it (I looked up
http://web.njit.edu/all_topics/Prog_Lang_Docs/html/ruby/syntax.html for
help).
BTW, is it really true that "%Q(hello (my) world)" is correct?
That web-page doesn't clearly mention such nesting.


        Stefan


=== modified file 'lisp/ChangeLog'
--- lisp/ChangeLog      2012-04-24 16:00:08 +0000
+++ lisp/ChangeLog      2012-04-24 16:35:47 +0000
@@ -1,3 +1,10 @@
+2012-04-24  Stefan Monnier  <monnier@iro.umontreal.ca>
+
+       * progmodes/ruby-mode.el: Simplify last change, and cleanup code.
+       (ruby-syntax-propertize-regexp): Remove.
+       (ruby-syntax-propertize-function): Split regexp into chunks.
+       Match following code directly.
+
 2012-04-24  Dmitry Gutov  <dgutov@yandex.ru>
 
        * progmodes/ruby-mode.el: Handle Cucumber defs (bug#6286).

=== modified file 'lisp/progmodes/ruby-mode.el'
--- lisp/progmodes/ruby-mode.el 2012-04-24 16:00:08 +0000
+++ lisp/progmodes/ruby-mode.el 2012-04-24 16:42:22 +0000
@@ -784,7 +784,7 @@
                       (not (looking-at "[a-z_]"))))
                (and (looking-at ruby-operator-re)
                     (not (ruby-special-char-p))
-                    ;; operator at the end of line
+                    ;; Operator at the end of line.
                     (let ((c (char-after (point))))
                       (and
 ;;                     (or (null begin)
@@ -794,8 +794,9 @@
 ;;                           (not (or (eolp) (looking-at "#")
 ;;                                    (and (eq (car (nth 1 state)) ?{)
 ;;                                         (looking-at "|"))))))
-                       ;; not a regexp or general delimited literal
-                       (null (nth 0 (ruby-parse-region (or begin parse-start) 
(point))))
+                       ;; Not a regexp or general delimited literal.
+                       (null (nth 0 (ruby-parse-region (or begin parse-start)
+                                                       (point))))
                        (or (not (eq ?| (char-after (point))))
                            (save-excursion
                              (or (eolp) (forward-char -1))
@@ -1110,6 +1111,8 @@
             mlist)))))
 
 (declare-function ruby-syntax-propertize-heredoc "ruby-mode" (limit))
+(declare-function ruby-syntax-general-delimiters-goto-beg "ruby-mode" ())
+(declare-function ruby-syntax-propertize-general-delimiters "ruby-mode" 
(limit))
 
 (if (eval-when-compile (fboundp #'syntax-propertize-rules))
     ;; New code that works independently from font-lock.
@@ -1121,18 +1124,37 @@
         (ruby-syntax-general-delimiters-goto-beg)
         (funcall
          (syntax-propertize-rules
-          ;; #{ }, #$hoge, #@foo are not comments
+          ;; #{ }, #$hoge, #@foo are not comments.
           ("\\(#\\)[{$@]" (1 "."))
-          ;; $' $" $` .... are variables
-          ;; ?' ?" ?` are ascii codes
+          ;; $' $" $` .... are variables.
+          ;; ?' ?" ?` are ascii codes.
           ("\\([?$]\\)[#\"'`]"
            (1 (unless (save-excursion
                         ;; Not within a string.
                         (nth 3 (syntax-ppss (match-beginning 0))))
                 (string-to-syntax "\\"))))
-          ;; regexps
-          ("\\(^\\|[[=(,~?:;<>]\\|\\(?:^\\|\\s 
\\)\\(?:if\\|elsif\\|unless\\|while\\|until\\|when\\|and\\|or\\|&&\\|||\\)\\|g?sub!?\\|scan\\|split!?\\)?\\s
 *\\(/\\)[^/\n\\\\]*\\(?:\\\\.[^/\n\\\\]*\\)*\\(/\\)"
-           (2 (ruby-syntax-propertize-regexp)))
+          ;; Regexps: regexps are distinguished from division either because
+          ;; of the keyword/symbol before them, or because of the code
+          ;; following them.
+          ((concat
+            ;; Special tokens that can't be followed by a division operator.
+            "\\(?:\\(^\\|[[=(,~?:;<>]\\|\\(?:^\\|\\s \\)"
+            (regexp-opt '("if" "elsif" "unless" "while" "until" "when" "and"
+                          "or" "&&" "||"
+                          "gsub" "gsub!" "sub" "sub!" "scan" "split" "split!"))
+            "\\)\\s *\\)?"
+            ;; The regular expression itself.
+            "\\(/\\)[^/\n\\\\]*\\(?:\\\\.[^/\n\\\\]*\\)*\\(/\\)"
+            ;; Special code that cannot follow a division operator.
+            ;; FIXME: Just because the second slash of "/foo/ do bar" can't
+            ;; be a division, doesn't mean it can't *start* a regexp, as in
+            ;; "x = toto/foo; if /do bar/".
+            "\\([imxo]*\\s *\\(?:,\\|\\_<do\\_>\\)\\)?")
+           (2 (when (or (match-beginning 1) (match-beginning 4))
+                (string-to-syntax "\"/")))
+           (3 (if (or (match-beginning 1) (match-beginning 4))
+                  (string-to-syntax "\"/")
+                (goto-char (match-end 2)))))
           ("^=en\\(d\\)\\_>" (1 "!"))
           ("^\\(=\\)begin\\_>" (1 "!"))
           ;; Handle here documents.
@@ -1143,21 +1165,6 @@
            (1 (prog1 "|" (ruby-syntax-propertize-general-delimiters end)))))
          (point) end))
 
-      (defun ruby-syntax-propertize-regexp ()
-        (let ((syn (string-to-syntax "\"/")))
-          (goto-char (match-end 3))
-          (if (or
-               ;; after paren, comma, operator, control flow keyword,
-               ;; or a method from hardcoded list
-               (match-beginning 1)
-               ;; followed by comma or block
-               (looking-at "[imxo]*\\s *\\(?:,\\|\\<do\\>\\)"))
-              (progn
-                (put-text-property (1- (point)) (point)
-                                   'syntax-table syn)
-                syn)
-            (goto-char (match-end 2)))))
-
       (defun ruby-syntax-propertize-heredoc (limit)
         (let ((ppss (syntax-ppss))
               (res '()))
@@ -1199,7 +1206,7 @@
                parse-sexp-lookup-properties)
           (ignore-errors
             (if cl
-                (progn  ; paired delimiters
+                (progn  ; Paired delimiters.
                   ;; Delimiter pairs of the same kind can be nested
                   ;; inside the literal, as long as they are balanced.
                   ;; Create syntax table that ignores other characters.
@@ -1210,10 +1217,10 @@
                     (save-restriction
                       (narrow-to-region (point) limit)
                       (forward-list))))  ; skip to the paired character
-              ;; single character delimiter
+              ;; Single character delimiter.
               (re-search-forward (concat "[^\\]\\(?:\\\\\\\\\\)*"
                                          (regexp-quote ops)) limit nil))
-            ;; if we reached here, the closing delimiter was found
+            ;; If we reached here, the closing delimiter was found.
             (put-text-property (1- (point)) (point)
                                'syntax-table (string-to-syntax "|")))))
       )
@@ -1260,7 +1267,7 @@
      (4 (7 . ?/))
      (6 (7 . ?/)))
     ("^=en\\(d\\)\\_>" 1 "!")
-    ;; general delimited string
+    ;; General delimited string.
     ("\\(^\\|[[ \t\n<+(,=]\\)\\(%[xrqQwW]?\\([^<[{(a-zA-Z0-9 
\n]\\)[^\n\\\\]*\\(\\\\.[^\n\\\\]*\\)*\\(\\3\\)\\)"
      (3 "\"")
      (5 "\""))

=== modified file 'test/ChangeLog'
--- test/ChangeLog      2012-04-11 03:24:26 +0000
+++ test/ChangeLog      2012-04-24 17:02:20 +0000
@@ -1,3 +1,7 @@
+2012-04-24  Stefan Monnier  <monnier@iro.umontreal.ca>
+
+       * indent/ruby.rb: New file, to test new syntax-propertize code.
+
 2012-04-11  Glenn Morris  <rgm@gnu.org>
 
        * automated/vc-bzr.el (vc-bzr-test-faulty-bzr-autoloads): New test.

=== added file 'test/indent/ruby.rb'
--- test/indent/ruby.rb 1970-01-01 00:00:00 +0000
+++ test/indent/ruby.rb 2012-04-24 17:00:30 +0000
@@ -0,0 +1,19 @@
+# Don't mis-match "sub" at the end of words.
+a = asub / aslb + bsub / bslb;
+
+b = %Q{This is a "string"}
+c = %w(foo
+ bar
+ baz)
+d = %!hello!
+
+# A "do" after a slash means that slash is not a division, but it doesn't imply
+# it's a regexp-ender, since it can be a regexp-starter instead!
+x = toto / foo; if /do bar/ then
+                  toto = 1
+                end
+
+# Some Cucumber code:
+Given /toto/ do
+  print "hello"
+end