Lex string arrays in bootstrap

bootstrap/crystal/lexer.cr

@@ -148,6 +148,8 @@ module Crystal
           else
             next_char :"-@"
           end
+        when '>'
+          next_char :"->"
         when '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'
           scan_number(@buffer - 1, 2)
         else
@@ -186,6 +188,13 @@ module Crystal
         case next_char
         when '='
           next_char :"%="
+        when 'w'
+          if @buffer[1] == '('
+            next_char
+            next_char :STRING_ARRAY_START
+          else
+            @token.type = :"%"
+          end
         else
           @token.type = :"%"
         end
@@ -565,6 +574,16 @@ module Crystal
         case next_char
         when '_'
           case next_char
+          when 'D'
+            if next_char == 'I' && next_char == 'R' next_char == '_' && next_char == '_'
+              if @buffer[1].ident_part_or_end?
+                scan_ident(start, start_column)
+              else
+                @token.type = :STRING
+                @token.value = @filename ? File.dirname(@filename) : "-"
+                return @token
+              end
+            end
           when 'F'
             if next_char == 'I' && next_char == 'L' && next_char == 'E' && next_char == '_' && next_char == '_'
               if @buffer[1].ident_part_or_end?
@@ -749,6 +768,38 @@ module Crystal
       end
     end
 
+    def next_string_array_token
+      while true
+        if @buffer.value == '\n'
+          next_char
+          @column_number = 1
+          @line_number += 1
+        elsif @buffer.value.whitespace?
+          next_char
+        else
+          break
+        end
+      end
+
+      if @buffer.value == ')'
+        next_char
+        @token.type = :STRING_ARRAY_END
+        return @token
+      end
+
+      start = @buffer
+      count = 0
+      while !@buffer.value.whitespace? && @buffer.value != '0' && @buffer.value != ')'
+        next_char
+        count += 1
+      end
+
+      @token.type = :STRING
+      @token.value = String.from_cstr(start, count)
+
+      @token
+    end
+
     def next_char_no_column_increment
       @buffer += 1
       @buffer.value

bootstrap/spec/crystal/lexer/lexer_spec.cr

@@ -171,7 +171,7 @@ describe "Lexer" do
   it_lexes_char "'\\0'", '0'
   it_lexes_char "'\\''", '\''
   it_lexes_char "'\\\\'", '\\'
-  it_lexes_operators [:"=", :"<", :"<=", :">", :">=", :"+", :"-", :"*", :"/", :"(", :")", :"==", :"!=", :"=~", :"!", :",", :".", :"..", :"...", :"!@", :"+@", :"-@", :"&&", :"||", :"|", :"{", :"}", :"?", :":", :"+=", :"-=", :"*=", :"/=", :"%=", :"&=", :"|=", :"^=", :"**=", :"<<", :">>", :"%", :"&", :"|", :"^", :"**", :"<<=", :">>=", :"~", :"~@", :"[]", :"[]=", :"[", :"]", :"::", :"<=>", :"=>", :"||=", :"&&=", :"===", :";"]
+  it_lexes_operators [:"=", :"<", :"<=", :">", :">=", :"+", :"-", :"*", :"/", :"(", :")", :"==", :"!=", :"=~", :"!", :",", :".", :"..", :"...", :"!@", :"+@", :"-@", :"&&", :"||", :"|", :"{", :"}", :"?", :":", :"+=", :"-=", :"*=", :"/=", :"%=", :"&=", :"|=", :"^=", :"**=", :"<<", :">>", :"%", :"&", :"|", :"^", :"**", :"<<=", :">>=", :"~", :"~@", :"[]", :"[]=", :"[", :"]", :"::", :"<=>", :"=>", :"||=", :"&&=", :"===", :";", :"->"]
   it_lexes "!@foo", :"!"
   it_lexes "+@foo", :"+"
   it_lexes "-@foo", :"-"
@@ -238,6 +238,14 @@ describe "Lexer" do
     token.value.should eq("foo")
   end
 
+  it "lexes __DIR__" do
+    lexer = Crystal::Lexer.new "__DIR__"
+    lexer.filename = "/Users/foo/bar.cr"
+    token = lexer.next_token
+    token.type.should eq(:STRING)
+    token.value.should eq("/Users/foo")
+  end
+
   it "lexes dot and ident" do
     lexer = Crystal::Lexer.new ".read"
     token = lexer.next_token

spec/lexer/lexer_spec.rb

@@ -197,7 +197,7 @@ describe Lexer do
 
   it "lexes __DIR__" do
     lexer = Lexer.new "__DIR__"
-    lexer.filename = '/Users/foo/bar'
+    lexer.filename = '/Users/foo/bar.cr'
     token = lexer.next_token
     token.type.should eq(:STRING)
     token.value.should eq('/Users/foo')