\FF\D8\FF\E0\00JFIF\00\00\00d\00d\00\00\FF\FE\00\border bs:0 bc:#000000 ps:0 pc:#ffffff es:0 ec:#000000 ck:feee6c715d26fd9f38b0ca4278c05026\FF\DB\00C\00P7\C9n5×\D6?\BDê\9Ds\EBp\9F[`8m\B7)o\B5\E8\E6I\99\FE3]]A2\BA\8Cw\D6E\93\\DEv\C8\009\F2\F1NI?uc\\F5\EA\96k\xN<~buv\EA\C8\D7 \8B\84\CEcxI\BBg\AE\9E=\D6+n\EC\80\C8A\8C\AE\EB\CF\D5\DA\E9"2\A4\B9j5\EB\F3W\B63\96\B30Yu\DA\FC8\ED\DF\E7Ms\FB\F1\8E\B3\FA\EA\E8\E6(\883zs\F2_\8DFk\8Bh \00\8C\DCw\D3R\B5+6X\BA\B2\C4j\AB0\B4\FCMw\C2I\8E\9B\E3\A9~9u\FA\D3l\80\C8%p\EE\FDn2€ \00 $\FEj\C4e\A9\DB\~\95\A7\A5\80EK\BB\8DDsP\00@@AD'k\CF\E8\DB\D2(\80\9AK\D3\85\D6lb\F2\BA\8C*\80\00)\95 59\A3R:\F3\CE"\B6\80\88\00\00i1u4ê\E9\F2á\A6\A2\FACM\93WMb*\E0*\00\00\00\00\00(\A8\80\00\00\80\00\00\00\00\00\00\00\00\00\FF\D9 C/// # frozen_string_literal: true # # $Id$ # # Copyright (c) 2004,2005 Minero Aoki # # This program is free software. # You can distribute and/or modify this program under the Ruby License. # For details of Ruby License, see ruby/COPYING. # require 'ripper/core' class Ripper # Tokenizes the Ruby program and returns an array of strings. # # p Ripper.tokenize("def m(a) nil end") # # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"] # def Ripper.tokenize(src, filename = '-', lineno = 1) Lexer.new(src, filename, lineno).tokenize end # Tokenizes the Ruby program and returns an array of an array, # which is formatted like # [[lineno, column], type, token, state]. # # require 'ripper' # require 'pp' # # pp Ripper.lex("def m(a) nil end") # #=> [[[1, 0], :on_kw, "def", FNAME ], # [[1, 3], :on_sp, " ", FNAME ], # [[1, 4], :on_ident, "m", ENDFN ], # [[1, 5], :on_lparen, "(", BEG|LABEL], # [[1, 6], :on_ident, "a", ARG ], # [[1, 7], :on_rparen, ")", ENDFN ], # [[1, 8], :on_sp, " ", BEG ], # [[1, 9], :on_kw, "nil", END ], # [[1, 12], :on_sp, " ", END ], # [[1, 13], :on_kw, "end", END ]] # def Ripper.lex(src, filename = '-', lineno = 1) Lexer.new(src, filename, lineno).lex end class Lexer < ::Ripper #:nodoc: internal use only State = Struct.new(:to_int, :to_s) do alias to_i to_int def initialize(i) super(i, Ripper.lex_state_name(i)).freeze end # def inspect; "#<#{self.class}: #{self}>" end alias inspect to_s def pretty_print(q) q.text(to_s) end def ==(i) super or to_int == i end def &(i) self.class.new(to_int & i) end def |(i) self.class.new(to_int | i) end def allbits?(i) to_int.allbits?(i) end def anybits?(i) to_int.anybits?(i) end def nobits?(i) to_int.nobits?(i) end end Elem = Struct.new(:pos, :event, :tok, :state, :message) do def initialize(pos, event, tok, state, message = nil) super(pos, event, tok, State.new(state), message) end def inspect "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>" end def pretty_print(q) q.group(2, "#<#{self.class}:", ">") { q.breakable q.text("#{event}@#{pos[0]}:#{pos[1]}") q.breakable q.text(state) q.breakable q.text("token: ") tok.pretty_print(q) if message q.breakable q.text("message: ") q.text(message) end } end def to_a a = super a.pop unless a.last a end end attr_reader :errors def tokenize parse().sort_by(&:pos).map(&:tok) end def lex parse().sort_by(&:pos).map(&:to_a) end # parse the code and returns elements including errors. def scan result = (parse() + errors + @stack.flatten).uniq.sort_by {|e| [*e.pos, (e.message ? -1 : 0)]} result.each_with_index do |e, i| if e.event == :on_parse_error and e.tok.empty? and (pre = result[i-1]) and pre.pos[0] == e.pos[0] and (pre.pos[1] + pre.tok.size) == e.pos[1] e.tok = pre.tok e.pos[1] = pre.pos[1] result[i-1] = e result[i] = pre end end result end def parse @errors = [] @buf = [] @stack = [] super @buf.flatten! @buf end private unless SCANNER_EVENT_TABLE.key?(:ignored_sp) SCANNER_EVENT_TABLE[:ignored_sp] = 1 SCANNER_EVENTS << :ignored_sp EVENTS << :ignored_sp end def on_heredoc_dedent(v, w) ignored_sp = [] heredoc = @buf.last heredoc.each_with_index do |e, i| if Elem === e and e.event == :on_tstring_content and e.pos[1].zero? tok = e.tok.dup if w > 0 and /\A\s/ =~ e.tok if (n = dedent_string(e.tok, w)) > 0 if e.tok.empty? e.tok = tok[0, n] e.event = :on_ignored_sp next end ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n], e.state)] e.pos[1] += n end end end ignored_sp.reverse_each do |i, e| heredoc[i, 0] = [e] end v end def on_heredoc_beg(tok) @stack.push @buf buf = [] @buf.push buf @buf = buf @buf.push Elem.new([lineno(), column()], __callee__, tok, state()) end def on_heredoc_end(tok) @buf.push Elem.new([lineno(), column()], __callee__, tok, state()) @buf = @stack.pop end def _push_token(tok) @buf.push Elem.new([lineno(), column()], __callee__, tok, state()) end def on_error(mesg) @errors.push Elem.new([lineno(), column()], __callee__, token(), state(), mesg) end alias on_parse_error on_error alias compile_error on_error (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event| alias_method event, :_push_token end end # [EXPERIMENTAL] # Parses +src+ and return a string which was matched to +pattern+. # +pattern+ should be described as Regexp. # # require 'ripper' # # p Ripper.slice('def m(a) nil end', 'ident') #=> "m" # p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+') #=> "m(a)" # p Ripper.slice("< "string\n" # def Ripper.slice(src, pattern, n = 0) if m = token_match(src, pattern) then m.string(n) else nil end end def Ripper.token_match(src, pattern) #:nodoc: TokenPattern.compile(pattern).match(src) end class TokenPattern #:nodoc: class Error < ::StandardError # :nodoc: end class CompileError < Error # :nodoc: end class MatchError < Error # :nodoc: end class << self alias compile new end def initialize(pattern)