def scan
@token = '*** debug ***'
@token_str = '*** debug ***'
while (ch = getchar()) != nil do
if space?(ch)
next
end
if ch == ?\#
while (ch = getchar()) != ?\n && ch != nil do; end
next
end
break
end
if ch == nil
@token = @token_str = nil
return @token
end
if alpha?(ch)
s = ch.chr
s << getchar() while word?(nextchar())
@token_str = s
case s
when 'true'
@token = :true
when 'false'
@token = :false
when 'null', 'nil'
@token = :null
when 'empty'
@token = :empty
else
@token = :name
end
return @token
end
if num?(ch)
s = ch.chr
s << getchar() while num?(nextchar())
if nextchar() == ?.
s << getchar()
s << getchar() while num?(nextchar())
end
@token_str = s
@token = :number
return @token
end
if ch == ?\:
s = ':'
if nextchar() == ?\:
getchar()
if nextchar() == ?\:
getchar()
@token = ':::'
@token_str = cutstr(?\n)
@token_str.chomp!
else
@token = '::'
@token_str = cutstr(?\n)
@token_str.chomp!
end
elsif alpha?(nextchar())
s << getchar() while alpha?(nextchar())
@token_str = s
@token = Keywords[s]
unless @token != nil
raise SyntaxError.new("'#{s}': invalid token.", self)
end
else
@token = @token_str = ':'
end
return @token
end
if ch == ?"
s = ""
while (ch = getchar()) != ?"
raise SyntaxError.new("string is not closed.", self) unless ch
if ch == ?\\
case ch = getchar()
when ?n ; ch = ?\n
when ?r ; ch = ?\r
when ?t ; ch = ?\t
when nil ; raise SyntaxError.new("string is not closed.", self)
end
end
s << ch
end
@token_str = s
@token = :string
return @token
end
if ch == ?'
s = ""
while (ch = getchar()) != ?'
raise SyntaxError.new("string is not closed.", self) unless ch
if ch == ?\\
ch = getchar()
raise SyntaxError.new("string is not closed.", self) unless ch
if ch != ?\\ && ch != ?'
s << '\\'
end
end
s << ch
end
@token_str = s
@token = :string
return @token
end
case ch
when ?(, ?), ?{, ?}, ?]
@token = @token_str = ch.chr
when ?[
if nextchar() == ?:
getchar()
@token = @token_str = '[:'
else
@token = @token_str = '['
end
when ?.
case nextchar()
when ?+, ?<, ?>, ?=, ?!
c = getchar()
if nextchar() == ?=
getchar()
@token = @token_str = ".#{c.chr}="
else
@token = @token_str = ".#{c.chr}"
end
else
@token = @token_str = '.'
end
when ?!
if nextchar() == ?=
getchar()
@token = @token_str = '!='
else
@token = @token_str = '!'
end
when ?<, ?>, ?=
case nextchar()
when ?\=
getchar()
@token = @token_str = ch.chr + '='
else
@token = @token_str = ch.chr
end
when ?+, ?-, ?*, ?/, ?%, ?^
if nextchar() == ?=
getchar()
@token = @token_str = ch.chr + '='
else
@token = @token_str = ch.chr
end
when ?\\, ??, ?\,
@token = @token_str = ch.chr
when ?&, ?|
if nextchar() == ch
getchar()
@token = @token_str = ch.chr + ch.chr
else
raise SyntaxError.new("invalid char: #{ch.chr}", self)
end
when ?@
s = ''
s << getchar() while word?(nextchar())
case s
when 'stag', 'cont', 'etag'
@token = s.intern
@token_str = s
else
@token = :expand2
@token_str = s
if s.empty?
msg = "'@': macro name required."
raise SyntaxError.new(msg)
end
end
else
raise SyntaxError.new("invalid char: #{ch.chr}", self)
end
return @token
end
def scan_all
s = ''
while scan() != nil do
if @token == :name || @token == :number
s << @token_str
elsif @token == :true || @token == :false || @token == :null || @token == :empty
s << @token_str
elsif @token == :string
s << @token_str.inspect
elsif @token == :expand2
s << '@' + @token_str
elsif @token.is_a?(Symbol)
s << @token.inspect
elsif @token == ':::'
s << @token + @token_str
elsif @token == '::'
s << @token + @token_str
else
s << @token
end
s << @newline
end
return s
end
end