# File lib/wirble.rb, line 131
131:       def self.tokenize(str)
132:         raise 'missing block' unless block_given?
133:         chars = str.split(//)
134: 
135:         # $stderr.puts "DEBUG: chars = #{chars.join(',')}"
136: 
137:         state, val, i, lc = [], '', 0, nil
138:         while i <= chars.size
139:           repeat = false
140:           c = chars[i]
141: 
142:           # $stderr.puts "DEBUG: state = #{state}"
143: 
144:           case state[-1]
145:           when nil
146:             case c
147:             when ':'
148:               state << :symbol
149:             when '"'
150:               state << :string
151:             when '#'
152:               state << :object
153:             when /[a-z]/i
154:               state << :keyword
155:               repeat = true
156:             when /[0-9-]/
157:               state << :number
158:               repeat = true
159:             when '{'
160:               yield :open_hash, '{'
161:             when '['
162:               yield :open_array, '['
163:             when ']'
164:               yield :close_array, ']'
165:             when '}'
166:               yield :close_hash, '}'
167:             when /\s/
168:               yield :whitespace, c
169:             when ','
170:               yield :comma, ','
171:             when '>'
172:               yield :refers, '=>' if lc == '='
173:             when '.'
174:               yield :range, '..' if lc == '.'
175:             when '='
176:               # ignore these, they're used elsewhere
177:               nil
178:             else 
179:               # $stderr.puts "DEBUG: ignoring char #{c}"
180:             end
181:           when :symbol
182:             case c
183:             # XXX: should have =, but that messes up foo=>bar
184:             when /[a-z0-9_!?]/
185:               val << c
186:             else
187:               yield :symbol_prefix, ':'
188:               yield state[-1], val
189:               state.pop; val = ''
190:               repeat = true
191:             end
192:           when :string
193:             case c
194:             when '"'
195:               if lc == "\\"
196:                 val[-1] = ?"
197:               else
198:                 yield :open_string, '"'
199:                 yield state[-1], val
200:                 state.pop; val = ''
201:                 yield :close_string, '"'
202:               end
203:             else
204:               val << c
205:             end
206:           when :keyword
207:             case c
208:             when /[a-z0-9_]/i
209:               val << c
210:             else
211:               # is this a class?
212:               st = val =~ /^[A-Z]/ ? :class : state[-1]
213: 
214:               yield st, val
215:               state.pop; val = ''
216:               repeat = true
217:             end
218:           when :number
219:             case c
220:             when /[0-9e-]/
221:               val << c
222:             when '.'
223:               if lc == '.'
224:                 val[/\.$/] = ''
225:                 yield state[-1], val
226:                 state.pop; val = ''
227:                 yield :range, '..'
228:               else
229:                 val << c
230:               end
231:             else
232:               yield state[-1], val
233:               state.pop; val = ''
234:               repeat = true
235:             end
236:           when :object
237:             case c
238:             when '<' 
239:               yield :open_object, '#<'
240:               state << :object_class
241:             when ':' 
242:               state << :object_addr
243:             when '@' 
244:               state << :object_line
245:             when '>'
246:               yield :close_object, '>'
247:               state.pop; val = ''
248:             end
249:           when :object_class
250:             case c
251:             when ':'
252:               yield state[-1], val
253:               state.pop; val = ''
254:               repeat = true
255:             else
256:               val << c
257:             end
258:           when :object_addr
259:             case c
260:             when '>'
261:             when '@'
262:               yield :object_addr_prefix, ':'
263:               yield state[-1], val
264:               state.pop; val = ''
265:               repeat = true
266:             else
267:               val << c
268:             end
269:           when :object_line
270:             case c
271:             when '>'
272:               yield :object_line_prefix, '@'
273:               yield state[-1], val
274:               state.pop; val = ''
275:               repeat = true
276:             else
277:               val << c
278:             end
279:           else
280:             raise "unknown state #{state}"
281:           end
282: 
283:           unless repeat
284:             i += 1
285:             lc = c
286:           end
287:         end
288:       end