From: noah morrison <noah@morrison.ph>
Date: Fri, 31 Oct 2014 20:36:57 +0000 (-0400)
Subject: Changed tokenize to use file like objects
X-Git-Url: https://git.devinivas.org/?a=commitdiff_plain;h=3eeda9ebf75653bf23d94a7efb58af48b6681f4d;p=chevron.git

Changed tokenize to use file like objects

tokenize now takes a file object (or file like object).
If tokenize is given a string it turns it into a file like object.

Also renamed look to peek.
---

diff --git a/chevron.py b/chevron.py
index d248235..0edf757 100755
--- a/chevron.py
+++ b/chevron.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
 from sys import argv
+from io import StringIO
 
 
 def tokenize(template):
@@ -8,14 +9,16 @@ def tokenize(template):
         pass
 
     def get(amount=1):
-        return template[current:current + amount]
-
-    def look(ahead=0, amount=1):
-        idx = current + ahead
-        if idx > len(template):
+        return template.read(amount)
+
+    def peek(ahead=0, amount=1):
+        current = template.tell()
+        template.seek(current + ahead)
+        data = template.read(amount)
+        template.seek(current)
+        if len(data) != amount:
             raise EOFError()
-
-        return template[idx:idx + amount]
+        return data
 
     tag_types = {
         '!': 'comment',
@@ -28,14 +31,16 @@ def tokenize(template):
         '&': 'no escape'
     }
 
-    current = 0
+    if type(template) is str:
+        template = StringIO(template)
+
     open_sections = []
     l_del = '{{'
     r_del = '}}'
-    while current < len(template):
+    while not template.closed:
         try:
             size = 0
-            while look(size, 2) != l_del:
+            while peek(size, 2) != l_del:
                 size += 1
         except EOFError:
             yield ('literal', get(size))
@@ -44,27 +49,27 @@ def tokenize(template):
         if size != 0:
             yield ('literal', get(size))
 
-        current += size + 2
+        get(2)
 
-        tag_type = tag_types.get(get(), 'variable')
+        tag_type = tag_types.get(peek(0, 1), 'variable')
         if tag_type != 'variable':
-            current += 1
+            template.seek(template.tell() + 1)
 
-        size = 1
-        while look(size, 2) != r_del:
+        size = 0
+        while peek(size, 2) != r_del:
             size += 1
 
         tag_key = get(size).strip()
-        current += size + 2
 
         if tag_type == 'no escape?':
-            if look(-2, 3) == '}}}':
+            if peek(0, 3) == '}}}':
                 tag_type = 'no escape'
-                current += 1
+                get(1)
 
         elif tag_type == 'set delimiter?':
-            if look(-3) == '=':
+            if tag_key[-1] == '=':
                 l_del, r_del = tag_key[:-1].split(' ')
+                get(2)
                 continue
 
         elif tag_type in ['section', 'inverted section']:
@@ -75,8 +80,12 @@ def tokenize(template):
             if tag_key != last_section:
                 raise UnclosedSection()
 
+        get(2)
         yield (tag_type, tag_key)
 
+    if open_sections:
+        raise UnclosedSection()
+
     return
 
 
@@ -84,9 +93,7 @@ if __name__ == '__main__':
     data = argv[1]
     template = argv[2]
 
-    with open(template, 'r') as f:
-        template = f.read()
+    tokens = tokenize(open(template, 'r'))
 
-    tokens = tokenize(template)
     for token in tokens:
         print(token)