diff --git a/src/PIL/ImtImagePlugin.py b/src/PIL/ImtImagePlugin.py
index 5bcb959f143..dc707801274 100644
--- a/src/PIL/ImtImagePlugin.py
+++ b/src/PIL/ImtImagePlugin.py
@@ -39,15 +39,19 @@ def _open(self):
         # Quick rejection: if there's not a LF among the first
         # 100 bytes, this is (probably) not a text header.
 
-        if b"\n" not in self.fp.read(100):
+        buffer = self.fp.read(100)
+        if b"\n" not in buffer:
             raise SyntaxError("not an IM file")
-        self.fp.seek(0)
 
         xsize = ysize = 0
 
         while True:
 
-            s = self.fp.read(1)
+            if buffer:
+                s = buffer[:1]
+                buffer = buffer[1:]
+            else:
+                s = self.fp.read(1)
             if not s:
                 break
 
@@ -55,7 +59,12 @@ def _open(self):
 
                 # image data begins
                 self.tile = [
-                    ("raw", (0, 0) + self.size, self.fp.tell(), (self.mode, 0, 1))
+                    (
+                        "raw",
+                        (0, 0) + self.size,
+                        self.fp.tell() - len(buffer),
+                        (self.mode, 0, 1),
+                    )
                 ]
 
                 break
@@ -63,8 +72,11 @@ def _open(self):
             else:
 
                 # read key/value pair
-                # FIXME: dangerous, may read whole file
-                s = s + self.fp.readline()
+                if b"\n" not in buffer:
+                    buffer += self.fp.read(100)
+                lines = buffer.split(b"\n")
+                s += lines.pop(0)
+                buffer = b"\n".join(lines)
                 if len(s) == 1 or len(s) > 100:
                     break
                 if s[0] == ord(b"*"):