diff --git a/src/PIL/Image.py b/src/PIL/Image.py
index e197c018240..9bdf1b4cd14 100644
--- a/src/PIL/Image.py
+++ b/src/PIL/Image.py
@@ -1901,7 +1901,7 @@ def remap_palette(self, dest_map, source_palette=None):
                 source_palette = bytearray(i // 3 for i in range(768))
 
         palette_bytes = b""
-        new_positions = [0] * 256
+        new_positions = [0] * min(256, max(dest_map) + 1)
 
         # pick only the used colors from the palette
         for i, oldPosition in enumerate(dest_map):