diff --git a/Tests/images/different_transparency_merged.gif b/Tests/images/different_transparency_merged.gif
deleted file mode 100644
index 94d0f53e0dd..00000000000
Binary files a/Tests/images/different_transparency_merged.gif and /dev/null differ
diff --git a/Tests/images/different_transparency_merged.png b/Tests/images/different_transparency_merged.png
new file mode 100644
index 00000000000..3438f62a6f4
Binary files /dev/null and b/Tests/images/different_transparency_merged.png differ
diff --git a/Tests/images/dispose_bgnd_rgba.gif b/Tests/images/dispose_bgnd_rgba.gif
new file mode 100644
index 00000000000..c18a0ba71f1
Binary files /dev/null and b/Tests/images/dispose_bgnd_rgba.gif differ
diff --git a/Tests/images/dispose_none_load_end_second.gif b/Tests/images/dispose_none_load_end_second.gif
deleted file mode 100644
index 5d8462cebb8..00000000000
Binary files a/Tests/images/dispose_none_load_end_second.gif and /dev/null differ
diff --git a/Tests/images/dispose_none_load_end_second.png b/Tests/images/dispose_none_load_end_second.png
new file mode 100644
index 00000000000..dc01ccbdd0c
Binary files /dev/null and b/Tests/images/dispose_none_load_end_second.png differ
diff --git a/Tests/images/dispose_prev_first_frame_seeked.gif b/Tests/images/dispose_prev_first_frame_seeked.gif
deleted file mode 100644
index bc3eb1393c1..00000000000
Binary files a/Tests/images/dispose_prev_first_frame_seeked.gif and /dev/null differ
diff --git a/Tests/images/dispose_prev_first_frame_seeked.png b/Tests/images/dispose_prev_first_frame_seeked.png
new file mode 100644
index 00000000000..85a3753e16d
Binary files /dev/null and b/Tests/images/dispose_prev_first_frame_seeked.png differ
diff --git a/Tests/images/missing_background_first_frame.gif b/Tests/images/missing_background_first_frame.gif
deleted file mode 100644
index be2c95b9993..00000000000
Binary files a/Tests/images/missing_background_first_frame.gif and /dev/null differ
diff --git a/Tests/images/missing_background_first_frame.png b/Tests/images/missing_background_first_frame.png
new file mode 100644
index 00000000000..25237ba5d20
Binary files /dev/null and b/Tests/images/missing_background_first_frame.png differ
diff --git a/Tests/test_file_gif.py b/Tests/test_file_gif.py
index fae7e912c8c..13eb06897b9 100644
--- a/Tests/test_file_gif.py
+++ b/Tests/test_file_gif.py
@@ -163,6 +163,32 @@ def test_roundtrip_save_all(tmp_path):
         assert reread.n_frames == 5
 
 
+@pytest.mark.parametrize(
+    "path, mode",
+    (
+        ("Tests/images/dispose_bgnd.gif", "RGB"),
+        # Hexeditted copy of dispose_bgnd to add transparency
+        ("Tests/images/dispose_bgnd_rgba.gif", "RGBA"),
+    ),
+)
+def test_loading_multiple_palettes(path, mode):
+    with Image.open(path) as im:
+        assert im.mode == "P"
+        first_frame_colors = im.palette.colors.keys()
+        original_color = im.convert("RGB").load()[0, 0]
+
+        im.seek(1)
+        assert im.mode == mode
+        if mode == "RGBA":
+            im = im.convert("RGB")
+
+        # Check a color only from the old palette
+        assert im.load()[0, 0] == original_color
+
+        # Check a color from the new palette
+        assert im.load()[24, 24] not in first_frame_colors
+
+
 def test_headers_saving_for_animated_gifs(tmp_path):
     important_headers = ["background", "version", "duration", "loop"]
     # Multiframe image
@@ -324,7 +350,7 @@ def test_dispose_none_load_end():
     with Image.open("Tests/images/dispose_none_load_end.gif") as img:
         img.seek(1)
 
-        assert_image_equal_tofile(img, "Tests/images/dispose_none_load_end_second.gif")
+        assert_image_equal_tofile(img, "Tests/images/dispose_none_load_end_second.png")
 
 
 def test_dispose_background():
@@ -340,12 +366,16 @@ def test_dispose_background():
 def test_dispose_background_transparency():
     with Image.open("Tests/images/dispose_bgnd_transparency.gif") as img:
         img.seek(2)
-        px = img.convert("RGBA").load()
+        px = img.load()
         assert px[35, 30][3] == 0
 
 
 def test_transparent_dispose():
-    expected_colors = [(2, 1, 2), (0, 1, 0), (2, 1, 2)]
+    expected_colors = [
+        (2, 1, 2),
+        ((0, 255, 24, 255), (0, 0, 255, 255), (0, 255, 24, 255)),
+        ((0, 0, 0, 0), (0, 0, 255, 255), (0, 0, 0, 0)),
+    ]
     with Image.open("Tests/images/transparent_dispose.gif") as img:
         for frame in range(3):
             img.seek(frame)
@@ -368,7 +398,7 @@ def test_dispose_previous_first_frame():
     with Image.open("Tests/images/dispose_prev_first_frame.gif") as im:
         im.seek(1)
         assert_image_equal_tofile(
-            im, "Tests/images/dispose_prev_first_frame_seeked.gif"
+            im, "Tests/images/dispose_prev_first_frame_seeked.png"
         )
 
 
@@ -508,7 +538,7 @@ def test_dispose2_background(tmp_path):
 
     with Image.open(out) as im:
         im.seek(1)
-        assert im.getpixel((0, 0)) == 0
+        assert im.getpixel((0, 0)) == (255, 0, 0)
 
 
 def test_transparency_in_second_frame():
@@ -517,9 +547,9 @@ def test_transparency_in_second_frame():
 
         # Seek to the second frame
         im.seek(im.tell() + 1)
-        assert im.info["transparency"] == 0
+        assert "transparency" not in im.info
 
-        assert_image_equal_tofile(im, "Tests/images/different_transparency_merged.gif")
+        assert_image_equal_tofile(im, "Tests/images/different_transparency_merged.png")
 
 
 def test_no_transparency_in_second_frame():
@@ -926,4 +956,4 @@ def test_missing_background():
     # but the disposal method is "Restore to background color"
     with Image.open("Tests/images/missing_background.gif") as im:
         im.seek(1)
-        assert_image_equal_tofile(im, "Tests/images/missing_background_first_frame.gif")
+        assert_image_equal_tofile(im, "Tests/images/missing_background_first_frame.png")
diff --git a/docs/handbook/image-file-formats.rst b/docs/handbook/image-file-formats.rst
index 692d4353149..bd44f63a3bb 100644
--- a/docs/handbook/image-file-formats.rst
+++ b/docs/handbook/image-file-formats.rst
@@ -91,8 +91,9 @@ Pillow reads GIF87a and GIF89a versions of the GIF file format. The library
 writes run-length encoded files in GIF87a by default, unless GIF89a features
 are used or GIF89a is already in use.
 
-Note that GIF files are always read as grayscale (``L``)
-or palette mode (``P``) images.
+GIF files are initially read as grayscale (``L``) or palette mode (``P``)
+images, but seeking to later frames in an image will change the mode to either
+``RGB`` or ``RGBA``, depending on whether the first frame had transparency.
 
 The :py:meth:`~PIL.Image.open` method sets the following
 :py:attr:`~PIL.Image.Image.info` properties:
diff --git a/src/PIL/GifImagePlugin.py b/src/PIL/GifImagePlugin.py
index 5b466884470..55b09abec54 100644
--- a/src/PIL/GifImagePlugin.py
+++ b/src/PIL/GifImagePlugin.py
@@ -124,8 +124,7 @@ def seek(self, frame):
         if not self._seek_check(frame):
             return
         if frame < self.__frame:
-            if frame != 0:
-                self.im = None
+            self.im = None
             self._seek(0)
 
         last_frame = self.__frame
@@ -165,12 +164,21 @@ def _seek(self, frame):
                 pass
             self.__offset = 0
 
+        if self.__frame == 1:
+            self.pyaccess = None
+            if "transparency" in self.info:
+                self.mode = "RGBA"
+                self.im.putpalettealpha(self.info["transparency"], 0)
+                self.im = self.im.convert("RGBA", Image.FLOYDSTEINBERG)
+
+                del self.info["transparency"]
+            else:
+                self.mode = "RGB"
+                self.im = self.im.convert("RGB", Image.FLOYDSTEINBERG)
         if self.dispose:
             self.im.paste(self.dispose, self.dispose_extent)
 
-        from copy import copy
-
-        self.palette = copy(self.global_palette)
+        palette = None
 
         info = {}
         frame_transparency = None
@@ -246,7 +254,7 @@ def _seek(self, frame):
 
                 if flags & 128:
                     bits = (flags & 7) + 1
-                    self.palette = ImagePalette.raw("RGB", self.fp.read(3 << bits))
+                    palette = ImagePalette.raw("RGB", self.fp.read(3 << bits))
 
                 # image data
                 bits = self.fp.read(1)[0]
@@ -257,6 +265,15 @@ def _seek(self, frame):
                 pass
                 # raise OSError, "illegal GIF tag `%x`" % s[0]
 
+        frame_palette = palette or self.global_palette
+
+        def _rgb(color):
+            if frame_palette:
+                color = tuple(frame_palette.palette[color * 3 : color * 3 + 3])
+            else:
+                color = (color, color, color)
+            return color
+
         try:
             if self.disposal_method < 2:
                 # do not dispose or none specified
@@ -272,9 +289,13 @@ def _seek(self, frame):
 
                 # by convention, attempt to use transparency first
                 color = self.info.get("transparency", frame_transparency)
-                if color is None:
-                    color = self.info.get("background", 0)
-                self.dispose = Image.core.fill("P", dispose_size, color)
+                if color is not None:
+                    dispose_mode = "RGBA"
+                    color = _rgb(color) + (0,)
+                else:
+                    dispose_mode = "RGB"
+                    color = _rgb(self.info.get("background", 0))
+                self.dispose = Image.core.fill(dispose_mode, dispose_size, color)
             else:
                 # replace with previous contents
                 if self.im:
@@ -286,24 +307,20 @@ def _seek(self, frame):
 
                     Image._decompression_bomb_check(dispose_size)
                     self.dispose = Image.core.fill(
-                        "P", dispose_size, frame_transparency
+                        "RGBA", dispose_size, _rgb(frame_transparency) + (0,)
                     )
         except AttributeError:
             pass
 
         if interlace is not None:
-            transparency = -1
-            if frame_transparency is not None:
-                if frame == 0:
-                    self.info["transparency"] = frame_transparency
-                else:
-                    transparency = frame_transparency
+            if frame == 0 and frame_transparency is not None:
+                self.info["transparency"] = frame_transparency
             self.tile = [
                 (
                     "gif",
                     (x0, y0, x1, y1),
                     self.__offset,
-                    (bits, interlace, transparency),
+                    (bits, interlace),
                 )
             ]
         else:
@@ -316,16 +333,54 @@ def _seek(self, frame):
             elif k in self.info:
                 del self.info[k]
 
-        self.mode = "L"
-        if self.palette:
-            self.mode = "P"
+        if frame == 0:
+            self.mode = "P" if frame_palette else "L"
+
+            if self.mode == "P" and not palette:
+                from copy import copy
+
+                palette = copy(self.global_palette)
+            self.palette = palette
+        else:
+            self._frame_palette = frame_palette
+            self._frame_transparency = frame_transparency
 
     def load_prepare(self):
-        if not self.im and "transparency" in self.info:
-            self.im = Image.core.fill(self.mode, self.size, self.info["transparency"])
+        if self.__frame == 0:
+            if "transparency" in self.info:
+                self.im = Image.core.fill(
+                    self.mode, self.size, self.info["transparency"]
+                )
+        else:
+            self._prev_im = self.im
+            if self._frame_palette:
+                self.mode = "P"
+                self.im = Image.core.fill("P", self.size, self._frame_transparency or 0)
+                self.im.putpalette(*self._frame_palette.getdata())
+                self._frame_palette = None
+            else:
+                self.mode = "L"
+                self.im = None
 
         super().load_prepare()
 
+    def load_end(self):
+        if self.__frame == 0:
+            return
+        if self._frame_transparency is not None:
+            self.im.putpalettealpha(self._frame_transparency, 0)
+            frame_im = self.im.convert("RGBA")
+        else:
+            frame_im = self.im.convert("RGB")
+        frame_im = self._crop(frame_im, self.dispose_extent)
+
+        self.im = self._prev_im
+        self.mode = self.im.mode
+        if frame_im.mode == "RGBA":
+            self.im.paste(frame_im, self.dispose_extent, frame_im)
+        else:
+            self.im.paste(frame_im, self.dispose_extent)
+
     def tell(self):
         return self.__frame
 
diff --git a/src/PIL/PdfImagePlugin.py b/src/PIL/PdfImagePlugin.py
index 49ba077e667..1131c63256e 100644
--- a/src/PIL/PdfImagePlugin.py
+++ b/src/PIL/PdfImagePlugin.py
@@ -135,7 +135,7 @@ def _save(im, fp, filename, save_all=False):
                 procset = "ImageB"  # grayscale
             elif im.mode == "P":
                 filter = "ASCIIHexDecode"
-                palette = im.im.getpalette("RGB")
+                palette = im.getpalette()
                 colorspace = [
                     PdfParser.PdfName("Indexed"),
                     PdfParser.PdfName("DeviceRGB"),
diff --git a/src/decode.c b/src/decode.c
index cb018a4e706..e236264cdb4 100644
--- a/src/decode.c
+++ b/src/decode.c
@@ -433,8 +433,7 @@ PyImaging_GifDecoderNew(PyObject *self, PyObject *args) {
     char *mode;
     int bits = 8;
     int interlace = 0;
-    int transparency = -1;
-    if (!PyArg_ParseTuple(args, "s|iii", &mode, &bits, &interlace, &transparency)) {
+    if (!PyArg_ParseTuple(args, "s|ii", &mode, &bits, &interlace)) {
         return NULL;
     }
 
@@ -452,7 +451,6 @@ PyImaging_GifDecoderNew(PyObject *self, PyObject *args) {
 
     ((GIFDECODERSTATE *)decoder->state.context)->bits = bits;
     ((GIFDECODERSTATE *)decoder->state.context)->interlace = interlace;
-    ((GIFDECODERSTATE *)decoder->state.context)->transparency = transparency;
 
     return (PyObject *)decoder;
 }
diff --git a/src/libImaging/Gif.h b/src/libImaging/Gif.h
index 91132e2e627..4029bbfe5f1 100644
--- a/src/libImaging/Gif.h
+++ b/src/libImaging/Gif.h
@@ -30,9 +30,6 @@ typedef struct {
      */
     int interlace;
 
-    /* The transparent palette index, or -1 for no transparency. */
-    int transparency;
-
     /* PRIVATE CONTEXT (set by decoder) */
 
     /* Interlace parameters */
diff --git a/src/libImaging/GifDecode.c b/src/libImaging/GifDecode.c
index 301f604b9b8..30478e24aac 100644
--- a/src/libImaging/GifDecode.c
+++ b/src/libImaging/GifDecode.c
@@ -248,33 +248,27 @@ ImagingGifDecode(Imaging im, ImagingCodecState state, UINT8 *buffer, Py_ssize_t
         /* To squeeze some extra pixels out of this loop, we test for
            some common cases and handle them separately. */
 
-        /* If we have transparency, we need to use the regular loop. */
-        if (context->transparency == -1) {
-            if (i == 1) {
-                if (state->x < state->xsize - 1) {
-                    /* Single pixel, not at the end of the line. */
-                    *out++ = p[0];
-                    state->x++;
-                    continue;
-                }
-            } else if (state->x + i <= state->xsize) {
-                /* This string fits into current line. */
-                memcpy(out, p, i);
-                out += i;
-                state->x += i;
-                if (state->x == state->xsize) {
-                    NEWLINE(state, context);
-                }
+        if (i == 1) {
+            if (state->x < state->xsize - 1) {
+                /* Single pixel, not at the end of the line. */
+                *out++ = p[0];
+                state->x++;
                 continue;
             }
+        } else if (state->x + i <= state->xsize) {
+            /* This string fits into current line. */
+            memcpy(out, p, i);
+            out += i;
+            state->x += i;
+            if (state->x == state->xsize) {
+                NEWLINE(state, context);
+            }
+            continue;
         }
 
         /* No shortcut, copy pixel by pixel */
         for (c = 0; c < i; c++) {
-            if (p[c] != context->transparency) {
-                *out = p[c];
-            }
-            out++;
+            *out++ = p[c];
             if (++state->x >= state->xsize) {
                 NEWLINE(state, context);
             }