diff --git a/.gitignore b/.gitignore
index 2a21776aa54..b8506ea06cb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,7 +20,7 @@ __blobstorage__
 
 # .bak files
 *.bak
-
+*.bak2
 # OS-specific .gitignores
 
 # Mac .gitignore
diff --git a/dev/release/update_change_log.sh b/dev/release/update_change_log.sh
index 252cd285d92..a3af50a8a6e 100755
--- a/dev/release/update_change_log.sh
+++ b/dev/release/update_change_log.sh
@@ -29,16 +29,45 @@
 
 set -e
 
-SINCE_TAG="21.0.0"
-FUTURE_RELEASE="22.0.0"
+SINCE_TAG="22.0.0"
+FUTURE_RELEASE="23.0.0"
 
 SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
 
 OUTPUT_PATH="${SOURCE_TOP_DIR}/CHANGELOG.md"
+OLD_OUTPUT_PATH="${SOURCE_TOP_DIR}/CHANGELOG-old.md"
 
 # remove license header so github-changelog-generator has a clean base to append
-sed -i.bak '1,18d' "${OUTPUT_PATH}"
+sed -i.bak '1,21d' "${OUTPUT_PATH}"
+sed -i.bak '1,21d' "${OLD_OUTPUT_PATH}"
+# remove the github-changelog-generator footer from the old CHANGELOG.md
+LINE_COUNT=$(wc -l <"${OUTPUT_PATH}")
+sed -i.bak2 "$(( $LINE_COUNT-4+1 )),$ d" "${OUTPUT_PATH}"
+
+# Copy the previous CHANGELOG.md to CHANGELOG-old.md
+echo '<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Historical Changelog
+' | cat - "${OUTPUT_PATH}" "${OLD_OUTPUT_PATH}" > "${OLD_OUTPUT_PATH}".tmp
+mv "${OLD_OUTPUT_PATH}".tmp "${OLD_OUTPUT_PATH}"
 
 # use exclude-tags-regex to filter out tags used for object_store
 # crates and only only look at tags that DO NOT begin with `object_store_`