Merge branch 'development-v6' into new/migrate_dnsmasq_conf

.github/workflows/codeql-analysis.yml

@@ -25,7 +25,7 @@ jobs:
     steps:
     -
       name: Checkout repository
-      uses: actions/checkout@v4.1.1
+      uses: actions/checkout@v4.1.5
     # Initializes the CodeQL tools for scanning.
     -
       name: Initialize CodeQL

.github/workflows/merge-conflict.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check if PRs are have merge conflicts
-        uses: eps1lon/actions-label-merge-conflict@v2.1.0
+        uses: eps1lon/actions-label-merge-conflict@v3.0.1
         with:
           dirtyLabel: "PR: Merge Conflict"
           repoToken: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/stale.yml

@@ -40,7 +40,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.1.5
       - name: Remove 'stale' label
         run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }}
         env:

.github/workflows/sync-back-to-dev.yml

@@ -33,7 +33,7 @@ jobs:
     name: Syncing branches
     steps:
       - name: Checkout
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.1.5
       - name: Opening pull request
         run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal'
         env:

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.1.5
 
       - name: Check scripts in repository are executable
         run: |
@@ -62,19 +62,20 @@ jobs:
             ubuntu_20,
             ubuntu_22,
             ubuntu_23,
+            ubuntu_24,
             centos_8,
             centos_9,
-             fedora_38,
             fedora_39,
+            fedora_40,
           ]
     env:
       DISTRO: ${{matrix.distro}}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.1.5
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.1.0
         with:
           python-version: "3.10"
 

advanced/Scripts/piholeDebug.sh

@@ -865,8 +865,6 @@ make_array_from_file() {
     local limit=${2}
     # A local iterator for testing if we are at the limit above
     local i=0
-    # Set the array to be empty so we can start fresh when the function is used
-    local file_content=()
     # If the file is a directory
     if [[ -d "${filename}" ]]; then
         # do nothing since it cannot be parsed
@@ -878,11 +876,14 @@ make_array_from_file() {
             new_line=$(echo "${line}" | sed -e 's/^\s*#.*$//' -e '/^$/d')
             # If the line still has content (a non-zero value)
             if [[ -n "${new_line}" ]]; then
-                # Put it into the array
-                file_content+=("${new_line}")
-            else
-                # Otherwise, it's a blank line or comment, so do nothing
-                :
+
+                # If the string contains "### CHANGED", highlight this part in red
+                if [[ "${new_line}" == *"### CHANGED"* ]]; then
+                    new_line="${new_line//### CHANGED/${COL_RED}### CHANGED${COL_NC}}"
+                fi
+
+                # Finally, write this line to the log
+                log_write "   ${new_line}"
             fi
             # Increment the iterator +1
             i=$((i+1))
@@ -894,12 +895,6 @@ make_array_from_file() {
                 break
             fi
         done < "${filename}"
-        # Now the we have made an array of the file's content
-        for each_line in "${file_content[@]}"; do
-            # Print each line
-            # At some point, we may want to check the file line-by-line, so that's the reason for an array
-            log_write "   ${each_line}"
-        done
     fi
 }
 

advanced/Scripts/query.sh

@@ -41,14 +41,14 @@ Options:
 
 GenerateOutput() {
     local data gravity_data lists_data num_gravity num_lists search_type_str
-    local gravity_data_csv lists_data_csv line current_domain
+    local gravity_data_csv lists_data_csv line current_domain url type color
     data="${1}"
 
     # construct a new json for the list results where each object contains the domain and the related type
     lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]')
 
     # construct a new json for the gravity results where each object contains the adlist URL and the related domains
-    gravity_data=$(printf %s "${data}" | jq '.search.gravity  | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })')
+    gravity_data=$(printf %s "${data}" | jq '.search.gravity  | group_by(.address,.type) | map({ address: (.[0].address), type: (.[0].type), domains: [.[] | .domain] })')
 
     # number of objects in each json
     num_gravity=$(printf %s "${gravity_data}" | jq length)
@@ -78,15 +78,27 @@ GenerateOutput() {
     if [ "${num_gravity}" -gt 0 ]; then
         # Convert the data to a csv, each line is a "URL,domain,domain,...." string
         # not using jq's @csv here as it quotes each value individually
-        gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")')
+        gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .type, .domains[]] | join(",")')
 
         # Generate line-by-line output for each csv line
         echo "${gravity_data_csv}" | while read -r line; do
+            # Get first part of the line, the URL
+            url=${line%%,*}
+
+            # cut off URL, leaving "type,domain,domain,...."
+            line=${line#*,}
+            type=${line%%,*}
+            # type == "block" -> red, type == "allow" -> green
+            if [ "${type}" = "block" ]; then
+                color="${COL_RED}"
+            else
+                color="${COL_GREEN}"
+            fi
 
             # print adlist URL
-            printf "%s\n\n" "  - ${COL_BLUE}${line%%,*}${COL_NC}"
+            printf "%s (%s)\n\n" "  - ${COL_BLUE}${url}${COL_NC}" "${color}${type}${COL_NC}"
 
-            # cut off URL, leaving "domain,domain,...."
+            # cut off type, leaving "domain,domain,...."
             line=${line#*,}
             # print each domain and remove it from the string until nothing is left
             while [ ${#line} -gt 0 ]; do

advanced/Templates/gravity.db.sql

@@ -27,7 +27,7 @@ CREATE TABLE domainlist
 CREATE TABLE adlist
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	address TEXT UNIQUE NOT NULL,
+	address TEXT NOT NULL,
 	enabled BOOLEAN NOT NULL DEFAULT 1,
 	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
@@ -37,7 +37,8 @@ CREATE TABLE adlist
 	invalid_domains INTEGER NOT NULL DEFAULT 0,
 	status INTEGER NOT NULL DEFAULT 0,
 	abp_entries INTEGER NOT NULL DEFAULT 0,
-	type INTEGER NOT NULL DEFAULT 0
+	type INTEGER NOT NULL DEFAULT 0,
+	UNIQUE(address, type)
 );
 
 CREATE TABLE adlist_by_group

automated install/basic-install.sh

@@ -826,12 +826,12 @@ If you want to specify a port other than 53, separate it with a hash.\
             printf -v PIHOLE_DNS_1 "%s" "${piholeDNS%%,*}"
             printf -v PIHOLE_DNS_2 "%s" "${piholeDNS##*,}"
 
-            # If the first DNS value is invalid or empty, this if statement will be true and we will set PIHOLE_DNS_1="Invalid"
-            if ! valid_ip "${PIHOLE_DNS_1}" || [[ ! "${PIHOLE_DNS_1}" ]]; then
+            # If the first DNS value is invalid (neither IPv4 nor IPv6) or empty, set PIHOLE_DNS_1="Invalid"
+            if ! valid_ip "${PIHOLE_DNS_1}" && ! valid_ip6 "${PIHOLE_DNS_1}" || [[ -z "${PIHOLE_DNS_1}" ]]; then
                 PIHOLE_DNS_1=${strInvalid}
             fi
-            # If the second DNS value is invalid or empty, this if statement will be true and we will set PIHOLE_DNS_2="Invalid"
-            if ! valid_ip "${PIHOLE_DNS_2}" && [[ "${PIHOLE_DNS_2}" ]]; then
+            # If the second DNS value is invalid but not empty, set PIHOLE_DNS_2="Invalid"
+            if ! valid_ip "${PIHOLE_DNS_2}" && ! valid_ip6 "${PIHOLE_DNS_2}" && [[ -n "${PIHOLE_DNS_2}" ]]; then
                 PIHOLE_DNS_2=${strInvalid}
             fi
             # If either of the DNS servers are invalid,
@@ -2005,9 +2005,11 @@ FTLcheckUpdate() {
     local localSha1
 
     if [[ ! "${ftlBranch}" == "master" ]]; then
-        # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update!
+        # This is not the master branch
         local path
         path="${ftlBranch}/${binary}"
+
+        # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update!
         # shellcheck disable=SC1090
         check_download_exists "$path"
         local ret=$?
@@ -2026,23 +2028,22 @@ FTLcheckUpdate() {
         fi
 
         if [[ ${ftlLoc} ]]; then
-            # We already have a pihole-FTL binary downloaded.
-            # Alt branches don't have a tagged version against them, so just confirm the checksum of the local vs remote to decide whether we download or not
-            remoteSha1=$(curl -sSL --fail "https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" | cut -d ' ' -f 1)
-            localSha1=$(sha1sum "$(command -v pihole-FTL)" | cut -d ' ' -f 1)
-
-            if [[ "${remoteSha1}" != "${localSha1}" ]]; then
-                printf "  %b Checksums do not match, downloading from ftl.pi-hole.net.\\n" "${INFO}"
-                return 0
-            else
-                printf "  %b Checksum of installed binary matches remote. No need to download!\\n" "${INFO}"
-                return 1
-            fi
+            # We already have a pihole-FTL binary installed, check if it's the
+            # same as the remote one
+            # Alt branches don't have a tagged version against them, so just
+            # confirm the checksum of the local vs remote to decide whether we
+            # download or not
+            printf "  %b FTL binary already installed, verifying integrity...\\n" "${INFO}"
+            checkSumFile="https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1"
+            # Continue further down...
         else
             return 0
         fi
     else
+        # This is the master branch
         if [[ ${ftlLoc} ]]; then
+            # We already have a pihole-FTL binary installed, check if it's the
+            # same as the remote one
             local FTLversion
             FTLversion=$(/usr/bin/pihole-FTL tag)
             local FTLlatesttag
@@ -2056,25 +2057,39 @@ FTLcheckUpdate() {
 
             # Check if the installed version matches the latest version
             if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then
+                # If the installed version does not match the latest version,
+                # then download
                 return 0
             else
-                printf "  %b Latest FTL Binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}"
-
-                remoteSha1=$(curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" | cut -d ' ' -f 1)
-                localSha1=$(sha1sum "$(command -v pihole-FTL)" | cut -d ' ' -f 1)
-
-                if [[ "${remoteSha1}" != "${localSha1}" ]]; then
-                    printf "  %b Corruption detected...\\n" "${INFO}"
-                    return 0
-                else
-                    printf "  %b Checksum correct. No need to download!\\n" "${INFO}"
-                    return 1
-                fi
+                # If the installed version matches the latest version, then
+                # check the installed sha1sum of the binary vs the remote
+                # sha1sum. If they do not match, then download
+                printf "  %b Latest FTL binary already installed (%s), verifying integrity...\\n" "${INFO}" "${FTLlatesttag}"
+                checkSumFile="https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1"
+                # Continue further down...
             fi
         else
             return 0
         fi
     fi
+
+    # If we reach this point, we need to check the checksum of the local vs
+    # remote to decide whether we download or not
+    remoteSha1=$(curl -sSL --fail "${checkSumFile}" | cut -d ' ' -f 1)
+    localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1)
+
+    # Check we downloaded a valid checksum (no 404 or other error like
+    # no DNS resolution)
+    if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then
+        printf "  %b Remote checksum not available, trying to redownload...\\n" "${CROSS}"
+        return 0
+    elif [[ "${remoteSha1}" != "${localSha1}" ]]; then
+        printf "  %b Remote binary is different, downloading...\\n" "${CROSS}"
+        return 0
+    fi
+
+    printf "  %b Local binary up-to-date. No need to download!\\n" "${INFO}"
+    return 1
 }
 
 # Detect suitable FTL binary platform