Merge pull request #2308 from pi-hole/release/v4.0

Release/v4.0

.editorconfig

@@ -9,7 +9,7 @@ end_of_line = lf
 insert_final_newline = true
 indent_style = space
 indent_size = tab
-tab_width = 2
+tab_width = 4
 charset = utf-8
 trim_trailing_whitespace = true
 

.github/dco.yml

+require:
+  members: false

.gitignore

@@ -3,6 +3,11 @@
 *.swp
 __pycache__
 .cache
+.pytest_cache
+.tox
+.eggs
+*.egg-info
+
 
 # Created by https://www.gitignore.io/api/jetbrains+iml
 

.idea/codeStyles/Project.xml

 <component name="ProjectCodeStyleConfiguration">
   <code_scheme name="Project" version="173">
-    <option name="OTHER_INDENT_OPTIONS">
-      <value>
-        <option name="INDENT_SIZE" value="2" />
-        <option name="TAB_SIZE" value="2" />
-      </value>
-    </option>
     <MarkdownNavigatorCodeStyleSettings>
       <option name="RIGHT_MARGIN" value="72" />
     </MarkdownNavigatorCodeStyleSettings>

.stickler.yml

 linters:
   shellcheck:
     shell: bash
+  phpcs:
+  csslint:
+  flake8:

.travis.yml

@@ -7,4 +7,6 @@ python:
 install:
   - pip install -r requirements.txt
 
-script: py.test -vv
+script:
+  # tox.ini handles setup, ordering of docker build first, and then run tests
+  - tox

CONTRIBUTING.md

-_This template was created based on the work of [`udemy-dl`](https://github.com/nishad/udemy-dl/blob/master/LICENSE)._
-
 # Contributors Guide
 
 Please read and understand the contribution guide before creating an issue or pull request.

README.md

@@ -3,7 +3,7 @@
 <b>Network-wide ad blocking via your own Linux hardware</b><br/>
 </p>
 
-The Pi-hole is a [DNS sinkhole](https://en.wikipedia.org/wiki/DNS_Sinkhole) that protects your devices from unwanted content, without installing any client-side software.
+The Pi-hole[®](https://pi-hole.net/trademark-rules-and-brand-guidelines/) is a [DNS sinkhole](https://en.wikipedia.org/wiki/DNS_Sinkhole) that protects your devices from unwanted content, without installing any client-side software.
 
 - **Easy-to-install**: our versatile installer walks you through the process, and [takes less than ten minutes](https://www.youtube.com/watch?v=vKWjx1AQYgs)
 - **Resolute**: content is blocked in _non-browser locations_, such as ad-laden mobile apps and smart TVs
@@ -27,7 +27,7 @@ Those who want to get started quickly and conveniently, may install Pi-hole usin
 #### `curl -sSL https://install.pi-hole.net | bash`
 
 ## Alternative Install Methods
-[Piping to `bash` is controversial](https://pi-hole.net/2016/07/25/curling-and-piping-to-bash), as it prevents you from [reading code that is about to run](https://github.com/pi-hole/pi-hole/blob/master/automated%20install/basic-install.sh) on your system. Therefore, we provide these alternative installation methods which allow code review before installation: 
+[Piping to `bash` is controversial](https://pi-hole.net/2016/07/25/curling-and-piping-to-bash), as it prevents you from [reading code that is about to run](https://github.com/pi-hole/pi-hole/blob/master/automated%20install/basic-install.sh) on your system. Therefore, we provide these alternative installation methods which allow code review before installation:
 
 ### Method 1: Clone our repository and run
 ```
@@ -60,16 +60,21 @@ Make no mistake: **your support is absolutely vital to help keep us innovating!*
 ### Donations
 Sending a donation using our links below is **extremely helpful** in offsetting a portion of our monthly expenses:
 
-&nbsp;<img src="https://pi-hole.github.io/graphics/Badges/paypal-badge-black.svg" width="24" height="24" alt="PP"/> <a href="https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY">Donate via PayPal</a><br/>
-&nbsp;<img src="https://pi-hole.github.io/graphics/Badges/bitcoin-badge-black.svg" width="24" height="24" alt="BTC"/> Bitcoin Address: <code>1GKnevUnVaQM2pQieMyeHkpr8DXfkpfAtL</code>
+- <img src="https://pi-hole.github.io/graphics/Badges/paypal-badge-black.svg" width="24" height="24" alt="PP"/> <a href="https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY">Donate via PayPal</a><br/>
+- <img src="https://pi-hole.github.io/graphics/Badges/bitcoin-badge-black.svg" width="24" height="24" alt="BTC"/> [Bitcoin](https://commerce.coinbase.com/checkout/fb7facaf-bebd-46be-bb77-b358f4546763): <code>
+3MDPzjXu2hjw5sGLJvKUi1uXbvQPzVrbpF</code></br>
+- <img src="https://pi-hole.github.io/graphics/Badges/bitcoin-badge-black.svg" width="24" height="24" alt="BTC"/> [Bitcoin Cash](https://commerce.coinbase.com/checkout/fb7facaf-bebd-46be-bb77-b358f4546763): <code>qzqsz4aju2eecc6uhs7tus4vlwhhela24sdruf4qp5</code></br>
+- <img src="https://pi-hole.github.io/graphics/Badges/ethereum-badge-black.svg" width="24" height="24" alt="BTC"/> [Ethereum](https://commerce.coinbase.com/checkout/fb7facaf-bebd-46be-bb77-b358f4546763): <code>0x79d4e90A4a0C732819526c93e21A3F1356A2FAe1</code>
 
 ### Alternative support
-If you'd rather not donate (_which is okay!_), there are other ways you can help support us:
-
-- [Digital Ocean](http://www.digitalocean.com/?refcode=344d234950e1) affiliate link
-- [Vultr](http://www.vultr.com/?ref=7190426) affiliate link
-- [UNIXstickers.com](http://unixstickers.refr.cc/jacobs) affiliate link
-- [Pi-hole Swag Store](https://pi-hole.net/shop/)
+If you'd rather not [donate](https://pi-hole.net/donate/) (_which is okay!_), there are other ways you can help support us:
+- [Patreon](https://patreon.com/pihole) _Become a patron for rewards_
+- [Digital Ocean](http://www.digitalocean.com/?refcode=344d234950e1) _affiliate link_
+- [UNIXstickers.com](http://unixstickers.refr.cc/jacobs) _save $5 when you spend $9 using our affiliate link_
+- [Pi-hole Swag Store](https://pi-hole.net/shop/) _affiliate link_
+- [Amazon](http://www.amazon.com/exec/obidos/redirect-home/pihole09-20) _affiliate link_
+- [DNS Made Easy](https://cp.dnsmadeeasy.com/u/133706) _affiliate link_
+- [Vultr](http://www.vultr.com/?ref=7190426) _affiliate link_
 - Spreading the word about our software, and how you have benefited from it
 
 ### Contributing via GitHub
@@ -93,9 +98,6 @@ While we are primarily reachable on our <a href="https://discourse.pi-hole.net/"
   <li><a href="https://discourse.pi-hole.net/c/faqs">Frequently Asked Questions</a></li>
   <li><a href="https://github.com/pi-hole/pi-hole/wiki">Pi-hole Wiki</a></li>
   <li><a href="https://discourse.pi-hole.net/c/feature-requests?order=votes">Feature Requests</a></li>
-</ul>
-<br/>
-<ul>
   <li><a href="https://discourse.pi-hole.net/">Discourse User Forum</a></li>
   <li><a href="https://www.reddit.com/r/pihole/">Reddit</a></li>
   <li><a href="https://gitter.im/pi-hole/pi-hole">Gitter</a> (Real-time chat)</li>
@@ -127,7 +129,7 @@ You can read our [Core Feature Breakdown](https://github.com/pi-hole/pi-hole/wik
 ### The Web Interface Dashboard
 This [optional dashboard](https://github.com/pi-hole/AdminLTE) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve!
 
-<a href="https://pi-hole.github.io/graphics/Screenshots/dashboard.png"><img src="https://pi-hole.github.io/graphics/Screenshots/dashboard.png" width="888" height="522" alt="Pi-hole Dashboard"/></a>
+<img src="https://pi-hole.github.io/graphics/Screenshots/pihole-dashboard.png"  alt="Pi-hole Dashboard"/></a>
 
 Some notable features include:
 * Mobile friendly interface
@@ -142,11 +144,11 @@ Some notable features include:
 There are several ways to [access the dashboard](https://discourse.pi-hole.net/t/how-do-i-access-pi-holes-dashboard-admin-interface/3168):
 
 1. `http://<IP_ADDPRESS_OF_YOUR_PI_HOLE>/admin/`
-2. `http:/pi.hole/admin/` (when using Pi-hole as your DNS server)
+2. `http://pi.hole/admin/` (when using Pi-hole as your DNS server)
 3. `http://pi.hole/` (when using Pi-hole as your DNS server)
 
-## The Faster-Than-Light Engine
-The [FTL Engine](https://github.com/pi-hole/FTL) is a lightweight, purpose-built daemon used to provide statistics needed for the Web Interface, and its API can be easily integrated into your own projects. As the name implies, FTL does this all *very quickly*!
+## Faster-than-light Engine
+FTLDNS[™](https://pi-hole.net/trademark-rules-and-brand-guidelines/) is a lightweight, purpose-built daemon used to provide statistics needed for the Web Interface, and its API can be easily integrated into your own projects. As the name implies, FTLDNS does this all *very quickly*!
 
 Some of the statistics you can integrate include:
 * Total number of domains being blocked
@@ -172,31 +174,13 @@ Pi-hole being a **advertising-aware DNS/Web server**, makes use of the following
 * [AdminLTE Dashboard](https://github.com/almasaeed2010/AdminLTE) - premium admin control panel based on Bootstrap 3.x
 
 While quite outdated at this point, [this original blog post about Pi-hole](https://jacobsalmela.com/2015/06/16/block-millions-ads-network-wide-with-a-raspberry-pi-hole-2-0/) goes into **great detail** about how Pi-hole was originally setup and how it works. Syntactically, it's no longer accurate, but the same basic principles and logic still apply to Pi-hole's current state.
-
------
-
-## Pi-hole Projects
-- [The Big Blocklist Collection](https://wally3k.github.io)
-- [Docker Pi-hole container (x86 and ARM)](https://hub.docker.com/r/diginc/pi-hole/)
-- [Pi-Hole in the cloud](http://blog.codybunch.com/2015/07/28/Pi-Hole-in-the-cloud/)
-- [Pie in the Sky-Hole [A Pi-Hole in the cloud for ad-blocking via DNS]](https://dlaa.me/blog/post/skyhole)
-- [Pi-hole Enable/Disable Button](http://thetimmy.silvernight.org/pages/endisbutton/)
-- [Minibian Pi-hole](https://munkjensen.net/wiki/index.php/See_my_Pi-Hole#Minibian_Pi-hole)
-- [CHiP-hole: Network-wide Ad-blocker](https://www.hackster.io/jacobsalmela/chip-hole-network-wide-ad-blocker-98e037)
-- [Chrome Extension: Pi-Hole List Editor](https://chrome.google.com/webstore/detail/pi-hole-list-editor/hlnoeoejkllgkjbnnnhfolapllcnaglh) ([Source Code](https://github.com/packtloss/pihole-extension))
-- [Splunk: Pi-hole Visualiser](https://splunkbase.splunk.com/app/3023/)
-- [Adblocking with Pi-hole and Ubuntu 14.04 on VirtualBox](https://hbalagtas.blogspot.com.au/2016/02/adblocking-with-pi-hole-and-ubuntu-1404.html)
-- [Pi-hole stats in your Mac's menu bar](https://getbitbar.com/plugins/Network/pi-hole.1m.py)
-- [Pi-hole unRAID Template](https://forums.lime-technology.com/topic/36810-support-spants-nodered-mqtt-dashing-couchdb/)
-- [Copernicus: Windows Tray Application](https://github.com/goldbattle/copernicus)
-- [Let your blink1 device blink when Pi-hole filters ads](https://gist.github.com/elpatron68/ec0b4c582e5abf604885ac1e068d233f)
-- [Pi-hole metrics](https://github.com/nlamirault/pihole_exporter) exporter for [Prometheus](https://prometheus.io/)
-- [Magic Mirror with DNS Filtering](https://zonksec.com/blog/magic-mirror-dns-filtering/#dnssoftware)
-- [Pi-hole Droid: Android client](https://github.com/friimaind/pi-hole-droid)
-- [Windows DNS Swapper](https://github.com/roots84/DNS-Swapper), see [#1400](https://github.com/pi-hole/pi-hole/issues/1400)
 -----
 
 ## Coverage
+- [Software Engineering Daily: Interview with the creator of Pi-hole](https://softwareengineeringdaily.com/2018/05/29/pi-hole-ad-blocker-hardware-with-jacob-salmela/)
+- [Bloomberg Business Week: Brotherhood of the Ad blockers](https://www.bloomberg.com/news/features/2018-05-10/inside-the-brotherhood-of-pi-hole-ad-blockers)
+- [Securing DNS across all of my devices with Pi-Hole + DNS-over-HTTPS + 1.1.1.1](https://scotthelme.co.uk/securing-dns-across-all-of-my-devices-with-pihole-dns-over-https-1-1-1-1/)
+- [Adafruit: installing Pi-hole on a Pi Zero W](https://learn.adafruit.com/pi-hole-ad-blocker-with-pi-zero-w/install-pi-hole)
 - [Lifehacker: Turn A Raspberry Pi Into An Ad Blocker With A Single Command](https://www.lifehacker.com.au/2015/02/turn-a-raspberry-pi-into-an-ad-blocker-with-a-single-command/)
 - [MakeUseOf: Adblock Everywhere: The Raspberry Pi-Hole Way](http://www.makeuseof.com/tag/adblock-everywhere-raspberry-pi-hole-way/)
 - [Catchpoint: Ad-Blocking on Apple iOS9: Valuing the End User Experience](http://blog.catchpoint.com/2015/09/14/ad-blocking-apple/)
@@ -215,3 +199,12 @@ While quite outdated at this point, [this original blog post about Pi-hole](http
 - [CryptoAUSTRALIA: How We Tried 5 Privacy Focused Raspberry Pi Projects](https://blog.cryptoaustralia.org.au/2017/10/05/5-privacy-focused-raspberry-pi-projects/)
 - [CryptoAUSTRALIA: Pi-hole Workshop](https://blog.cryptoaustralia.org.au/2017/11/02/pi-hole-network-wide-ad-blocker/)
 - [Know How 355: Killing ads with a Raspberry Pi-Hole!](https://www.twit.tv/shows/know-how/episodes/355)
+
+-----
+
+## Pi-hole Projects
+- [The Big Blocklist Collection](https://wally3k.github.io)
+- [Pie in the Sky-Hole](https://dlaa.me/blog/post/skyhole)
+- [Copernicus: Windows Tray Application](https://github.com/goldbattle/copernicus)
+- [Magic Mirror with DNS Filtering](https://zonksec.com/blog/magic-mirror-dns-filtering/#dnssoftware)
+- [Windows DNS Swapper](https://github.com/roots84/DNS-Swapper)

adlists.default

-# The below list amalgamates several lists we used previously.
-# See `https://github.com/StevenBlack/hosts` for details
-##StevenBlack's list
-https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
-
-##MalwareDomains
-https://mirror1.malwaredomains.com/files/justdomains
-
-##Cameleon
-http://sysctl.org/cameleon/hosts
-
-##Zeustracker
-https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist
-
-##Disconnect.me Tracking
-https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt
-
-##Disconnect.me Ads
-https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt
-
-##Hosts-file.net
-https://hosts-file.net/ad_servers.txt
-

advanced/Scripts/list.sh

@@ -13,10 +13,11 @@ basename=pihole
 piholeDir=/etc/"${basename}"
 whitelist="${piholeDir}"/whitelist.txt
 blacklist="${piholeDir}"/blacklist.txt
-readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
+readonly regexlist="/etc/pihole/regex.list"
 reload=false
 addmode=true
 verbose=true
+wildcard=false
 
 domList=()
 
@@ -28,16 +29,19 @@ source ${colfile}
 
 
 helpFunc() {
-  if [[ "${listMain}" == "${whitelist}" ]]; then
-    param="w"
-    type="white"
-  elif [[ "${listMain}" == "${wildcardlist}" ]]; then
-    param="wild"
-    type="wildcard black"
-  else
-    param="b"
-    type="black"
-  fi
+    if [[ "${listMain}" == "${whitelist}" ]]; then
+        param="w"
+        type="white"
+    elif [[ "${listMain}" == "${regexlist}" && "${wildcard}" == true ]]; then
+        param="-wild"
+        type="wildcard black"
+    elif [[ "${listMain}" == "${regexlist}" ]]; then
+        param="-regex"
+        type="regex black"
+    else
+        param="b"
+        type="black"
+    fi
 
     echo "Usage: pihole -${param} [options] <domain> <domain2 ...>
 Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com'
@@ -55,212 +59,216 @@ Options:
 }
 
 EscapeRegexp() {
-  # This way we may safely insert an arbitrary
-  # string in our regular expressions
-  # Also remove leading "." if present
-  echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g"
+    # This way we may safely insert an arbitrary
+    # string in our regular expressions
+    # This sed is intentionally executed in three steps to ease maintainability
+    # The first sed removes any amount of leading dots
+    echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g"
 }
 
 HandleOther() {
-  # Convert to lowercase
-  domain="${1,,}"
-
-  # Check validity of domain
-  if [[ "${#domain}" -le 253 ]]; then
-    validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check
-    validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label
-  fi
-
-  if [[ -n "${validDomain}" ]]; then
-    domList=("${domList[@]}" ${validDomain})
-  else
-    echo -e "  ${CROSS} ${domain} is not a valid argument or domain name!"
-  fi
+    # Convert to lowercase
+    domain="${1,,}"
+
+    # Check validity of domain (don't check for regex entries)
+    if [[ "${#domain}" -le 253 ]]; then
+        if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then
+            validDomain="${domain}"
+        else
+            validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
+            validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label
+        fi
+    fi
+
+    if [[ -n "${validDomain}" ]]; then
+        domList=("${domList[@]}" ${validDomain})
+    else
+        echo -e "  ${CROSS} ${domain} is not a valid argument or domain name!"
+    fi
 }
 
 PoplistFile() {
-  # Check whitelist file exists, and if not, create it
-  if [[ ! -f "${whitelist}" ]]; then
-    touch "${whitelist}"
-  fi
-
-  # Check blacklist file exists, and if not, create it
-  if [[ ! -f "${blacklist}" ]]; then
-    touch "${blacklist}"
-  fi
-
-  for dom in "${domList[@]}"; do
-      # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other
-    if ${addmode}; then
-      AddDomain "${dom}" "${listMain}"
-      RemoveDomain "${dom}" "${listAlt}"
-      if [[ "${listMain}" == "${whitelist}" || "${listMain}" == "${blacklist}" ]]; then
-        RemoveDomain "${dom}" "${wildcardlist}"
-      fi
-    else
-      RemoveDomain "${dom}" "${listMain}"
+    # Check whitelist file exists, and if not, create it
+    if [[ ! -f "${whitelist}" ]]; then
+        touch "${whitelist}"
+    fi
+
+    # Check blacklist file exists, and if not, create it
+    if [[ ! -f "${blacklist}" ]]; then
+        touch "${blacklist}"
     fi
+
+    for dom in "${domList[@]}"; do
+        # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other
+        if ${addmode}; then
+            AddDomain "${dom}" "${listMain}"
+            RemoveDomain "${dom}" "${listAlt}"
+        else
+            RemoveDomain "${dom}" "${listMain}"
+        fi
   done
 }
 
 AddDomain() {
-  list="$2"
-  domain=$(EscapeRegexp "$1")
-
-  [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
-  [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
-  [[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist"
-
-  if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
-    [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
-    [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
-    bool=true
-    # Is the domain in the list we want to add it to?
-    grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
-
-    if [[ "${bool}" == false ]]; then
-      # Domain not found in the whitelist file, add it!
-      if [[ "${verbose}" == true ]]; then
-      echo -e "  ${INFO} Adding $1 to $listname..."
-      fi
-      reload=true
-      # Add it to the list we want to add it to
-      echo "$1" >> "${list}"
-    else
-      if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
-      fi
+    list="$2"
+    domain=$(EscapeRegexp "$1")
+
+    [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
+    [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
+
+    if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
+        [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
+        [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
+        bool=true
+        # Is the domain in the list we want to add it to?
+        grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
+
+        if [[ "${bool}" == false ]]; then
+            # Domain not found in the whitelist file, add it!
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} Adding ${1} to ${listname}..."
+            fi
+            reload=true
+            # Add it to the list we want to add it to
+            echo "$1" >> "${list}"
+        else
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
+            fi
+        fi
+    elif [[ "${list}" == "${regexlist}" ]]; then
+        [[ -z "${type}" ]] && type="--wildcard-only"
+        bool=true
+        domain="${1}"
+
+        [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
+
+        # Is the domain in the list?
+        # Search only for exactly matching lines
+        grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
+
+        if [[ "${bool}" == false ]]; then
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} Adding ${domain} to regex list..."
+            fi
+            reload="restart"
+            echo "$domain" >> "${regexlist}"
+        else
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} ${domain} already exists in regex list, no need to add!"
+            fi
+        fi
     fi
-  elif [[ "${list}" == "${wildcardlist}" ]]; then
-    source "${piholeDir}/setupVars.conf"
-    # Remove the /* from the end of the IP addresses
-    IPV4_ADDRESS=${IPV4_ADDRESS%/*}
-    IPV6_ADDRESS=${IPV6_ADDRESS%/*}
-    [[ -z "${type}" ]] && type="--wildcard-only"
-    bool=true
-    # Is the domain in the list?
-    grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false
-
-    if [[ "${bool}" == false ]]; then
-      if [[ "${verbose}" == true ]]; then
-      echo -e "  ${INFO} Adding $1 to wildcard blacklist..."
-      fi
-      reload="restart"
-      echo "address=/$1/${IPV4_ADDRESS}" >> "${wildcardlist}"
-      if [[ "${#IPV6_ADDRESS}" > 0 ]]; then
-        echo "address=/$1/${IPV6_ADDRESS}" >> "${wildcardlist}"
-      fi
-    else
-      if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} ${1} already exists in wildcard blacklist, no need to add!"
-      fi
-    fi
-  fi
 }
 
 RemoveDomain() {
-  list="$2"
-  domain=$(EscapeRegexp "$1")
-
-  [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
-  [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
-  [[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist"
-
-  if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
-    bool=true
-    [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
-    [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
-    # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa
-    grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
-    if [[ "${bool}" == true ]]; then
-      # Remove it from the other one
-      echo -e "  ${INFO} Removing $1 from $listname..."
-      # /I flag: search case-insensitive
-      sed -i "/${domain}/Id" "${list}"
-      reload=true
-    else
-      if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
-      fi
+    list="$2"
+    domain=$(EscapeRegexp "$1")
+
+    [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
+    [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
+
+    if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
+        bool=true
+        [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
+        [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
+        # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa
+        grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
+        if [[ "${bool}" == true ]]; then
+            # Remove it from the other one
+            echo -e "  ${INFO} Removing $1 from ${listname}..."
+            # /I flag: search case-insensitive
+            sed -i "/${domain}/Id" "${list}"
+            reload=true
+        else
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
+            fi
+        fi
+    elif [[ "${list}" == "${regexlist}" ]]; then
+        [[ -z "${type}" ]] && type="--wildcard-only"
+        domain="${1}"
+
+        [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
+
+        bool=true
+        # Is it in the list?
+        grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
+        if [[ "${bool}" == true ]]; then
+            # Remove it from the other one
+            echo -e "  ${INFO} Removing $domain from regex list..."
+            local lineNumber
+            lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:)
+            sed -i "${lineNumber}d" "${list}"
+            reload=true
+        else
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} ${domain} does not exist in regex list, no need to remove!"
+            fi
+        fi
     fi
-  elif [[ "${list}" == "${wildcardlist}" ]]; then
-    [[ -z "${type}" ]] && type="--wildcard-only"
-    bool=true
-    # Is it in the list?
-    grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false
-    if [[ "${bool}" == true ]]; then
-      # Remove it from the other one
-      echo -e "  ${INFO} Removing $1 from $listname..."
-      # /I flag: search case-insensitive
-      sed -i "/address=\/${domain}/Id" "${list}"
-      reload=true
-    else
-      if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
-      fi
-    fi
-  fi
 }
 
 # Update Gravity
 Reload() {
-  echo ""
-  pihole -g --skip-download "${type:-}"
+    echo ""
+    pihole -g --skip-download "${type:-}"
 }
 
 Displaylist() {
-  if [[ -f ${listMain} ]]; then
-    if [[ "${listMain}" == "${whitelist}" ]]; then
-      string="gravity resistant domains"
+    if [[ -f ${listMain} ]]; then
+        if [[ "${listMain}" == "${whitelist}" ]]; then
+            string="gravity resistant domains"
+        else
+            string="domains caught in the sinkhole"
+        fi
+        verbose=false
+        echo -e "Displaying $string:\n"
+        count=1
+        while IFS= read -r RD || [ -n "${RD}" ]; do
+            echo "  ${count}: ${RD}"
+            count=$((count+1))
+        done < "${listMain}"
     else
-      string="domains caught in the sinkhole"
+        echo -e "  ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}"
     fi
-    verbose=false
-    echo -e "Displaying $string:\n"
-    count=1
-    while IFS= read -r RD; do
-      echo "  ${count}: ${RD}"
-      count=$((count+1))
-    done < "${listMain}"
-  else
-    echo -e "  ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}"
-  fi
-  exit 0;
+    exit 0;
 }
 
 NukeList() {
-  if [[ -f "${listMain}" ]]; then
-    # Back up original list
-    cp "${listMain}" "${listMain}.bck~"
-    # Empty out file
-    echo "" > "${listMain}"
-  fi
+    if [[ -f "${listMain}" ]]; then
+        # Back up original list
+        cp "${listMain}" "${listMain}.bck~"
+        # Empty out file
+        echo "" > "${listMain}"
+    fi
 }
 
 for var in "$@"; do
-  case "${var}" in
-    "-w" | "whitelist"   ) listMain="${whitelist}"; listAlt="${blacklist}";;
-    "-b" | "blacklist"   ) listMain="${blacklist}"; listAlt="${whitelist}";;
-    "-wild" | "wildcard" ) listMain="${wildcardlist}";;
-    "-nr"| "--noreload"  ) reload=false;;
-    "-d" | "--delmode"   ) addmode=false;;
-    "-q" | "--quiet"     ) verbose=false;;
-    "-h" | "--help"      ) helpFunc;;
-    "-l" | "--list"      ) Displaylist;;
-    "--nuke"             ) NukeList;;
-    *                    ) HandleOther "${var}";;
-  esac
+    case "${var}" in
+        "-w" | "whitelist"   ) listMain="${whitelist}"; listAlt="${blacklist}";;
+        "-b" | "blacklist"   ) listMain="${blacklist}"; listAlt="${whitelist}";;
+        "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;;
+        "--regex" | "regex"   ) listMain="${regexlist}";;
+        "-nr"| "--noreload"  ) reload=false;;
+        "-d" | "--delmode"   ) addmode=false;;
+        "-q" | "--quiet"     ) verbose=false;;
+        "-h" | "--help"      ) helpFunc;;
+        "-l" | "--list"      ) Displaylist;;
+        "--nuke"             ) NukeList;;
+        *                    ) HandleOther "${var}";;
+    esac
 done
 
 shift
 
 if [[ $# = 0 ]]; then
-  helpFunc
+    helpFunc
 fi
 
 PoplistFile
 
 if [[ "${reload}" != false ]]; then
-  # Ensure that "restart" is used for Wildcard updates
-  Reload "${reload}"
+    # Ensure that "restart" is used for Wildcard updates
+    Reload "${reload}"
 fi

advanced/Scripts/piholeCheckout.sh

@@ -17,346 +17,179 @@ source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh"
 # piholeGitURL set in basic-install.sh
 # is_repo() sourced from basic-install.sh
 # setupVars set in basic-install.sh
+# check_download_exists sourced from basic-install.sh
+# fully_fetch_repo sourced from basic-install.sh
+# get_available_branches sourced from basic-install.sh
+# fetch_checkout_pull_branch sourced from basic-install.sh
+# checkout_pull_branch sourced from basic-install.sh
 
 source "${setupVars}"
-update="false"
-
-coltable="/opt/pihole/COL_TABLE"
-source ${coltable}
-
-check_download_exists() {
-  status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1)
-  if grep -q "404" <<< "$status"; then
-    return 1
-  else
-    return 0
-  fi
-}
-
-FTLinstall() {
-  # Download and install FTL binary
-  local binary
-  binary="${1}"
-  local path
-  path="${2}"
-  local str
-  str="Installing FTL"
-  echo -ne "  ${INFO} ${str}..."
-
-  if curl -sSL --fail "https://ftl.pi-hole.net/${path}" -o "/tmp/${binary}"; then
-    # Get sha1 of the binary we just downloaded for verification.
-    curl -sSL --fail "https://ftl.pi-hole.net/${path}.sha1" -o "/tmp/${binary}.sha1"
-    # Check if we just downloaded text, or a binary file.
-    cd /tmp || return 1
-    if sha1sum --status --quiet -c "${binary}".sha1; then
-      echo -n "transferred... "
-      stop_service pihole-FTL &> /dev/null
-      install -T -m 0755 "/tmp/${binary}" "/usr/bin/pihole-FTL"
-      rm "/tmp/${binary}" "/tmp/${binary}.sha1"
-      start_service pihole-FTL &> /dev/null
-      echo -e "${OVER}  ${TICK} ${str}"
-      return 0
-    else
-      echo -e "${OVER}  ${CROSS} ${str}"
-      echo -e "  ${COL_LIGHT_RED}Error: Download of binary from ftl.pi-hole.net failed${COL_NC}"
-      return 1
-    fi
-  else
-    echo -e "${OVER}  ${CROSS} ${str}"
-    echo -e "  ${COL_LIGHT_RED}Error: URL not found${COL_NC}"
-  fi
-}
-
-get_binary_name() {
-  local machine
-  machine=$(uname -m)
-
-  local str
-  str="Detecting architecture"
-  echo -ne "  ${INFO} ${str}..."
-  if [[ "${machine}" == "arm"* || "${machine}" == *"aarch"* ]]; then
-    # ARM
-    local rev
-    rev=$(uname -m | sed "s/[^0-9]//g;")
-    local lib
-    lib=$(ldd /bin/ls | grep -E '^\s*/lib' | awk '{ print $1 }')
-    if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then
-      echo -e "${OVER}  ${TICK} Detected ARM-aarch64 architecture"
-      binary="pihole-FTL-aarch64-linux-gnu"
-    elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then
-      if [[ "$rev" -gt "6" ]]; then
-        echo -e "${OVER}  ${TICK} Detected ARM-hf architecture (armv7+)"
-        binary="pihole-FTL-arm-linux-gnueabihf"
-      else
-        echo -e "${OVER}  ${TICK} Detected ARM-hf architecture (armv6 or lower) Using ARM binary"
-        binary="pihole-FTL-arm-linux-gnueabi"
-      fi
-    else
-      echo -e "${OVER}  ${TICK} Detected ARM architecture"
-      binary="pihole-FTL-arm-linux-gnueabi"
-    fi
-  elif [[ "${machine}" == "ppc" ]]; then
-    # PowerPC
-    echo -e "${OVER}  ${TICK} Detected PowerPC architecture"
-    binary="pihole-FTL-powerpc-linux-gnu"
-  elif [[ "${machine}" == "x86_64" ]]; then
-    # 64bit
-    echo -e "${OVER}  ${TICK} Detected x86_64 architecture"
-    binary="pihole-FTL-linux-x86_64"
-  else
-    # Something else - we try to use 32bit executable and warn the user
-    if [[ ! "${machine}" == "i686" ]]; then
-      echo -e "${OVER}  ${CROSS} ${str}...
-      ${COL_LIGHT_RED}Not able to detect architecture (unknown: ${machine}), trying 32bit executable
-      Contact support if you experience issues (e.g: FTL not running)${COL_NC}"
-    else
-      echo -e "${OVER}  ${TICK} Detected 32bit (i686) architecture"
-    fi
-    binary="pihole-FTL-linux-x86_32"
-  fi
-}
-
-fully_fetch_repo() {
-  # Add upstream branches to shallow clone
-  local directory="${1}"
-
-  cd "${directory}" || return 1
-  if is_repo "${directory}"; then
-    git remote set-branches origin '*' || return 1
-    git fetch --quiet || return 1
-  else
-    return 1
-  fi
-  return 0
-}
-
-get_available_branches() {
-  # Return available branches
-  local directory
-  directory="${1}"
-  local output
-
-  cd "${directory}" || return 1
-  # Get reachable remote branches, but store STDERR as STDOUT variable
-  output=$( { git remote show origin | grep 'tracked' | sed 's/tracked//;s/ //g'; } 2>&1 )
-  echo "$output"
-  return
-}
-
-fetch_checkout_pull_branch() {
-  # Check out specified branch
-  local directory
-  directory="${1}"
-  local branch
-  branch="${2}"
-
-  # Set the reference for the requested branch, fetch, check it put and pull it
-  cd "${directory}" || return 1
-  git remote set-branches origin "${branch}" || return 1
-  git stash --all --quiet &> /dev/null || true
-  git clean --quiet --force -d || true
-  git fetch --quiet || return 1
-  checkout_pull_branch "${directory}" "${branch}" || return 1
-}
-
-checkout_pull_branch() {
-  # Check out specified branch
-  local directory
-  directory="${1}"
-  local branch
-  branch="${2}"
-  local oldbranch
-
-  cd "${directory}" || return 1
-
-  oldbranch="$(git symbolic-ref HEAD)"
-
-  str="Switching to branch: '${branch}' from '${oldbranch}'"
-  echo -ne "  ${INFO} $str"
-  git checkout "${branch}" --quiet || return 1
-  echo -e "${OVER}  ${TICK} $str"
-
-
-  if [[ "$(git diff "${oldbranch}" | grep -c "^")" -gt "0" ]]; then
-    update="true"
-  fi
-
-  git_pull=$(git pull || return 1)
-
-  if [[ "$git_pull" == *"up-to-date"* ]]; then
-    echo -e "  ${INFO} ${git_pull}"
-  else
-    echo -e "$git_pull\\n"
-  fi
-
-  return 0
-}
 
 warning1() {
-  echo "  Please note that changing branches severely alters your Pi-hole subsystems"
-  echo "  Features that work on the master branch, may not on a development branch"
-  echo -e "  ${COL_LIGHT_RED}This feature is NOT supported unless a Pi-hole developer explicitly asks!${COL_NC}"
-  read -r -p "  Have you read and understood this? [y/N] " response
-  case "${response}" in
-  [yY][eE][sS]|[yY])
-    echo ""
-    return 0
-    ;;
-  *)
-    echo -e "\\n  ${INFO} Branch change has been cancelled"
-    return 1
-    ;;
-  esac
+    echo "  Please note that changing branches severely alters your Pi-hole subsystems"
+    echo "  Features that work on the master branch, may not on a development branch"
+    echo -e "  ${COL_LIGHT_RED}This feature is NOT supported unless a Pi-hole developer explicitly asks!${COL_NC}"
+    read -r -p "  Have you read and understood this? [y/N] " response
+    case "${response}" in
+        [yY][eE][sS]|[yY])
+            echo ""
+            return 0
+            ;;
+        *)
+            echo -e "\\n  ${INFO} Branch change has been cancelled"
+            return 1
+            ;;
+    esac
 }
 
 checkout() {
-  local corebranches
-  local webbranches
+    local corebranches
+    local webbranches
 
-  # Avoid globbing
-  set -f
+    # Avoid globbing
+    set -f
 
-  # This is unlikely
-  if ! is_repo "${PI_HOLE_FILES_DIR}" ; then
-    echo -e "  ${COL_LIGHT_RED}Error: Core Pi-hole repo is missing from system!
-  Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}"
-    exit 1;
-  fi
-  if [[ "${INSTALL_WEB}" == "true" ]]; then
-    if ! is_repo "${webInterfaceDir}" ; then
-     echo -e "  ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!
-  Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}"
-      exit 1;
+    # This is unlikely
+    if ! is_repo "${PI_HOLE_FILES_DIR}" ; then
+        echo -e "  ${COL_LIGHT_RED}Error: Core Pi-hole repo is missing from system!"
+        echo -e "  Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}"
+        exit 1;
     fi
-  fi
-
-  if [[ -z "${1}" ]]; then
-    echo -e "  ${COL_LIGHT_RED}Invalid option${COL_NC}
-  Try 'pihole checkout --help' for more information."
-    exit 1
-  fi
-
-  if ! warning1 ; then
-    exit 1
-  fi
-
-  if [[ "${1}" == "dev" ]] ; then
-    # Shortcut to check out development branches
-    echo -e "  ${INFO} Shortcut \"dev\" detected - checking out development / devel branches..."
-    echo ""
-    echo -e "  ${INFO} Pi-hole Core"
-    fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "development" || { echo "  ${CROSS} Unable to pull Core developement branch"; exit 1; }
-    if [[ "${INSTALL_WEB}" == "true" ]]; then
-      echo ""
-      echo -e "  ${INFO} Web interface"
-      fetch_checkout_pull_branch "${webInterfaceDir}" "devel" || { echo "  ${CROSS} Unable to pull Web development branch"; exit 1; }
+    if [[ "${INSTALL_WEB_INTERFACE}" == "true" ]]; then
+        if ! is_repo "${webInterfaceDir}" ; then
+            echo -e "  ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!"
+            echo -e "  Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}"
+            exit 1;
+        fi
     fi
-    #echo -e "  ${TICK} Pi-hole Core"
 
-    get_binary_name
-    local path
-    path="development/${binary}"
-    echo "development" > /etc/pihole/ftlbranch
-    FTLinstall "${binary}" "${path}"
-  elif [[ "${1}" == "master" ]] ; then
-    # Shortcut to check out master branches
-    echo -e "  ${INFO} Shortcut \"master\" detected - checking out master branches..."
-    echo -e "  ${INFO} Pi-hole core"
-    fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "master" || { echo "  ${CROSS} Unable to pull Core master branch"; exit 1; }
-    if [[ ${INSTALL_WEB} == "true" ]]; then
-      echo -e "  ${INFO} Web interface"
-      fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo "  ${CROSS} Unable to pull Web master branch"; exit 1; }
-    fi
-    #echo -e "  ${TICK} Web Interface"
-    get_binary_name
-    local path
-    path="master/${binary}"
-    echo "master" > /etc/pihole/ftlbranch
-    FTLinstall "${binary}" "${path}"
-  elif [[ "${1}" == "core" ]] ; then
-    str="Fetching branches from ${piholeGitUrl}"
-    echo -ne "  ${INFO} $str"
-    if ! fully_fetch_repo "${PI_HOLE_FILES_DIR}" ; then
-      echo -e "${OVER}  ${CROSS} $str"
-      exit 1
-    fi
-    corebranches=($(get_available_branches "${PI_HOLE_FILES_DIR}"))
-
-    if [[ "${corebranches[*]}" == *"master"* ]]; then
-      echo -e "${OVER}  ${TICK} $str
-  ${INFO} ${#corebranches[@]} branches available for Pi-hole Core"
-    else
-      # Print STDERR output from get_available_branches
-      echo -e "${OVER}  ${CROSS} $str\\n\\n${corebranches[*]}"
-      exit 1
-    fi
-
-    echo ""
-    # Have the user choose the branch they want
-    if ! (for e in "${corebranches[@]}"; do [[ "$e" == "${2}" ]] && exit 0; done); then
-      echo -e "  ${INFO} Requested branch \"${2}\" is not available"
-      echo -e "  ${INFO} Available branches for Core are:"
-      for e in "${corebranches[@]}"; do echo "      - $e"; done
-      exit 1
-    fi
-    checkout_pull_branch "${PI_HOLE_FILES_DIR}" "${2}"
-  elif [[ "${1}" == "web" ]] && [[ "${INSTALL_WEB}" == "true" ]] ; then
-    str="Fetching branches from ${webInterfaceGitUrl}"
-    echo -ne "  ${INFO} $str"
-    if ! fully_fetch_repo "${webInterfaceDir}" ; then
-      echo -e "${OVER}  ${CROSS} $str"
-      exit 1
+    if [[ -z "${1}" ]]; then
+        echo -e "  ${COL_LIGHT_RED}Invalid option${COL_NC}"
+        echo -e "  Try 'pihole checkout --help' for more information."
+        exit 1
     fi
-    webbranches=($(get_available_branches "${webInterfaceDir}"))
 
-    if [[ "${webbranches[*]}" == *"master"* ]]; then
-      echo -e "${OVER}  ${TICK} $str
-  ${INFO} ${#webbranches[@]} branches available for Web Admin"
-    else
-      # Print STDERR output from get_available_branches
-      echo -e "${OVER}  ${CROSS} $str\\n\\n${webbranches[*]}"
-      exit 1
+    if ! warning1 ; then
+        exit 1
     fi
 
-    echo ""
-    # Have the user choose the branch they want
-    if ! (for e in "${webbranches[@]}"; do [[ "$e" == "${2}" ]] && exit 0; done); then
-      echo -e "  ${INFO} Requested branch \"${2}\" is not available"
-      echo -e "  ${INFO} Available branches for Web Admin are:"
-      for e in "${webbranches[@]}"; do echo "      - $e"; done
-      exit 1
-    fi
-    checkout_pull_branch "${webInterfaceDir}" "${2}"
-  elif [[ "${1}" == "ftl" ]] ; then
-    get_binary_name
-    local path
-    path="${2}/${binary}"
+    if [[ "${1}" == "dev" ]] ; then
+        # Shortcut to check out development branches
+        echo -e "  ${INFO} Shortcut \"dev\" detected - checking out development / devel branches..."
+        echo ""
+        echo -e "  ${INFO} Pi-hole Core"
+        fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "development" || { echo "  ${CROSS} Unable to pull Core developement branch"; exit 1; }
+        if [[ "${INSTALL_WEB_INTERFACE}" == "true" ]]; then
+            echo ""
+            echo -e "  ${INFO} Web interface"
+            fetch_checkout_pull_branch "${webInterfaceDir}" "devel" || { echo "  ${CROSS} Unable to pull Web development branch"; exit 1; }
+        fi
+        #echo -e "  ${TICK} Pi-hole Core"
+
+        get_binary_name
+        local path
+        path="development/${binary}"
+        echo "development" > /etc/pihole/ftlbranch
+    elif [[ "${1}" == "master" ]] ; then
+        # Shortcut to check out master branches
+        echo -e "  ${INFO} Shortcut \"master\" detected - checking out master branches..."
+        echo -e "  ${INFO} Pi-hole core"
+        fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "master" || { echo "  ${CROSS} Unable to pull Core master branch"; exit 1; }
+        if [[ ${INSTALL_WEB_INTERFACE} == "true" ]]; then
+            echo -e "  ${INFO} Web interface"
+            fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo "  ${CROSS} Unable to pull Web master branch"; exit 1; }
+        fi
+        #echo -e "  ${TICK} Web Interface"
+        get_binary_name
+        local path
+        path="master/${binary}"
+        echo "master" > /etc/pihole/ftlbranch
+    elif [[ "${1}" == "core" ]] ; then
+        str="Fetching branches from ${piholeGitUrl}"
+        echo -ne "  ${INFO} $str"
+        if ! fully_fetch_repo "${PI_HOLE_FILES_DIR}" ; then
+            echo -e "${OVER}  ${CROSS} $str"
+            exit 1
+        fi
+        corebranches=($(get_available_branches "${PI_HOLE_FILES_DIR}"))
+
+        if [[ "${corebranches[*]}" == *"master"* ]]; then
+            echo -e "${OVER}  ${TICK} $str"
+            echo -e "${INFO} ${#corebranches[@]} branches available for Pi-hole Core"
+        else
+            # Print STDERR output from get_available_branches
+            echo -e "${OVER}  ${CROSS} $str\\n\\n${corebranches[*]}"
+            exit 1
+        fi
+
+        echo ""
+        # Have the user choose the branch they want
+        if ! (for e in "${corebranches[@]}"; do [[ "$e" == "${2}" ]] && exit 0; done); then
+            echo -e "  ${INFO} Requested branch \"${2}\" is not available"
+            echo -e "  ${INFO} Available branches for Core are:"
+            for e in "${corebranches[@]}"; do echo "      - $e"; done
+            exit 1
+        fi
+        checkout_pull_branch "${PI_HOLE_FILES_DIR}" "${2}"
+    elif [[ "${1}" == "web" ]] && [[ "${INSTALL_WEB_INTERFACE}" == "true" ]] ; then
+        str="Fetching branches from ${webInterfaceGitUrl}"
+        echo -ne "  ${INFO} $str"
+        if ! fully_fetch_repo "${webInterfaceDir}" ; then
+            echo -e "${OVER}  ${CROSS} $str"
+            exit 1
+        fi
+        webbranches=($(get_available_branches "${webInterfaceDir}"))
+
+        if [[ "${webbranches[*]}" == *"master"* ]]; then
+            echo -e "${OVER}  ${TICK} $str"
+            echo -e "${INFO} ${#webbranches[@]} branches available for Web Admin"
+        else
+            # Print STDERR output from get_available_branches
+            echo -e "${OVER}  ${CROSS} $str\\n\\n${webbranches[*]}"
+            exit 1
+        fi
+
+        echo ""
+        # Have the user choose the branch they want
+        if ! (for e in "${webbranches[@]}"; do [[ "$e" == "${2}" ]] && exit 0; done); then
+            echo -e "  ${INFO} Requested branch \"${2}\" is not available"
+            echo -e "  ${INFO} Available branches for Web Admin are:"
+            for e in "${webbranches[@]}"; do echo "      - $e"; done
+            exit 1
+        fi
+        checkout_pull_branch "${webInterfaceDir}" "${2}"
+    elif [[ "${1}" == "ftl" ]] ; then
+        get_binary_name
+        local path
+        path="${2}/${binary}"
+
+        if check_download_exists "$path"; then
+            echo "  ${TICK} Branch ${2} exists"
+            echo "${2}" > /etc/pihole/ftlbranch
+            FTLinstall "${binary}"
+            start_service pihole-FTL
+            enable_service pihole-FTL
+        else
+            echo "  ${CROSS} Requested branch \"${2}\" is not available"
+            ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') )
+            echo -e "  ${INFO} Available branches for FTL are:"
+            for e in "${ftlbranches[@]}"; do echo "      - $e"; done
+            exit 1
+        fi
 
-    if check_download_exists "$path"; then
-        echo "  ${TICK} Branch ${2} exists"
-        echo "${2}" > /etc/pihole/ftlbranch
-        FTLinstall "${binary}" "${path}"
     else
-        echo "  ${CROSS} Requested branch \"${2}\" is not available"
-        ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') )
-        echo -e "  ${INFO} Available branches for FTL are:"
-        for e in "${ftlbranches[@]}"; do echo "      - $e"; done
+        echo -e "  ${INFO} Requested option \"${1}\" is not available"
         exit 1
     fi
 
-  else
-    echo -e "  ${INFO} Requested option \"${1}\" is not available"
-    exit 1
-  fi
-
-  # Force updating everything
-  if [[ ( ! "${1}" == "web" && ! "${1}" == "ftl" ) && "${update}" == "true" ]]; then
-    echo -e "  ${INFO} Running installer to upgrade your installation"
-    if "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" --unattended; then
-      exit 0
-    else
-      echo -e "  ${COL_LIGHT_RED} Error: Unable to complete update, please contact support${COL_NC}"
-      exit 1
+    # Force updating everything
+    if [[  ! "${1}" == "web" && ! "${1}" == "ftl" ]]; then
+        echo -e "  ${INFO} Running installer to upgrade your installation"
+        if "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" --unattended; then
+            exit 0
+        else
+            echo -e "  ${COL_LIGHT_RED} Error: Unable to complete update, please contact support${COL_NC}"
+            exit 1
+        fi
     fi
-  fi
 }

advanced/Scripts/piholeLogFlush.sh

@@ -16,48 +16,51 @@ source ${colfile}
 # Constructed to return nothing when
 # a) the setting is not present in the config file, or
 # b) the setting is commented out (e.g. "#DBFILE=...")
-DBFILE="$(sed -n -e 's/^\s^.DBFILE\s*=\s*//p' /etc/pihole/pihole-FTL.conf)"
+FTLconf="/etc/pihole/pihole-FTL.conf"
+if [ -e "$FTLconf" ]; then
+    DBFILE="$(sed -n -e 's/^\s*DBFILE\s*=\s*//p' ${FTLconf})"
+fi
 # Test for empty string. Use standard path in this case.
 if [ -z "$DBFILE" ]; then
-  DBFILE="/etc/pihole/pihole-FTL.db"
+    DBFILE="/etc/pihole/pihole-FTL.db"
 fi
 
 if [[ "$@" != *"quiet"* ]]; then
-  echo -ne "  ${INFO} Flushing /var/log/pihole.log ..."
+    echo -ne "  ${INFO} Flushing /var/log/pihole.log ..."
 fi
 if [[ "$@" == *"once"* ]]; then
-  # Nightly logrotation
-  if command -v /usr/sbin/logrotate >/dev/null; then
-    # Logrotate once
-    /usr/sbin/logrotate --force /etc/pihole/logrotate
-  else
-    # Copy pihole.log over to pihole.log.1
-    # and empty out pihole.log
-    # Note that moving the file is not an option, as
-    # dnsmasq would happily continue writing into the
-    # moved file (it will have the same file handler)
-    cp /var/log/pihole.log /var/log/pihole.log.1
-    echo " " > /var/log/pihole.log
-  fi
+    # Nightly logrotation
+    if command -v /usr/sbin/logrotate >/dev/null; then
+        # Logrotate once
+        /usr/sbin/logrotate --force /etc/pihole/logrotate
+    else
+        # Copy pihole.log over to pihole.log.1
+        # and empty out pihole.log
+        # Note that moving the file is not an option, as
+        # dnsmasq would happily continue writing into the
+        # moved file (it will have the same file handler)
+        cp /var/log/pihole.log /var/log/pihole.log.1
+        echo " " > /var/log/pihole.log
+    fi
 else
-  # Manual flushing
-  if command -v /usr/sbin/logrotate >/dev/null; then
-    # Logrotate twice to move all data out of sight of FTL
-    /usr/sbin/logrotate --force /etc/pihole/logrotate; sleep 3
-    /usr/sbin/logrotate --force /etc/pihole/logrotate
-  else
-    # Flush both pihole.log and pihole.log.1 (if existing)
-    echo " " > /var/log/pihole.log
-    if [ -f /var/log/pihole.log.1 ]; then
-      echo " " > /var/log/pihole.log.1
+    # Manual flushing
+    if command -v /usr/sbin/logrotate >/dev/null; then
+        # Logrotate twice to move all data out of sight of FTL
+        /usr/sbin/logrotate --force /etc/pihole/logrotate; sleep 3
+        /usr/sbin/logrotate --force /etc/pihole/logrotate
+    else
+        # Flush both pihole.log and pihole.log.1 (if existing)
+        echo " " > /var/log/pihole.log
+        if [ -f /var/log/pihole.log.1 ]; then
+            echo " " > /var/log/pihole.log.1
+        fi
     fi
-  fi
-  # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)
-  deleted=$(sqlite3 "${DBFILE}" "DELETE FROM queries WHERE timestamp >= strftime('%s','now')-86400; select changes() from queries limit 1")
+    # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)
+    deleted=$(sqlite3 "${DBFILE}" "DELETE FROM queries WHERE timestamp >= strftime('%s','now')-86400; select changes() from queries limit 1")
 
 fi
 
 if [[ "$@" != *"quiet"* ]]; then
-  echo -e "${OVER}  ${TICK} Flushed /var/log/pihole.log"
-  echo -e "  ${TICK} Deleted ${deleted} queries from database"
+    echo -e "${OVER}  ${TICK} Flushed /var/log/pihole.log"
+    echo -e "  ${TICK} Deleted ${deleted} queries from database"
 fi

advanced/Scripts/query.sh

+#!/usr/bin/env bash
+# shellcheck disable=SC1090
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2018 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# Query Domain Lists
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+# Globals
+piholeDir="/etc/pihole"
+adListsList="$piholeDir/adlists.list"
+wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
+options="$*"
+adlist=""
+all=""
+exact=""
+blockpage=""
+matchType="match"
+
+colfile="/opt/pihole/COL_TABLE"
+source "${colfile}"
+
+# Print each subdomain
+# e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com"
+processWildcards() {
+    IFS="." read -r -a array <<< "${1}"
+    for (( i=${#array[@]}-1; i>=0; i-- )); do
+        ar=""
+        for (( j=${#array[@]}-1; j>${#array[@]}-i-2; j-- )); do
+            if [[ $j == $((${#array[@]}-1)) ]]; then
+                ar="${array[$j]}"
+            else
+                ar="${array[$j]}.${ar}"
+            fi
+        done
+        echo "${ar}"
+    done
+}
+
+# Scan an array of files for matching strings
+scanList(){
+    # Escape full stops
+    local domain="${1//./\\.}" lists="${2}" type="${3:-}"
+
+    # Prevent grep from printing file path
+    cd "$piholeDir" || exit 1
+
+    # Prevent grep -i matching slowly: http://bit.ly/2xFXtUX
+    export LC_CTYPE=C
+
+    # /dev/null forces filename to be printed when only one list has been generated
+    # shellcheck disable=SC2086
+    case "${type}" in
+        "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
+        "wc"    ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;;
+        *       ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;;
+    esac
+}
+
+if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then
+    echo "Usage: pihole -q [option] <domain>
+Example: 'pihole -q -exact domain.com'
+Query the adlists for a specified domain
+
+Options:
+  -adlist             Print the name of the block list URL
+  -exact              Search the block lists for exact domain matches
+  -all                Return all query matches within a block list
+  -h, --help          Show this help dialog"
+  exit 0
+fi
+
+if [[ ! -e "$adListsList" ]]; then
+    echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}"
+    exit 1
+fi
+
+# Handle valid options
+if [[ "${options}" == *"-bp"* ]]; then
+    exact="exact"; blockpage=true
+else
+    [[ "${options}" == *"-adlist"* ]] && adlist=true
+    [[ "${options}" == *"-all"* ]] && all=true
+    if [[ "${options}" == *"-exact"* ]]; then
+        exact="exact"; matchType="exact ${matchType}"
+    fi
+fi
+
+# Strip valid options, leaving only the domain and invalid options
+# This allows users to place the options before or after the domain
+options=$(sed -E 's/ ?-(bp|adlists?|all|exact) ?//g' <<< "${options}")
+
+# Handle remaining options
+# If $options contain non ASCII characters, convert to punycode
+case "${options}" in
+    ""             ) str="No domain specified";;
+    *" "*          ) str="Unknown query option specified";;
+    *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");;
+    *              ) domainQuery="${options}";;
+esac
+
+if [[ -n "${str:-}" ]]; then
+    echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information."
+    exit 1
+fi
+
+# Scan Whitelist and Blacklist
+lists="whitelist.txt blacklist.txt"
+mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")"
+if [[ -n "${results[*]}" ]]; then
+    wbMatch=true
+    # Loop through each result in order to print unique file title once
+    for result in "${results[@]}"; do
+        fileName="${result%%.*}"
+        if [[ -n "${blockpage}" ]]; then
+            echo "π ${result}"
+            exit 0
+        elif [[ -n "${exact}" ]]; then
+            echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
+        else
+            # Only print filename title once per file
+            if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
+                echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
+                fileName_prev="${fileName}"
+            fi
+        echo "   ${result#*:}"
+        fi
+    done
+fi
+
+# Scan Wildcards
+if [[ -e "${wildcardlist}" ]]; then
+    # Determine all subdomains, domain and TLDs
+    mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")"
+    for match in "${wildcards[@]}"; do
+        # Search wildcard list for matches
+        mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")"
+        if [[ -n "${results[*]}" ]]; then
+            if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then
+                wcMatch=true
+                echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:"
+            fi
+            case "${blockpage}" in
+                true ) echo "π ${wildcardlist##*/}"; exit 0;;
+                *    ) echo "   *.${match}";;
+            esac
+        fi
+    done
+fi
+
+# Get version sorted *.domains filenames (without dir path)
+lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)")
+
+# Query blocklists for occurences of domain
+mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")"
+
+# Handle notices
+if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then
+    echo -e "  ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists"
+    exit 0
+elif [[ -z "${results[*]}" ]]; then
+    # Result found in WL/BL/Wildcards
+    exit 0
+elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then
+    echo -e "  ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC}
+        This can be overridden using the -all option"
+    exit 0
+fi
+
+# Remove unwanted content from non-exact $results
+if [[ -z "${exact}" ]]; then
+    # Delete lines starting with #
+    # Remove comments after domain
+    # Remove hosts format IP address
+    mapfile -t results <<< "$(IFS=$'\n'; sed \
+        -e "/:#/d" \
+        -e "s/[ \\t]#.*//g" \
+        -e "s/:.*[ \\t]/:/g" \
+        <<< "${results[*]}")"
+    # Exit if result was in a comment
+    [[ -z "${results[*]}" ]] && exit 0
+fi
+
+# Get adlist file content as array
+if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
+    for adlistUrl in $(< "${adListsList}"); do
+        if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then
+            adlists+=("${adlistUrl}")
+        fi
+    done
+fi
+
+# Print "Exact matches for" title
+if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then
+    plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es"
+    echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:"
+fi
+
+for result in "${results[@]}"; do
+    fileName="${result/:*/}"
+
+    # Determine *.domains URL using filename's number
+    if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
+        fileNum="${fileName/list./}"; fileNum="${fileNum%%.*}"
+        fileName="${adlists[$fileNum]}"
+
+        # Discrepency occurs when adlists has been modified, but Gravity has not been run
+        if [[ -z "${fileName}" ]]; then
+            fileName="${COL_LIGHT_RED}(no associated adlists URL found)${COL_NC}"
+        fi
+    fi
+
+    if [[ -n "${blockpage}" ]]; then
+        echo "${fileNum} ${fileName}"
+    elif [[ -n "${exact}" ]]; then
+        echo "   ${fileName}"
+    else
+        if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
+            count=""
+            echo " ${matchType^} found in ${COL_BOLD}${fileName}${COL_NC}:"
+            fileName_prev="${fileName}"
+        fi
+        : $((count++))
+
+        # Print matching domain if $max_count has not been reached
+        [[ -z "${all}" ]] && max_count="50"
+        if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then
+            [[ "${count}" -gt "${max_count}" ]] && continue
+            echo "   ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}"
+        else
+            echo "   ${result#*:}"
+        fi
+    fi
+done
+
+exit 0

advanced/Scripts/setupLCD.sh

@@ -15,28 +15,28 @@
 # Borrowed from adafruit-pitft-helper < borrowed from raspi-config
 # https://github.com/adafruit/Adafruit-PiTFT-Helper/blob/master/adafruit-pitft-helper#L324-L334
 getInitSys() {
-	if command -v systemctl > /dev/null && systemctl | grep -q '\-\.mount'; then
-		SYSTEMD=1
-	elif [ -f /etc/init.d/cron ] && [ ! -h /etc/init.d/cron ]; then
-		SYSTEMD=0
-	else
-		echo "Unrecognised init system"
-		return 1
-	fi
+    if command -v systemctl > /dev/null && systemctl | grep -q '\-\.mount'; then
+        SYSTEMD=1
+    elif [ -f /etc/init.d/cron ] && [ ! -h /etc/init.d/cron ]; then
+        SYSTEMD=0
+    else
+        echo "Unrecognised init system"
+        return 1
+    fi
 }
 
 # Borrowed from adafruit-pitft-helper:
 # https://github.com/adafruit/Adafruit-PiTFT-Helper/blob/master/adafruit-pitft-helper#L274-L285
 autoLoginPiToConsole() {
-	if [ -e /etc/init.d/lightdm ]; then
-		if [ ${SYSTEMD} -eq 1 ]; then
-			systemctl set-default multi-user.target
-			ln -fs /etc/systemd/system/autologin@.service /etc/systemd/system/getty.target.wants/getty@tty1.service
-		else
-			update-rc.d lightdm disable 2
-			sed /etc/inittab -i -e "s/1:2345:respawn:\/sbin\/getty --noclear 38400 tty1/1:2345:respawn:\/bin\/login -f pi tty1 <\/dev\/tty1 >\/dev\/tty1 2>&1/"
-		fi
-	fi
+    if [ -e /etc/init.d/lightdm ]; then
+        if [ ${SYSTEMD} -eq 1 ]; then
+            systemctl set-default multi-user.target
+            ln -fs /etc/systemd/system/autologin@.service /etc/systemd/system/getty.target.wants/getty@tty1.service
+        else
+            update-rc.d lightdm disable 2
+            sed /etc/inittab -i -e "s/1:2345:respawn:\/sbin\/getty --noclear 38400 tty1/1:2345:respawn:\/bin\/login -f pi tty1 <\/dev\/tty1 >\/dev\/tty1 2>&1/"
+        fi
+    fi
 }
 
 ######### SCRIPT ###########

advanced/Scripts/update.sh

@@ -19,6 +19,9 @@ readonly PI_HOLE_FILES_DIR="/etc/.pihole"
 # shellcheck disable=SC2034
 PH_TEST=true
 
+# when --check-only is passed to this script, it will not perform the actual update
+CHECK_ONLY=false
+
 # shellcheck disable=SC1090
 source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh"
 # shellcheck disable=SC1091
@@ -28,201 +31,161 @@ source "/opt/pihole/COL_TABLE"
 # make_repo() sourced from basic-install.sh
 # update_repo() source from basic-install.sh
 # getGitFiles() sourced from basic-install.sh
+# get_binary_name() sourced from basic-install.sh
+# FTLcheckUpdate() sourced from basic-install.sh
 
 GitCheckUpdateAvail() {
-  local directory="${1}"
-  curdir=$PWD
-  cd "${directory}" || return
-
-  # Fetch latest changes in this repo
-  git fetch --quiet origin
-
-  # @ alone is a shortcut for HEAD. Older versions of git
-  # need @{0}
-  LOCAL="$(git rev-parse "@{0}")"
-
-  # The suffix @{upstream} to a branchname
-  # (short form <branchname>@{u}) refers
-  # to the branch that the branch specified
-  # by branchname is set to build on top of#
-  # (configured with branch.<name>.remote and
-  # branch.<name>.merge). A missing branchname
-  # defaults to the current one.
-  REMOTE="$(git rev-parse "@{upstream}")"
-
-  if [[ "${#LOCAL}" == 0 ]]; then
-    echo -e "\\n  ${COL_LIGHT_RED}Error: Local revision could not be obtained, please contact Pi-hole Support
-  Additional debugging output:${COL_NC}"
-    git status
-    exit
-  fi
-  if [[ "${#REMOTE}" == 0 ]]; then
-    echo -e "\\n  ${COL_LIGHT_RED}Error: Remote revision could not be obtained, please contact Pi-hole Support
-  Additional debugging output:${COL_NC}"
-    git status
-    exit
-  fi
-
-  # Change back to original directory
-  cd "${curdir}" || exit
-
-  if [[ "${LOCAL}" != "${REMOTE}" ]]; then
-    # Local branch is behind remote branch -> Update
-    return 0
-  else
-    # Local branch is up-to-date or in a situation
-    # where this updater cannot be used (like on a
-    # branch that exists only locally)
-    return 1
-  fi
-}
+    local directory
+    directory="${1}"
+    curdir=$PWD
+    cd "${directory}" || return
+
+    # Fetch latest changes in this repo
+    git fetch --quiet origin
+
+    # @ alone is a shortcut for HEAD. Older versions of git
+    # need @{0}
+    LOCAL="$(git rev-parse "@{0}")"
+
+    # The suffix @{upstream} to a branchname
+    # (short form <branchname>@{u}) refers
+    # to the branch that the branch specified
+    # by branchname is set to build on top of#
+    # (configured with branch.<name>.remote and
+    # branch.<name>.merge). A missing branchname
+    # defaults to the current one.
+    REMOTE="$(git rev-parse "@{upstream}")"
+
+    if [[ "${#LOCAL}" == 0 ]]; then
+        echo -e "\\n  ${COL_LIGHT_RED}Error: Local revision could not be obtained, please contact Pi-hole Support"
+        echo -e "  Additional debugging output:${COL_NC}"
+        git status
+        exit
+    fi
+    if [[ "${#REMOTE}" == 0 ]]; then
+        echo -e "\\n  ${COL_LIGHT_RED}Error: Remote revision could not be obtained, please contact Pi-hole Support"
+        echo -e "  Additional debugging output:${COL_NC}"
+        git status
+        exit
+    fi
+
+    # Change back to original directory
+    cd "${curdir}" || exit
 
-FTLcheckUpdate() {
-	local FTLversion
-	FTLversion=$(/usr/bin/pihole-FTL tag)
-	local FTLlatesttag
-	FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep 'Location' | awk -F '/' '{print $NF}' | tr -d '\r\n')
-
-	if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then
-		return 0
-	else
-		return 1
-	fi
+    if [[ "${LOCAL}" != "${REMOTE}" ]]; then
+        # Local branch is behind remote branch -> Update
+        return 0
+    else
+        # Local branch is up-to-date or in a situation
+        # where this updater cannot be used (like on a
+        # branch that exists only locally)
+        return 1
+    fi
 }
 
 main() {
-  local pihole_version_current
-  local web_version_current
-  local basicError="\\n  ${COL_LIGHT_RED}Unable to complete update, please contact Pi-hole Support${COL_NC}"
-  
-  # shellcheck disable=1090,2154
-  source "${setupVars}"
-
-  # This is unlikely
-  if ! is_repo "${PI_HOLE_FILES_DIR}" ; then
-    echo -e "\\n  ${COL_LIGHT_RED}Error: Core Pi-hole repo is missing from system!
-  Please re-run install script from https://pi-hole.net${COL_NC}"
-    exit 1;
-  fi
-
-  echo -e "  ${INFO} Checking for updates..."
-
-  if GitCheckUpdateAvail "${PI_HOLE_FILES_DIR}" ; then
-    core_update=true
-    echo -e "  ${INFO} Pi-hole Core:\\t${COL_YELLOW}update available${COL_NC}"
-  else
-    core_update=false
-    echo -e "  ${INFO} Pi-hole Core:\\t${COL_LIGHT_GREEN}up to date${COL_NC}"
-  fi
+    local basicError="\\n  ${COL_LIGHT_RED}Unable to complete update, please contact Pi-hole Support${COL_NC}"
+    local core_update
+    local web_update
+    local FTL_update
 
-  if FTLcheckUpdate ; then
-    FTL_update=true
-    echo -e "  ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}"
-  else
+    core_update=false
+    web_update=false
     FTL_update=false
-    echo -e "  ${INFO} FTL:\\t\\t${COL_LIGHT_GREEN}up to date${COL_NC}"
-  fi
 
-  # Logic: Don't update FTL when there is a core update available
-  # since the core update will run the installer which will itself
-  # re-install (i.e. update) FTL
-  if ${FTL_update} && ! ${core_update}; then
-    echo ""
-    echo -e "  ${INFO} FTL out of date"
-    FTLdetect
-    echo ""
-  fi
+    # shellcheck disable=1090,2154
+    source "${setupVars}"
 
-  if [[ "${INSTALL_WEB}" == true ]]; then
-    if ! is_repo "${ADMIN_INTERFACE_DIR}" ; then
-      echo -e "\\n  ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!
-  Please re-run install script from https://pi-hole.net${COL_NC}"
-      exit 1;
+    # This is unlikely
+    if ! is_repo "${PI_HOLE_FILES_DIR}" ; then
+        echo -e "\\n  ${COL_LIGHT_RED}Error: Core Pi-hole repo is missing from system!"
+        echo -e "  Please re-run install script from https://pi-hole.net${COL_NC}"
+        exit 1;
     fi
 
-    if GitCheckUpdateAvail "${ADMIN_INTERFACE_DIR}" ; then
-      web_update=true
-      echo -e "  ${INFO} Web Interface:\\t${COL_YELLOW}update available${COL_NC}"
+    echo -e "  ${INFO} Checking for updates..."
+
+    if GitCheckUpdateAvail "${PI_HOLE_FILES_DIR}" ; then
+        core_update=true
+        echo -e "  ${INFO} Pi-hole Core:\\t${COL_YELLOW}update available${COL_NC}"
     else
-      web_update=false
-      echo -e "  ${INFO} Web Interface:\\t${COL_LIGHT_GREEN}up to date${COL_NC}"
+        core_update=false
+        echo -e "  ${INFO} Pi-hole Core:\\t${COL_LIGHT_GREEN}up to date${COL_NC}"
+    fi
+
+    if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
+        if ! is_repo "${ADMIN_INTERFACE_DIR}" ; then
+            echo -e "\\n  ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!"
+            echo -e "  Please re-run install script from https://pi-hole.net${COL_NC}"
+            exit 1;
+        fi
+
+        if GitCheckUpdateAvail "${ADMIN_INTERFACE_DIR}" ; then
+            web_update=true
+            echo -e "  ${INFO} Web Interface:\\t${COL_YELLOW}update available${COL_NC}"
+        else
+            web_update=false
+            echo -e "  ${INFO} Web Interface:\\t${COL_LIGHT_GREEN}up to date${COL_NC}"
+        fi
     fi
 
-    # Logic
-    # If Core up to date AND web up to date:
-    #            Do nothing
-    # If Core up to date AND web NOT up to date:
-    #            Pull web repo
-    # If Core NOT up to date AND web up to date:
-    #            pull pihole repo, run install --unattended -- reconfigure
-    # if Core NOT up to date AND web NOT up to date:
-    #            pull pihole repo run install --unattended
-
-    if ! ${core_update} && ! ${web_update} ; then
-      if ! ${FTL_update} ; then
+    if FTLcheckUpdate > /dev/null; then
+        FTL_update=true
+        echo -e "  ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}"
+    else
+        case $? in
+            1)
+                echo -e "  ${INFO} FTL:\\t\\t${COL_LIGHT_GREEN}up to date${COL_NC}"
+                ;;
+            2)
+                echo -e "  ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Branch is not available.${COL_NC}\\n\\t\\t\\tUse ${COL_LIGHT_GREEN}pihole checkout ftl [branchname]${COL_NC} to switch to a valid branch."
+                ;;
+            *)
+                echo -e "  ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, contact support${COL_NC}"
+        esac
+        FTL_update=false
+    fi
+
+    if [[ "${core_update}" == false && "${web_update}" == false && "${FTL_update}" == false ]]; then
         echo ""
         echo -e "  ${TICK} Everything is up to date!"
         exit 0
-      fi
-    elif ! ${core_update} && ${web_update} ; then
-      echo ""
-      echo -e "  ${INFO} Pi-hole Web Admin files out of date"
-      getGitFiles "${ADMIN_INTERFACE_DIR}" "${ADMIN_INTERFACE_GIT_URL}"
-    elif ${core_update} && ! ${web_update} ; then
-      echo ""
-      echo -e "  ${INFO} Pi-hole core files out of date"
-      getGitFiles "${PI_HOLE_FILES_DIR}" "${PI_HOLE_GIT_URL}"
-      ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --reconfigure --unattended || \
-        echo -e "${basicError}" && exit 1
-    elif ${core_update} && ${web_update} ; then
-      echo ""
-      echo -e "  ${INFO} Updating Pi-hole core and web admin files"
-      getGitFiles "${PI_HOLE_FILES_DIR}" "${PI_HOLE_GIT_URL}"
-      ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --unattended || \
-        echo -e "${basicError}" && exit 1
-    else
-      echo -e "  ${COL_LIGHT_RED}Update script has malfunctioned, please contact Pi-hole Support${COL_NC}"
-      exit 1
     fi
-  else # Web Admin not installed, so only verify if core is up to date
-    if ! ${core_update}; then
-      if ! ${FTL_update} ; then
+
+    if [[ "${CHECK_ONLY}" == true ]]; then
         echo ""
-        echo -e "  ${INFO} Everything is up to date!"
         exit 0
-      fi
-    else
-      echo ""
-      echo -e "  ${INFO} Pi-hole Core files out of date"
-      getGitFiles "${PI_HOLE_FILES_DIR}" "${PI_HOLE_GIT_URL}"
-      ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --reconfigure --unattended || \
-        echo -e "${basicError}" && exit 1
     fi
-  fi
 
-  if [[ "${web_update}" == true ]]; then
-    web_version_current="$(/usr/local/bin/pihole version --admin --current)"
-    echo ""
-    echo -e "  ${INFO} Web Admin version is now at ${web_version_current/* v/v}
-  ${INFO} If you had made any changes in '/var/www/html/admin/', they have been stashed using 'git stash'"
-  fi
+    if [[ "${core_update}" == true ]]; then
+        echo ""
+        echo -e "  ${INFO} Pi-hole core files out of date, updating local repo."
+        getGitFiles "${PI_HOLE_FILES_DIR}" "${PI_HOLE_GIT_URL}"
+        echo -e "  ${INFO} If you had made any changes in '/etc/.pihole/', they have been stashed using 'git stash'"
+    fi
 
-  if [[ "${core_update}" == true ]]; then
-    pihole_version_current="$(/usr/local/bin/pihole version --pihole --current)"
+    if [[ "${web_update}" == true ]]; then
+        echo ""
+        echo -e "  ${INFO} Pi-hole Web Admin files out of date, updating local repo."
+        getGitFiles "${ADMIN_INTERFACE_DIR}" "${ADMIN_INTERFACE_GIT_URL}"
+        echo -e "  ${INFO} If you had made any changes in '/var/www/html/admin/', they have been stashed using 'git stash'"
+    fi
+
+    if [[ "${FTL_update}" == true ]]; then
+        echo ""
+        echo -e "  ${INFO} FTL out of date, it will be updated by the installer."
+    fi
+
+    if [[ "${FTL_update}" == true || "${core_update}" == true ]]; then
+        ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --reconfigure --unattended || \
+        echo -e "${basicError}" && exit 1
+    fi
     echo ""
-    echo -e "  ${INFO} Pi-hole version is now at ${pihole_version_current/* v/v}
-  ${INFO} If you had made any changes in '/etc/.pihole/', they have been stashed using 'git stash'"
-  fi
-
-  if [[ "${FTL_update}" == true ]]; then
-    FTL_version_current="$(/usr/bin/pihole-FTL tag)"
-    echo -e "\\n  ${INFO} FTL version is now at ${FTL_version_current/* v/v}"
-    start_service pihole-FTL
-    enable_service pihole-FTL
-  fi
-
-  echo ""
-  exit 0
+    exit 0
 }
 
+if [[ "$1" == "--check-only" ]]; then
+    CHECK_ONLY=true
+fi
+
 main

advanced/Scripts/updatecheck.sh

@@ -10,57 +10,57 @@
 
 # Credit: https://stackoverflow.com/a/46324904
 function json_extract() {
-  local key=$1
-  local json=$2
+    local key=$1
+    local json=$2
 
-  local string_regex='"([^"\]|\\.)*"'
-  local number_regex='-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?'
-  local value_regex="${string_regex}|${number_regex}|true|false|null"
-  local pair_regex="\"${key}\"[[:space:]]*:[[:space:]]*(${value_regex})"
+    local string_regex='"([^"\]|\\.)*"'
+    local number_regex='-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?'
+    local value_regex="${string_regex}|${number_regex}|true|false|null"
+    local pair_regex="\"${key}\"[[:space:]]*:[[:space:]]*(${value_regex})"
 
-  if [[ ${json} =~ ${pair_regex} ]]; then
-    echo $(sed 's/^"\|"$//g' <<< "${BASH_REMATCH[1]}")
-  else
-    return 1
-  fi
+    if [[ ${json} =~ ${pair_regex} ]]; then
+        echo $(sed 's/^"\|"$//g' <<< "${BASH_REMATCH[1]}")
+    else
+        return 1
+    fi
 }
 
 function get_local_branch() {
-  # Return active branch
-  cd "${1}" 2> /dev/null || return 1
-  git rev-parse --abbrev-ref HEAD || return 1
+    # Return active branch
+    cd "${1}" 2> /dev/null || return 1
+    git rev-parse --abbrev-ref HEAD || return 1
 }
 
 function get_local_version() {
-# Return active branch
-cd "${1}" 2> /dev/null || return 1
-git describe --long --dirty --tags || return 1
+    # Return active branch
+    cd "${1}" 2> /dev/null || return 1
+    git describe --long --dirty --tags || return 1
 }
 
 if [[ "$2" == "remote" ]]; then
 
-  if [[ "$3" == "reboot" ]]; then
-    sleep 30
-  fi
+    if [[ "$3" == "reboot" ]]; then
+        sleep 30
+    fi
 
-  GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")"
-  GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")"
-  GITHUB_FTL_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/FTL/releases/latest' 2> /dev/null)")"
+    GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")"
+    GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")"
+    GITHUB_FTL_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/FTL/releases/latest' 2> /dev/null)")"
 
-  echo -n "${GITHUB_CORE_VERSION} ${GITHUB_WEB_VERSION} ${GITHUB_FTL_VERSION}" > "/etc/pihole/GitHubVersions"
+    echo -n "${GITHUB_CORE_VERSION} ${GITHUB_WEB_VERSION} ${GITHUB_FTL_VERSION}" > "/etc/pihole/GitHubVersions"
 
 else
 
-  CORE_BRANCH="$(get_local_branch /etc/.pihole)"
-  WEB_BRANCH="$(get_local_branch /var/www/html/admin)"
-  FTL_BRANCH="$(pihole-FTL branch)"
+    CORE_BRANCH="$(get_local_branch /etc/.pihole)"
+    WEB_BRANCH="$(get_local_branch /var/www/html/admin)"
+    FTL_BRANCH="$(pihole-FTL branch)"
 
-  echo -n "${CORE_BRANCH} ${WEB_BRANCH} ${FTL_BRANCH}" > "/etc/pihole/localbranches"
+    echo -n "${CORE_BRANCH} ${WEB_BRANCH} ${FTL_BRANCH}" > "/etc/pihole/localbranches"
 
-  CORE_VERSION="$(get_local_version /etc/.pihole)"
-  WEB_VERSION="$(get_local_version /var/www/html/admin)"
-  FTL_VERSION="$(pihole-FTL version)"
+    CORE_VERSION="$(get_local_version /etc/.pihole)"
+    WEB_VERSION="$(get_local_version /var/www/html/admin)"
+    FTL_VERSION="$(pihole-FTL version)"
 
-  echo -n "${CORE_VERSION} ${WEB_VERSION} ${FTL_VERSION}" > "/etc/pihole/localversions"
+    echo -n "${CORE_VERSION} ${WEB_VERSION} ${FTL_VERSION}" > "/etc/pihole/localversions"
 
 fi