remote: T8829: fall back to GET when HEAD is not supported

HttpC.download() always probed remote URLs with HEAD before GET to
discover redirects and Content-Length. Some APIs (e.g. AbuseIPDB)
reject HEAD with 405 Method Not Allowed, causing firewall remote-group
downloads to fail and leave empty cached list files.

Treat HEAD 405/501 as unsupported and proceed with GET using the
original URL. When HEAD does not provide Content-Length, read it from
the GET response headers instead. Validate available storage after
determining file size and before opening the destination file.

Log sanitized download errors in vyos-domain-resolver when a remote-
group list-file fetch fails. Add unit tests with mock servers that
return 405 or 501 on HEAD and 200 on GET.

3 files changed, 133 insertions, 25 deletions

diff --git a/python/vyos/remote.py b/python/vyos/remote.py
index 3be87e179..724b46e98 100644
--- a/python/vyos/remote.py
+++ b/python/vyos/remote.py
@@ -413,33 +413,46 @@ class HttpC:
             # Not only would it potentially mess up with the progress bar but
             # `shutil.copyfileobj(request.raw, file)` does not handle automatic decoding.
             s.headers.update({'Accept-Encoding': 'identity'})
+            final_urlstring = self.urlstring
+            size = None
             with s.head(self.urlstring,
                         allow_redirects=True,
                         timeout=self.timeout) as r:
-                # Abort early if the destination is inaccessible.
+                # Some servers (e.g. AbuseIPDB) reject HEAD with 405 but allow GET.
+                if r.status_code not in (405, 501):
+                    # Abort early if the destination is inaccessible.
+                    r.raise_for_status()
+                    # If the request got redirected, keep the last URL we ended up with.
+                    final_urlstring = r.url
+                    if r.history and self.progressbar:
+                        print_error('Redirecting to ' + final_urlstring)
+                    # Check for the prospective file size.
+                    try:
+                        size = int(r.headers['Content-Length'])
+                    # In case the server does not supply the header.
+                    except KeyError:
+                        size = None
+            with s.get(final_urlstring, stream=True, timeout=self.timeout) as r:
                 r.raise_for_status()
-                # If the request got redirected, keep the last URL we ended up with.
-                final_urlstring = r.url
-                if r.history and self.progressbar:
-                    print_error('Redirecting to ' + final_urlstring)
-                # Check for the prospective file size.
-                try:
-                    size = int(r.headers['Content-Length'])
-                # In case the server does not supply the header.
-                except KeyError:
-                    size = None
-            if self.check_space:
-                check_storage(location, size)
-            with s.get(final_urlstring, stream=True,
-                       timeout=self.timeout) as r, open(location, 'wb') as f:
-                if self.progressbar and size:
-                    with Progressbar(CHUNK_SIZE / size) as p:
-                        for chunk in iter(lambda: begin(p.increment(), r.raw.read(CHUNK_SIZE)), b''):
-                            f.write(chunk)
-                else:
-                    # We'll try to stream the download directly with `copyfileobj()` so that large
-                    #  files (like entire VyOS images) don't occupy much memory.
-                    shutil.copyfileobj(r.raw, f)
+                if size is None:
+                    try:
+                        size = int(r.headers['Content-Length'])
+                    except KeyError:
+                        size = None
+                if self.check_space:
+                    check_storage(location, size)
+                with open(location, 'wb') as f:
+                    if self.progressbar and size:
+                        with Progressbar(CHUNK_SIZE / size) as p:
+                            for chunk in iter(
+                                lambda: begin(p.increment(), r.raw.read(CHUNK_SIZE)),
+                                b'',
+                            ):
+                                f.write(chunk)
+                    else:
+                        # We'll try to stream the download directly with `copyfileobj()` so that large
+                        #  files (like entire VyOS images) don't occupy much memory.
+                        shutil.copyfileobj(r.raw, f)
 
     def upload(self, location: str):
         # Does not yet support progressbars.
diff --git a/src/services/vyos-domain-resolver b/src/services/vyos-domain-resolver
index 65b2a476b..60fcb7df4 100755
--- a/src/services/vyos-domain-resolver
+++ b/src/services/vyos-domain-resolver
@@ -17,6 +17,8 @@ import json
 import time
 import logging
 import os
+from urllib.parse import urlsplit
+from urllib.parse import urlunsplit
 
 from vyos.configdict import dict_merge
 from vyos.configquery import ConfigTreeQuery
@@ -155,8 +157,18 @@ def update_remote_group(config):
             # Attempt to download file, use cached version if download fails
             try:
                 download(list_file, remote_config['url'], raise_error=True)
-            except:
-                logger.error(f'Failed to download list-file for {set_name} remote group')
+            except Exception as err:
+                url = urlsplit(remote_config['url'])
+                _, _, netloc = url.netloc.rpartition('@')
+                redacted_url = urlunsplit(url._replace(netloc=netloc, query='', fragment=''))
+                err_msg = str(err).replace(remote_config['url'], redacted_url)
+                logger.error(
+                    'Failed to download list-file for %s remote group: %s: %s',
+                    set_name,
+                    err.__class__.__name__,
+                    err_msg,
+                )
+                logger.debug('Download failure details for %s', set_name, exc_info=True)
                 logger.info(f'Using cached list-file for {set_name} remote group')
 
             # Read list file
diff --git a/src/tests/test_remote.py b/src/tests/test_remote.py
new file mode 100644
index 000000000..d0805cdff
--- /dev/null
+++ b/src/tests/test_remote.py
@@ -0,0 +1,83 @@
+# Copyright VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import tempfile
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler
+from http.server import HTTPServer
+from urllib.parse import urlsplit
+
+from vyos.remote import HttpC
+
+
+class HeadRejectHandler(BaseHTTPRequestHandler):
+    body = b'192.0.2.1\n'
+    head_status = 405
+
+    def do_HEAD(self):
+        self.send_response(self.head_status)
+        self.end_headers()
+
+    def do_GET(self):
+        self.send_response(200)
+        self.send_header('Content-Type', 'text/plain')
+        self.send_header('Content-Length', str(len(self.body)))
+        self.end_headers()
+        self.wfile.write(self.body)
+
+    def log_message(self, format, *args):
+        pass
+
+
+class Head405Handler(HeadRejectHandler):
+    head_status = 405
+
+
+class Head501Handler(HeadRejectHandler):
+    head_status = 501
+
+
+class TestHttpCDownload(unittest.TestCase):
+    def _download_from_server(self, handler):
+        server = HTTPServer(('127.0.0.1', 0), handler)
+        port = server.server_address[1]
+        thread = threading.Thread(target=server.serve_forever, daemon=True)
+        thread.start()
+
+        url = f'http://127.0.0.1:{port}/list.txt'
+        with tempfile.NamedTemporaryFile(delete=False) as tmp:
+            tmp_path = tmp.name
+
+        try:
+            client = HttpC(urlsplit(url))
+            client.download(tmp_path)
+            with open(tmp_path, 'rb') as downloaded:
+                self.assertEqual(downloaded.read(), handler.body)
+        finally:
+            server.shutdown()
+            server.server_close()
+            thread.join(timeout=1)
+            os.unlink(tmp_path)
+
+    def test_download_without_head_support_405(self):
+        self._download_from_server(Head405Handler)
+
+    def test_download_without_head_support_501(self):
+        self._download_from_server(Head501Handler)
+
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file