Install URLs: <url>/X.zip?md5=<hex> checks the download against that md5

CLAUDE.md

@@ -35,7 +35,7 @@ python3 -I tools/i18n.py check             # language files: unused strings, los
 
 pugwash flags: `--quiet --no-check --debug --no-colour --force-colour --no-log --offline --no-harbour`.
 
-To test on a device: `./do_release.sh test`, copy `dist/PortMaster.zip` over (stream it over ssh: `scp` has truncated files on some devices), then on the device run `./PortMaster/harbourmaster --no-check install ./PortMaster.zip` (the `./` prefix makes it install a local file).
+To test on a device: `./do_release.sh test`, copy `dist/PortMaster.zip` over (stream it over ssh: `scp` has truncated files on some devices), then on the device run `./PortMaster/harbourmaster --no-check install ./PortMaster.zip` (the `./` prefix makes it install a local file). `harbourmaster install` also takes a URL: `<url>/X.zip.md5` fetches the md5 file and then `X.zip`, and `<url>/X.zip?md5=<hex>` checks the download against that md5 (the `md5` parameter isn't sent to the server); both must agree when given together (`raw_download` in `source/__init__.py`).
 
 ## Local-dev mode (HM_TESTING)
 

PortMaster/pylibs/harbourmaster/source/__init__.py

@@ -4,7 +4,9 @@
 # PortMasterV3 is the only one left; raw_download handles installs from a plain url.
 
 from harbourmaster.i18n import _
-from urllib.parse import urlparse, urlunparse
+import re
+
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 from loguru import logger
 from ..info import port_info_load
 from ..util import name_cleaner
@@ -15,6 +17,8 @@
 
 ################################################################################
 ## Raw Downloader
+MD5_RE = re.compile(r'^[0-9a-f]{32}$')
+
 
 def raw_download(save_path, file_url, callback=None, file_name=None, md5_source=None):
     """
@@ -24,8 +28,29 @@ def raw_download(save_path, file_url, callback=None, file_name=None, md5_source=
     original_url = file_url
     url_info = urlparse(file_url)
 
+    ## "?md5=<hex>" on the url: the md5 the download must have, and not sent to the server.
+    query = parse_qsl(url_info.query, keep_blank_values=True)
+    md5_query = [value for key, value in query if key.lower() == 'md5']
+    if md5_query:
+        md5_query = md5_query[-1].strip().lower()
+        if not MD5_RE.match(md5_query):
+            if callback is not None:
+                callback.message_box(_("Unable to download file, its md5 is not valid."))
+
+            logger.error(f"Unable to download file: {file_url!r} [bad md5 {md5_query!r}]")
+            return None
+
+        url_info = url_info._replace(query=urlencode([(key, value) for key, value in query if key.lower() != 'md5']))
+        file_url = urlunparse(url_info)
+
+        if md5_source is None:
+            md5_source = md5_query
+
+    else:
+        md5_query = None
+
     if file_name is None:
-        file_name = url_info.path.rsplit('/', 1)[1]
+        file_name = url_info.path.rsplit('/', 1)[-1]
 
     if file_name.endswith('.md5') or file_name.endswith('.md5sum'):
         ## If it is an md5 file, we assume the actual zip is sans the md5/md5sum
@@ -36,7 +61,15 @@ def raw_download(save_path, file_url, callback=None, file_name=None, md5_source=
             logger.error(f"Unable to download file: {file_url!r}")
             return None
 
-        md5_source = md5_source.strip().split(' ', 1)[0]
+        md5_source = md5_source.strip().split(None, 1)[0].lower() if md5_source.strip() else ""
+
+        if md5_query is not None and md5_source != md5_query:
+            ## Two answers to "what should it be": neither can be trusted.
+            if callback is not None:
+                callback.message_box(_("Download validation failed."))
+
+            logger.error(f"Unable to download file: {file_url!r} [md5 file {md5_source!r} != url md5 {md5_query!r}]")
+            return None
 
         file_name = file_name.rsplit('.', 1)[0]
         file_url = urlunparse(url_info._replace(path=url_info.path.rsplit('.', 1)[0]))

PortMaster/pylibs/locales/de_DE.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "{runtime} konnte nicht heruntergeladen werden, möglicherweise funktioniert das Spiel nicht korrekt.",
   "Successfully downloaded {runtime}.": "Erfolgreich {runtime} heruntergeladen",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\nUm dies zu tun:\nMENÜ→-> UTILITIES -> Rom-Verzeichnis erneut scannen",
+  "Unable to download file, its md5 is not valid.": "Datei kann nicht heruntergeladen werden, ihre md5 ist ungültig.",
   "Unable to download verification file.": "Überprüfungsdatei konnte nicht heruntergeladen werden",
   "Unable to download non zip files.": "Nur Zip Dateien können heruntergeladen werden.",
   "Cache out of date.": "Cache veraltet",

PortMaster/pylibs/locales/es_ES.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "No se puede descargar {runtime}, es posible que el juego no funcione correctamente.",
   "Successfully downloaded {runtime}.": "{runtime} Descargado correctamente.",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\nPara hacerlo:\n\nMENÚ > UTILIDADES > REESCANEAR DIRECTORIO DE ROMS",
+  "Unable to download file, its md5 is not valid.": "No se puede descargar el archivo, su md5 no es válido.",
   "Unable to download verification file.": "No se puede descargar el archivo de verificación.",
   "Unable to download non zip files.": "No se pueden descargar archivos de tipo no zip.",
   "Cache out of date.": "Caché Obsoleta.",

PortMaster/pylibs/locales/fr_FR.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "Impossible de télécharger {runtime}, le jeu pourrait ne pas s'exécuter correctement.",
   "Successfully downloaded {runtime}.": "Téléchargement de {runtime} réussi.",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\nPour ce faire :\nMENU -> UTILITIES -> Rescan Rom Directory",
+  "Unable to download file, its md5 is not valid.": "Impossible de télécharger le fichier, son md5 n'est pas valide.",
   "Unable to download verification file.": "Impossible de télécharger le fichier de vérification.",
   "Unable to download non zip files.": "Impossible de télécharger des fichiers non compressés.",
   "Cache out of date.": "Cache obsolète.",

PortMaster/pylibs/locales/it_IT.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "Impossibile scaricare la runtime {runtime}, il gioco potrebbe presentare dei problemi.",
   "Successfully downloaded {runtime}.": "Scaricato con successo {runtime}",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\nPer farlo:\nMENU -> UTILITIES -> Rescan Rom Directory",
+  "Unable to download file, its md5 is not valid.": "Impossibile scaricare il file, il suo md5 non è valido.",
   "Unable to download verification file.": "Non posso scaricare i file di verifica",
   "Unable to download non zip files.": "Non posso scaricare i file non zip",
   "Cache out of date.": "Cache Obsoleta.",

PortMaster/pylibs/locales/ja_JP.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "{runtime} をダウンロードできません。ゲームが正しく実行されない可能性があります。",
   "Successfully downloaded {runtime}.": "{runtime} のダウンロードに成功しました。",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\n手順:\nMENU -> UTILITIES -> Rescan Rom Directory",
+  "Unable to download file, its md5 is not valid.": "ファイルをダウンロードできません。md5 が正しくありません。",
   "Unable to download verification file.": "確認情報が見つかりません。",
   "Unable to download non zip files.": "非 zip ファイルをダウンロードできません。",
   "Cache out of date.": "キャッシュが古いです。",

PortMaster/pylibs/locales/ko_KR.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "{runtime} 다운로드할 수 없습니다. 게임이 제대로 실행되지 않을 수 있습니다.",
   "Successfully downloaded {runtime}.": "{runtime} 다운로드 성공",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\n그렇게 하기 위해:\n메뉴 -> 유틸리티 -> 롬 디렉토리 다시 스캔",
+  "Unable to download file, its md5 is not valid.": "파일을 다운로드할 수 없습니다. md5가 올바르지 않습니다.",
   "Unable to download verification file.": "검증 파일을 다운로드할 수 없습니다.",
   "Unable to download non zip files.": "Zip 파일이 아니면 다운로드할 수 없습니다.",
   "Cache out of date.": "캐시가 오래되었습니다.",

PortMaster/pylibs/locales/nl_NL.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "Kan {runtime} niet downloaden, het spel kan mogelijk niet correct worden uitgevoerd.",
   "Successfully downloaded {runtime}.": "{runtime} succesvol gedownload.",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\nOm dit te doen:\nMENU → HULPPROGRAMMA'S → Rom-map opnieuw scannen",
+  "Unable to download file, its md5 is not valid.": "Kan bestand niet downloaden, de md5 is ongeldig.",
   "Unable to download verification file.": "Kan verificatiebestand niet downloaden.",
   "Unable to download non zip files.": "Kan geen non-zip bestanden downloaden.",
   "Cache out of date.": "Cache verouderd.",

PortMaster/pylibs/locales/pl_PL.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "Nie można pobrać {runtime}, gra może nie działać poprawnie.",
   "Successfully downloaded {runtime}.": "Pomyślnie pobrano {runtime}.",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\nAby to zrobić:\nMENU -> UTILITY -> Przeskanuj katalog Rom",
+  "Unable to download file, its md5 is not valid.": "Nie można pobrać pliku, jego md5 jest nieprawidłowe.",
   "Unable to download verification file.": "Nie można pobrać pliku weryfikacyjnego.",
   "Unable to download non zip files.": "Nie można pobrać plików bez zip.",
   "Cache out of date.": "Pamięć podręczna nieaktualna.",

PortMaster/pylibs/locales/pt_BR.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "Impossível baixar {runtime}, jogo pode não funcionar corretamente.",
   "Successfully downloaded {runtime}.": "{runtime} baixado com sucesso.",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\nPara fazer isso:\nMENU -> UTILIDADES -> Reescanear Diretório de Roms",
+  "Unable to download file, its md5 is not valid.": "Não foi possível baixar o arquivo, o md5 não é válido.",
   "Unable to download verification file.": "Impossível baixar arquivo de verificação.",
   "Unable to download non zip files.": "Incapaz de baixar arquivos que não sejam zip.",
   "Cache out of date.": "Cache desatualizado.",

PortMaster/pylibs/locales/ro_RO.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "Nu s-a putut descărca {runtime}, jocul ar putea să nu ruleze corect.",
   "Successfully downloaded {runtime}.": "{runtime} a fost descărcat cu succes.",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\nPentru a face acest lucru:\nMENU -> UTILITARE -> Rescanare Director ROM",
+  "Unable to download file, its md5 is not valid.": "Nu se poate descărca fișierul, md5-ul său nu este valid.",
   "Unable to download verification file.": "Nu s-a putut descărca fișierul de verificare.",
   "Unable to download non zip files.": "Nu s-au putut descărca fișierele non zip.",
   "Cache out of date.": "Cache-ul este expirat.",

PortMaster/pylibs/locales/ru_RU.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "Не удается загрузить {runtime}, игра может работать некорректно.",
   "Successfully downloaded {runtime}.": "{runtime} Успешно загружено.",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\nЧтобы сделать это:\nМЕНЮ -> ПРОГРАММЫ -> Повторить поиск каталога",
+  "Unable to download file, its md5 is not valid.": "Не удалось загрузить файл: неверный md5.",
   "Unable to download verification file.": "Не удалось загрузить проверочный файл.",
   "Unable to download non zip files.": "Невозможно загрузить не zip-файлы.",
   "Cache out of date.": "Кэш устарел.",

PortMaster/pylibs/locales/strings.json

@@ -27,6 +27,7 @@
   "Download validation failed.": [
     "pylibs/harbourmaster/hm_info.py:146",
     "pylibs/harbourmaster/hm_info.py:168",
+    "pylibs/harbourmaster/source/__init__.py:69",
     "pylibs/harbourmaster/util/net.py:148"
   ],
   "Unable to download file.": [
@@ -212,11 +213,14 @@
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": [
     "pylibs/harbourmaster/platform/retrodeck.py:18"
   ],
+  "Unable to download file, its md5 is not valid.": [
+    "pylibs/harbourmaster/source/__init__.py:38"
+  ],
   "Unable to download verification file.": [
-    "pylibs/harbourmaster/source/__init__.py:35"
+    "pylibs/harbourmaster/source/__init__.py:60"
   ],
   "Unable to download non zip files.": [
-    "pylibs/harbourmaster/source/__init__.py:46"
+    "pylibs/harbourmaster/source/__init__.py:79"
   ],
   "Cache out of date.": [
     "pylibs/harbourmaster/source/base.py:34"
@@ -229,19 +233,19 @@
   ],
   "Updating": [
     "pylibs/harbourmaster/source/portmaster.py:42",
-    "pylibs/pugwash/theme.py:581"
+    "pylibs/pugwash/theme.py:588"
   ],
   "Fetching latest info": [
     "pylibs/harbourmaster/source/portmaster.py:46",
-    "pylibs/pugwash/theme.py:591"
+    "pylibs/pugwash/theme.py:598"
   ],
   "Done.": [
     "pylibs/harbourmaster/source/portmaster.py:165",
-    "pylibs/pugwash/theme.py:614"
+    "pylibs/pugwash/theme.py:621"
   ],
   "Fetching info": [
     "pylibs/harbourmaster/source/portmaster.py:197",
-    "pylibs/pugwash/theme.py:620"
+    "pylibs/pugwash/theme.py:627"
   ],
   "Unable to find {port_name}.": [
     "pylibs/harbourmaster/source/portmaster.py:418"
@@ -1064,7 +1068,7 @@
     "pylibs/pugwash/theme.py:358"
   ],
   "Up to date already": [
-    "pylibs/pugwash/theme.py:588"
+    "pylibs/pugwash/theme.py:595"
   ],
   "You are switching from the {old_release_channel} to the {new_release_channel} version.\n\nDo you want to downgrade?": [
     "pylibs/pugwash/update.py:79"

PortMaster/pylibs/locales/sv_SE.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "Kunde inte ladda ner {runtime}, spelet kanske inte körs korrekt.",
   "Successfully downloaded {runtime}.": "Hämtat {runtime}.",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\nSå här gör du:\nMENU -> UTILITIES -> Rescan Rom Directory",
+  "Unable to download file, its md5 is not valid.": "Kan inte ladda ner filen, dess md5 är ogiltig.",
   "Unable to download verification file.": "Kunde inte hämta verifieringsfilen.",
   "Unable to download non zip files.": "Kan inte hämta icke-zip-filer.",
   "Cache out of date.": "Cache är föråldrad.",

PortMaster/pylibs/locales/uk_UA.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "Не вдалося завантажити {runtime}, гра може працювати некоректно.",
   "Successfully downloaded {runtime}.": "Успішно завантажено {runtime}.",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\nЩоб це зробити: \nМЕНЮ -> УТИЛІТИ -> Повторно сканувати папку з ROM'ами",
+  "Unable to download file, its md5 is not valid.": "Не вдалося завантажити файл: неправильний md5.",
   "Unable to download verification file.": "Не вдалося завантажити інформацію для перевірки.",
   "Unable to download non zip files.": "Не вдалося завантажити не zip-файли.",
   "Cache out of date.": "Кеш застарілий.",

PortMaster/pylibs/locales/zh_CN.json

@@ -60,6 +60,7 @@
   "Unable to download {runtime}, game may not run correctly.": "无法下载 {runtime}，游戏可能无法正常运行。",
   "Successfully downloaded {runtime}.": "成功下载 {runtime}。",
   "\n\nIn order to do so:\nMENU -> UTILITIES -> Rescan Rom Directory": "\n\n操作方法：\nMENU -> UTILITIES -> Rescan Rom Directory",
+  "Unable to download file, its md5 is not valid.": "无法下载文件，其 md5 无效。",
   "Unable to download verification file.": "无法下载验证信息。",
   "Unable to download non zip files.": "无法下载非zip文件。",
   "Cache out of date.": "缓存过期。",