[3rd-party images] Retrieve Debian and Fedora images (#4293)

Several notables changes presented here:

1. Debian and Fedora images presented in `multipass find`:

Details for 3rd-party cloud images are located in
`data/distributions/distribution-info.json` and roughly follow the
format of `VMImageInfo` and `simplestreams`. Once this PR is merged in,
the `CustomVMImageHost` will fetch this file from the Multipass GitHub
repository. For local developer testing, the environment variable
`MULTIPASS_DISTRIBUTIONS_URL` can be set to a local directory, i.e.
`file:///Users/you/path/to/your/manifest.json`. Reported images are
sorted in descending order by their `remote_name` placing 3rd-party
images at the bottom of the list, since the remote name is empty.

2. Ubuntu Core images are now fetched via the `UbuntuVMImageHost` class:

Ubuntu Core images utilize
[simplestreams](https://cdimage.ubuntu.com/ubuntu-core/streams/v1/com.ubuntu.cdimage:ubuntu-core.json)
for distribution. Several tweaks were made to
`simple_streams_manifest.cpp` to accommodate them. As a result, core
images are also now reported on ARM64 CPU architectures.

3. Golden files extracted out of `test_output_formatter.cpp`:

Test files for the formatters are now located under
`tests/test_data/formatters/<formatter>/`. This eliminates the need for
escaping characters, line length linting, and translating text output
from the command line into C++ strings.

4. Removed hardcoded "Ubuntu" strings within formatters, installers, and
CLI

- remove hardcoded "Ubuntu" strings from `info` and `list` formatters
- retroactively populate the image vault with new `OS` field
  - fix bug where query type was not being loaded from the DB correctly
- this function will need to be removed in a future release of
Multipass, although the function is idempotent
- all pre-existing instances will have the field populated with "Ubuntu"
- remove specific references to Ubuntu within the CLI and installer in
favour of more generic "cloud VMs/instances"

5. Fedora compatible disk info fetching

- Include BTRFS filesystems when counting disk info
- Filter out filesystems that are mounted on multiple directories

6. Drive-by fixes:

- Determining the location of `test_data` is now done at compile time
rather than runtime allowing the use of the location in objects that are
constructed at compile time.
- Moved image host classes to their own subdirectory and renamed some
classes to follow the established inheritance format of `virtual` ->
`base` -> implementation.
- `test_data` persistence optimization:
- When templated test files, i.e. `*\.in$`, are edited or new files are
added to `test_data`, a reconfigure is triggered by cmake.
- Non-templated test files are persisted into the build directory at
compile time if they have been edited.
- New exception type; `ImageNotFoundException`, created for when an
image is not found.
- Optimized sending of `remote_name` in the `FindReply` `protobuf`
message type.

Fixes #3541

---

MULTI-2048
MULTI-2074
MULTI-2151
MULTI-2153

Author: sharder996

.gitattributes

@@ -0,0 +1 @@
+tests/test_data/formatters/** -text

.github/workflows/lint.yml

@@ -36,9 +36,10 @@ jobs:
         uses: ./.github/actions/whitespace
         with:
           exclude: >-
-            '3rd-party/'    `# third-party submodules`
-            'data/'         `# data files`
-            'docs/.sphinx/' `# vendored starter pack`
+            '3rd-party/'        `# third-party submodules`
+            'data/'             `# data files`
+            'docs/.sphinx/'     `# vendored starter pack`
+            'tests/test_data/'  `# test data`
 
       - name: Install clang-format
         shell: bash
@@ -51,7 +52,11 @@ jobs:
           clang-format-16  --dump-config > /dev/null
 
           # On pull requests, HEAD^1 will always be the merge base, so consider that diff for formatting.
-          git diff -U0 --no-color HEAD^1 | clang-format-diff-16 -p1 | tee ${HOME}/clang-diff
+          git diff -U0 --no-color HEAD^1 \
+            | grep -vE '^\+\+\+ b/tests/test_data/formatters/' \
+            | clang-format-diff-16 -p1 \
+            | tee ${HOME}/clang-diff
+
           if [ "$( stat --printf='%s' ${HOME}/clang-diff )" -ne 0 ]; then
             echo "##[error] Please apply the above diff to correct formatting"
             exit 1

CMakeLists.txt

@@ -33,7 +33,7 @@ if("${HOST_OS_NAME}" STREQUAL "macOS")
   if (CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "arm64")
     set(CMAKE_OSX_DEPLOYMENT_TARGET "14.0")
   else()
-    set(CMAKE_OSX_DEPLOYMENT_TARGET "13.0")
+    set(CMAKE_OSX_DEPLOYMENT_TARGET "13.3")
   endif()
 elseif("${HOST_OS_NAME}" STREQUAL "Windows")
   set(VCPKG_HOST_OS "windows-static-md")

data/distributions/distribution-info.json

@@ -0,0 +1,44 @@
+{
+  "Debian": {
+    "aliases": "debian, trixie",
+    "items": {
+      "arm64": {
+        "id": "sha512:b24a431a9b2664ebd0ce3a1fe28ec48885ae4803a8f6f238a5aeda1ab3d223ed76525f7cd2c7ffc264ea879ee9d26c533c6c4a1951950580aaaf2a42b7f003f5",
+        "image_location": "https://cloud.debian.org/images/cloud/trixie/20250814-2204/debian-13-generic-arm64-20250814-2204.qcow2",
+        "size": 425984000,
+        "version": "20250814"
+      },
+      "x86_64": {
+        "id": "sha512:8f5c54d654b53951430b404efc3043b425cf2214467d5bf33d6c5157fa47c8fe4a1a2abf603050dafc7e54f57e9685f0d59a6c0d09d0cb2b7fcec75561c0df6f",
+        "image_location": "https://cloud.debian.org/images/cloud/trixie/20250814-2204/debian-13-generic-amd64-20250814-2204.qcow2",
+        "size": 432472064,
+        "version": "20250814"
+      }
+    },
+    "os": "Debian",
+    "release": "trixie",
+    "release_codename": "Trixie",
+    "release_title": "13"
+  },
+  "Fedora": {
+    "aliases": "fedora",
+    "items": {
+      "arm64": {
+        "id": "e10658419a8d50231037dc781c3155aa94180a8c7a74e5cac2a6b09eaa9342b7",
+        "image_location": "https://download.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/aarch64/images/Fedora-Cloud-Base-Generic-42-1.1.aarch64.qcow2",
+        "size": 533069824,
+        "version": "20250409"
+      },
+      "x86_64": {
+        "id": "e401a4db2e5e04d1967b6729774faa96da629bcf3ba90b67d8d9cce9906bec0f",
+        "image_location": "https://download.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-42-1.1.x86_64.qcow2",
+        "size": 532217856,
+        "version": "20250409"
+      }
+    },
+    "os": "Fedora",
+    "release": "",
+    "release_codename": "42",
+    "release_title": "42"
+  }
+}

include/multipass/cli/format_utils.h

@@ -38,13 +38,13 @@ namespace format
 static constexpr int col_buffer = 3;
 
 std::string status_string_for(const InstanceStatus& status);
-std::string image_string_for(const multipass::FindReply_AliasInfo& alias);
+std::string image_string_for(const std::string& remote_name, const std::string& alias);
 Formatter* formatter_for(const std::string& format);
 
 template <typename Container>
 Container sorted(const Container& items);
 
-void filter_aliases(google::protobuf::RepeatedPtrField<multipass::FindReply_AliasInfo>& aliases);
+void filter_aliases(google::protobuf::RepeatedPtrField<std::string>& aliases);
 
 // Computes the column width needed to display all the elements of a range [begin, end). get_width
 // is a function which takes as input the element in the range and returns its width in columns.
@@ -135,22 +135,3 @@ Container multipass::format::sorted(const Container& items)
 
     return ret;
 }
-
-namespace fmt
-{
-template <>
-struct formatter<multipass::FindReply_AliasInfo>
-{
-    template <typename ParseContext>
-    constexpr auto parse(ParseContext& ctx)
-    {
-        return ctx.begin();
-    }
-
-    template <typename FormatContext>
-    auto format(const multipass::FindReply_AliasInfo& a, FormatContext& ctx) const
-    {
-        return format_to(ctx.out(), "{}", a.alias());
-    }
-};
-} // namespace fmt

include/multipass/constants.h

@@ -27,7 +27,10 @@ namespace multipass
 constexpr auto client_name = "multipass";
 constexpr auto daemon_name = "multipassd";
 
+constexpr auto release_remote = "release";
+constexpr auto daily_remote = "daily";
 constexpr auto snapcraft_remote = "snapcraft";
+constexpr auto core_remote = "core";
 
 constexpr auto min_memory_size = "128M";
 constexpr auto min_disk_size = "512M";
@@ -44,6 +47,7 @@ constexpr auto home_automount_dir = "Home";
 
 constexpr auto multipass_storage_env_var = "MULTIPASS_STORAGE";
 constexpr auto driver_env_var = "MULTIPASS_VM_DRIVER";
+constexpr auto distributions_url_env_var = "MULTIPASS_DISTRIBUTIONS_URL";
 
 constexpr auto winterm_profile_guid =
     "{aaaa9e6d-1e09-4be6-b76c-82b4ba1885fb}"; // identifies the primary Multipass profile in Windows

include/multipass/exceptions/image_not_found_exception.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) Canonical, Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 3.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+#include <stdexcept>
+#include <string>
+
+#include <multipass/format.h>
+
+namespace multipass
+{
+class ImageNotFoundException : public std::runtime_error
+{
+public:
+    ImageNotFoundException(const std::string& hash)
+        : std::runtime_error{fmt::format("Image with hash \"{}\" not found", hash)}
+    {
+    }
+};
+} // namespace multipass

src/daemon/common_image_host.h renamed to include/multipass/image_host/base_image_host.h

@@ -17,7 +17,8 @@
 
 #pragma once
 
-#include "multipass/vm_image_host.h"
+#include <multipass/image_host/vm_image_host.h>
+#include <multipass/url_downloader.h>
 
 #include <QStringList>
 #include <QTimer>
@@ -27,12 +28,14 @@
 namespace multipass
 {
 
-class CommonVMImageHost : public VMImageHost
+class BaseVMImageHost : public VMImageHost
 {
 public:
+    BaseVMImageHost(URLDownloader* downloader);
+
     void for_each_entry_do(const Action& action) final;
     VMImageInfo info_for_full_hash(const std::string& full_hash) final;
-    void update_manifests(const bool is_force_update_from_network);
+    void update_manifests(const bool force_update);
 
 protected:
     void on_manifest_update_failure(const std::string& details);
@@ -41,7 +44,9 @@ class CommonVMImageHost : public VMImageHost
     virtual void for_each_entry_do_impl(const Action& action) = 0;
     virtual VMImageInfo info_for_full_hash_impl(const std::string& full_hash) = 0;
     virtual void clear() = 0;
-    virtual void fetch_manifests(const bool is_force_update_from_network) = 0;
+    virtual void fetch_manifests(const bool force_update) = 0;
+
+    URLDownloader* const url_downloader;
 };
 
 } // namespace multipass

src/daemon/custom_image_host.h renamed to include/multipass/image_host/custom_image_host.h

@@ -17,14 +17,12 @@
 
 #pragma once
 
-#include "common_image_host.h"
+#include <multipass/image_host/base_image_host.h>
+#include <multipass/vm_image_info.h>
 
 #include <QString>
 
-#include <memory>
 #include <string>
-#include <unordered_map>
-#include <utility>
 #include <vector>
 
 namespace multipass
@@ -39,10 +37,10 @@ struct CustomManifest
     CustomManifest(std::vector<VMImageInfo>&& images);
 };
 
-class CustomVMImageHost final : public CommonVMImageHost
+class CustomVMImageHost final : public BaseVMImageHost
 {
 public:
-    CustomVMImageHost(const QString& arch, URLDownloader* downloader);
+    CustomVMImageHost(URLDownloader* downloader);
 
     std::optional<VMImageInfo> info_for(const Query& query) override;
     std::vector<std::pair<std::string, VMImageInfo>> all_info_for(const Query& query) override;
@@ -53,13 +51,12 @@ class CustomVMImageHost final : public CommonVMImageHost
 private:
     void for_each_entry_do_impl(const Action& action) override;
     VMImageInfo info_for_full_hash_impl(const std::string& full_hash) override;
-    void fetch_manifests(const bool is_force_update_from_network) override;
+    void fetch_manifests(const bool force_update) override;
     void clear() override;
     CustomManifest* manifest_from(const std::string& remote_name);
 
     const QString arch;
-    URLDownloader* const url_downloader;
-    std::unordered_map<std::string, std::unique_ptr<CustomManifest>> custom_image_info;
-    std::vector<std::string> remotes;
+    std::pair<std::string, std::unique_ptr<CustomManifest>> manifest;
+    std::string remote;
 };
 } // namespace multipass

include/multipass/image_host/image_mutators.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) Canonical, Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 3.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#pragma once
+
+#include <multipass/vm_image_info.h>
+
+namespace multipass::image_mutators
+{
+bool snapcraft_mutator(VMImageInfo& info);
+bool core_mutator(VMImageInfo& info);
+bool release_mutator(VMImageInfo& info);
+} // namespace multipass::image_mutators