diff --git a/Makefile b/Makefile index 0b53b7d9f..4e8bcc6b3 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ ifneq ($(MAKECMDGOALS),rpm-dkms) ifneq ($(MAKECMDGOALS),srpm) ifneq ($(MAKECMDGOALS),deb) ifneq ($(MAKECMDGOALS),dsc) +ifneq ($(MAKECMDGOALS),upgrade) cd $@ && $(MAKE) $(MAKECMDGOALS) casadm: modules cd $@ && $(MAKE) $(MAKECMDGOALS) @@ -31,6 +32,7 @@ endif endif endif endif +endif archives: @tools/pckgen.sh $(PWD) tar zip @@ -49,3 +51,26 @@ deb: dsc: @tools/pckgen.sh $(PWD) dsc + +# In-flight upgrade: replace the installed release with this one and swap +# the running cas_cache for the fallback one built against the loaded cas_bd. +# The new cas_bd is loaded after reboot. +UPGRADE_DIR = $(PWD)/.upgrade + +upgrade: + @cd modules && $(MAKE) upgrade_check + @trap 'udevadm control --start-exec-queue' EXIT; \ + trap 'exit 1' INT TERM HUP; \ + set -e; \ + udevadm control --stop-exec-queue; \ + (cd modules && $(MAKE) upgrade_unload); \ + rm -rf $(UPGRADE_DIR); \ + (cd modules && $(MAKE) upgrade_preserve UPGRADE_DIR=$(UPGRADE_DIR)); \ + (cd utils && $(MAKE) upgrade_preserve UPGRADE_DIR=$(UPGRADE_DIR)); \ + (cd modules && $(MAKE) uninstall_files); \ + for dir in casadm utils extra; do (cd $$dir && $(MAKE) uninstall); done; \ + (cd modules && $(MAKE) install_files); \ + for dir in casadm utils extra; do (cd $$dir && $(MAKE) install); done; \ + (cd utils && $(MAKE) upgrade_restore UPGRADE_DIR=$(UPGRADE_DIR)); \ + (cd modules && $(MAKE) upgrade_load UPGRADE_DIR=$(UPGRADE_DIR)); \ + rm -rf $(UPGRADE_DIR) diff --git a/modules/Makefile b/modules/Makefile index ede1cfe74..c54210287 100644 --- a/modules/Makefile +++ b/modules/Makefile @@ -11,7 +11,9 @@ ifneq ($(M),) include $(M)/config.mk obj-y += cas_cache/ +ifneq ($(FALLBACK_BUILD),1) obj-y += cas_bd/ +endif # Otherwise we were called directly from the command # line; invoke the kernel build system. @@ -32,6 +34,28 @@ MODULES_DIR=/lib/modules/$(KERNEL_VERSION)/extra/block/opencas CACHE_MODULE = cas_cache DISK_MODULE = cas_bd +DISK_SYMVERS = $(DISK_MODULE).symvers + +# In-flight upgrade support. When FALLBACK_SYMVERS points at the symvers +# of the loaded cas_bd, a second cas_cache.ko is built against it. 'make +# upgrade' installs it under FALLBACK_INSTALL_DIR (outside /lib/modules/, so +# depmod ignores it) next to a copy of the loaded cas_bd, and loads it in +# place of the old cas_cache; after reboot the new cas_bd + cas_cache from +# the standard location take over. +FALLBACK_BUILD_DIR = $(PWD)/.fallback +FALLBACK_CACHE_KO = $(FALLBACK_BUILD_DIR)/cas_cache/$(CACHE_MODULE).ko +FALLBACK_BASE_SYMVERS = $(FALLBACK_BUILD_DIR)/$(DISK_SYMVERS) +FALLBACK_INSTALL_DIR ?= /lib/opencas/fallback/$(KERNEL_VERSION) + +# Directory holding the files of the loaded cas_bd: the standard location, +# or the fallback one if an upgrade was not followed by reboot +LOADED_DISK_DIR = $(shell loaded=$$(cat /sys/module/$(DISK_MODULE)/srcversion 2>/dev/null); \ + for dir in $(MODULES_DIR) $(FALLBACK_INSTALL_DIR); do \ + if [ -n "$$loaded" ] && [ -f $$dir/$(DISK_SYMVERS) ] && \ + [ "$$(modinfo -F srcversion $$dir/$(DISK_MODULE).ko 2>/dev/null)" = "$$loaded" ]; then \ + echo $$dir; break; \ + fi; \ + done) DEPMOD:=$(shell which depmod) RMMOD :=$(shell which rmmod) @@ -48,6 +72,22 @@ endif default: sync check_configured_kernel cd $(KERNEL_DIR) && $(MAKE) M=$(PWD) modules +ifneq ($(FALLBACK_SYMVERS),) + @test -f $(FALLBACK_SYMVERS) || { \ + echo "ERROR: FALLBACK_SYMVERS=$(FALLBACK_SYMVERS) not found" >&2; \ + exit 1; \ + } + @echo "=== Building fallback $(CACHE_MODULE) against $(FALLBACK_SYMVERS) ===" + @rm -rf $(FALLBACK_BUILD_DIR) + @mkdir -p $(FALLBACK_BUILD_DIR) + @find $(PWD) -mindepth 1 -maxdepth 1 ! -name .fallback \ + -exec cp -a {} $(FALLBACK_BUILD_DIR)/ \; + @cd $(KERNEL_DIR) && $(MAKE) M=$(FALLBACK_BUILD_DIR) FALLBACK_BUILD=1 clean + cd $(KERNEL_DIR) && $(MAKE) M=$(FALLBACK_BUILD_DIR) FALLBACK_BUILD=1 \ + METADATA_DIR=$(PWD)/../.metadata \ + KBUILD_EXTRA_SYMBOLS=$(FALLBACK_SYMVERS) modules + @cp $(FALLBACK_SYMVERS) $(FALLBACK_BASE_SYMVERS) +endif # generated_defines.h describes the kernel API './configure' probed. Compiling # it against another kernel produces code for an API that kernel may not have, @@ -65,35 +105,84 @@ check_configured_kernel: clean: cd $(KERNEL_DIR) && make M=$(PWD) clean + @rm -rf $(FALLBACK_BUILD_DIR) distclean: clean distsync @rm -f $(PWD)/generated_defines.h + @rm -f $(PWD)/$(DISK_SYMVERS) install: install_files @$(DEPMOD) - @$(MODPROBE) $(CACHE_MODULE) || ( \ - echo "See dmesg for more information" >&2 && \ - rm -f $(DESTDIR)$(MODULES_DIR)/$(CACHE_MODULE).ko && exit 1 \ - ) + @$(MODPROBE) $(CACHE_MODULE) || { \ + echo "See dmesg for more information" >&2; exit 1; \ + } install_files: @echo "Installing Open-CAS modules" @install -m 644 -D cas_cache/$(CACHE_MODULE).ko $(DESTDIR)$(MODULES_DIR)/$(CACHE_MODULE).ko @install -m 644 -D cas_bd/$(DISK_MODULE).ko $(DESTDIR)$(MODULES_DIR)/$(DISK_MODULE).ko + @grep 'cas_bd/$(DISK_MODULE)' Module.symvers > $(PWD)/$(DISK_SYMVERS) + @install -m 644 -D $(PWD)/$(DISK_SYMVERS) $(DESTDIR)$(MODULES_DIR)/$(DISK_SYMVERS) uninstall: - @echo "Uninstalling Open-CAS modules" + @echo "Unloading Open-CAS modules" $(call remove-module,$(CACHE_MODULE)) $(call remove-module,$(DISK_MODULE)) + @$(MAKE) uninstall_files +uninstall_files: + @echo "Uninstalling Open-CAS modules" $(call remove-file,$(DESTDIR)$(MODULES_DIR)/$(CACHE_MODULE).ko) $(call remove-file,$(DESTDIR)$(MODULES_DIR)/$(DISK_MODULE).ko) + $(call remove-file,$(DESTDIR)$(MODULES_DIR)/$(DISK_SYMVERS)) $(call remove-directory,$(DESTDIR)$(MODULES_DIR)) + @rm -rf $(DESTDIR)$(FALLBACK_INSTALL_DIR) + @$(DEPMOD) reinstall: uninstall install -.PHONY: all default check_configured_kernel clean distclean sync distsync install uninstall +# Steps of the in-flight upgrade, driven by 'make upgrade' in the top +# directory, which also passes UPGRADE_DIR to keep the preserved files in +upgrade_check: + @test -f $(FALLBACK_CACHE_KO) || { \ + echo "ERROR: Fallback $(CACHE_MODULE).ko not built, rebuild with" \ + "FALLBACK_SYMVERS=" >&2; \ + exit 1; \ + } + @test -n "$(LOADED_DISK_DIR)" || { \ + echo "ERROR: $(DISK_MODULE) not loaded or its installed files" \ + "do not match the loaded module" >&2; \ + exit 1; \ + } + @cmp -s $(FALLBACK_BASE_SYMVERS) $(LOADED_DISK_DIR)/$(DISK_SYMVERS) || { \ + echo "ERROR: Fallback $(CACHE_MODULE).ko was not built against" \ + "the loaded $(DISK_MODULE), rebuild with" \ + "FALLBACK_SYMVERS=$(LOADED_DISK_DIR)/$(DISK_SYMVERS)" >&2; \ + exit 1; \ + } + +upgrade_unload: + @echo "Unloading $(CACHE_MODULE) module" + @$(RMMOD) $(CACHE_MODULE) + +upgrade_preserve: + @echo "Preserving loaded $(DISK_MODULE)" + @install -m 644 -D $(LOADED_DISK_DIR)/$(DISK_MODULE).ko $(UPGRADE_DIR)/$(DISK_MODULE).ko + @install -m 644 -D $(LOADED_DISK_DIR)/$(DISK_SYMVERS) $(UPGRADE_DIR)/$(DISK_SYMVERS) + +upgrade_load: + @$(DEPMOD) + @echo "Installing fallback $(CACHE_MODULE) to $(FALLBACK_INSTALL_DIR)" + @install -m 644 -D $(UPGRADE_DIR)/$(DISK_MODULE).ko $(FALLBACK_INSTALL_DIR)/$(DISK_MODULE).ko + @install -m 644 -D $(UPGRADE_DIR)/$(DISK_SYMVERS) $(FALLBACK_INSTALL_DIR)/$(DISK_SYMVERS) + @install -m 644 -D $(FALLBACK_CACHE_KO) $(FALLBACK_INSTALL_DIR)/$(CACHE_MODULE).ko + @insmod $(FALLBACK_INSTALL_DIR)/$(CACHE_MODULE).ko || { \ + echo "See dmesg for more information" >&2; exit 1; \ + } + +.PHONY: all default check_configured_kernel clean distclean sync distsync install install_files \ + uninstall uninstall_files upgrade_check upgrade_unload upgrade_preserve upgrade_load endif diff --git a/modules/cas_bd/Makefile b/modules/cas_bd/Makefile index b7aacd6e3..65af26bd8 100644 --- a/modules/cas_bd/Makefile +++ b/modules/cas_bd/Makefile @@ -7,5 +7,5 @@ include $(M)/config.mk obj-m := cas_bd.o -cas_bd-c = $(shell find $(M)/cas_bd -name \*.c) +cas_bd-c = $(shell find $(M)/cas_bd -name \*.c ! -name \*.mod.c) cas_bd-objs = $(patsubst $(M)/cas_bd/%.c,%.o,$(cas_bd-c)) diff --git a/modules/cas_cache/Makefile b/modules/cas_cache/Makefile index 00d442653..d9a26d42d 100644 --- a/modules/cas_cache/Makefile +++ b/modules/cas_cache/Makefile @@ -6,5 +6,5 @@ include $(M)/config.mk obj-m := cas_cache.o -cas_cache-c = $(shell find $(M)/cas_cache -name \*.c) +cas_cache-c = $(shell find $(M)/cas_cache -name \*.c ! -name \*.mod.c) cas_cache-objs = $(patsubst $(M)/cas_cache/%.c,%.o,$(cas_cache-c)) diff --git a/modules/config.mk b/modules/config.mk index 0924235e1..aa1cb1f38 100644 --- a/modules/config.mk +++ b/modules/config.mk @@ -5,8 +5,9 @@ # SPDX-License-Identifier: BSD-3-Clause # -VERSION_FILE := $(M)/../.metadata/cas_version -BUILD_CONFIG_FILE := $(M)/../.metadata/cas_config +METADATA_DIR ?= $(M)/../.metadata +VERSION_FILE := $(METADATA_DIR)/cas_version +BUILD_CONFIG_FILE := $(METADATA_DIR)/cas_config check_cflag=$(shell echo "" | \ gcc -c -xc ${1} -o /dev/null - 2>/dev/null; \ diff --git a/test/functional/tests/upgrade/__init__.py b/test/functional/tests/upgrade/__init__.py new file mode 100644 index 000000000..ccf889cf8 --- /dev/null +++ b/test/functional/tests/upgrade/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright(c) 2026 Unvertical +# SPDX-License-Identifier: BSD-3-Clause +# diff --git a/test/functional/tests/upgrade/test_make_upgrade.py b/test/functional/tests/upgrade/test_make_upgrade.py new file mode 100644 index 000000000..499fa18e8 --- /dev/null +++ b/test/functional/tests/upgrade/test_make_upgrade.py @@ -0,0 +1,542 @@ +# +# Copyright(c) 2026 Unvertical +# SPDX-License-Identifier: BSD-3-Clause +# + +import os +import time +from collections import namedtuple +from datetime import timedelta + +import pytest + +from api.cas import casadm +from api.cas.cache_config import CacheMode, CacheStatus +from api.cas.installer import clean_opencas_repo, rsync_opencas_sources +from core.test_run import TestRun +from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan +from test_tools.fs_tools import remove +from type_def.size import Size, Unit + + +# Each variant is a separate copy of the sources, built once per test module +# and reused by its tests. Caches only load under the MAIN.MAJOR.MINOR that +# created them, so 'next' stays within the release and is made newer by its +# build number instead, which './configure' derives from git history. That +# makes installing 'next' over 'current' an upgrade and the other way around +# a downgrade. +upgrade_root = "/var/tmp/cas_make_upgrade_test" +variants = { + "current": None, + "next": "9999", +} +directions = { + "upgrade": ("current", "next"), + "downgrade": ("next", "current"), +} +disconnect_modes = { + "default": {}, + "no-flush": {"no_flush": True}, + "pass-through": {"pass_through": True}, +} + +# Shell expressions, to be used inside commands +modules_dir = "/lib/modules/$(uname -r)/extra/block/opencas" +fallback_dir = "/lib/opencas/fallback/$(uname -r)" +installed_symvers = f"{modules_dir}/cas_bd.symvers" + +config_file = "/etc/opencas/opencas.conf" +config_marker = "# make upgrade test marker" + +# Checksums of every installed CAS file, to tell whether anything changed +installed_files_checksums = ( + f"find {modules_dir} /lib/opencas /usr/lib/opencas /etc/opencas /usr/sbin/casadm " + f"/var/lib/opencas -type f -exec md5sum {{}} + 2>/dev/null | sort" +) + +data_file = "/var/tmp/cas_make_upgrade_data" +fio_log = "/var/tmp/cas_make_upgrade_fio.log" + +CACHE_SIZE = Size(4, Unit.GibiByte) +CORE_SIZE = Size(512, Unit.MebiByte) + +build_timeout = timedelta(minutes=30) + +Variant = namedtuple("Variant", ["path", "version"]) + + +@pytest.fixture(scope="module", autouse=True) +def fresh_builds(): + """Variants are built once for the whole module, from the sources under test.""" + rsync_opencas_sources() + clean_opencas_repo() + remove(upgrade_root, recursive=True, force=True, ignore_errors=True) + yield + + +@pytest.fixture(autouse=True) +def installation_restored(): + """Whatever a test leaves installed, hand the next one a regular installation.""" + yield + restore_installation() + + +@pytest.mark.parametrize("direction", directions.keys()) +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.nand, DiskType.optane])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_make_upgrade(direction): + """ + title: Upgrade and downgrade from sources with 'make upgrade'. + description: | + Install one version, stop a cache holding dirty data, switch to the other + version with 'make upgrade' and load the cache again. The new cas_cache + has to be running on top of the cas_bd of the version replaced. + pass_criteria: + - make upgrade succeeds + - the new version is installed, including casadm + - the loaded cas_cache is the new one, the loaded cas_bd the old one + - the fallback modules and symvers are installed + - user configuration and enabled services are preserved + - the cache loads with its dirty data and the core data is intact + """ + with TestRun.step("Build both versions"): + old, new = (build_variant(name) for name in directions[direction]) + + with TestRun.step(f"Install {old.version}"): + TestRun.executor.run("rmmod cas_cache; rmmod cas_bd") + TestRun.executor.run_expect_success(f"cd {old.path} && make install") + TestRun.executor.run_expect_success(f"echo '{config_marker}' >> {config_file}") + + with TestRun.step("Start a WB cache and write data to its core"): + cache_dev, core_dev = TestRun.disks["cache"], TestRun.disks["core"] + cache_dev.create_partitions([CACHE_SIZE]) + core_dev.create_partitions([CORE_SIZE]) + cache_part = cache_dev.partitions[0] + + cache = casadm.start_cache(cache_part, cache_mode=CacheMode.WB, force=True) + core = cache.add_core(core_dev.partitions[0]) + TestRun.executor.run_expect_success( + f"head -c {int(CORE_SIZE.get_value())} /dev/urandom > {data_file}" + ) + TestRun.executor.run_expect_success(f"dd if={data_file} of={core.path} bs=1M conv=fsync") + + if cache.get_dirty_blocks().get_value() == 0: + TestRun.fail("Cache holds no dirty data to carry across the upgrade") + + with TestRun.step("Stop the cache without flushing"): + cache.stop(no_data_flush=True) + + with TestRun.step(f"Build the fallback cas_cache of {new.version}"): + TestRun.executor.run_expect_success( + f"cd {new.path} && make -j$(nproc) FALLBACK_SYMVERS={installed_symvers}", + build_timeout, + ) + + with TestRun.step(f"Upgrade to {new.version}"): + TestRun.executor.run_expect_success(f"cd {new.path} && make upgrade", build_timeout) + + with TestRun.step("Check the installation"): + check_upgraded(old, new) + + with TestRun.step("Load the cache"): + cache = casadm.load_cache(cache_part) + + if cache.get_dirty_blocks().get_value() == 0: + TestRun.fail("Dirty data was lost across the upgrade") + + with TestRun.step("Check the core data"): + TestRun.executor.run_expect_success("echo 3 > /proc/sys/vm/drop_caches") + TestRun.executor.run_expect_success( + f"cmp -n {int(CORE_SIZE.get_value())} {data_file} {core.path}" + ) + + with TestRun.step("Stop the cache"): + cache.stop() + + +@pytest.mark.parametrize("disconnect_mode", disconnect_modes.keys()) +@pytest.mark.parametrize("direction", directions.keys()) +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.nand, DiskType.optane])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_make_upgrade_under_io(direction, disconnect_mode): + """ + title: In-flight upgrade and downgrade with 'make upgrade' under I/O. + description: | + Disconnect a running WB cache while I/O keeps being issued to its + exported objects, replace CAS with 'make upgrade', and connect the + cache with the new cas_cache running on top of the old cas_bd. + pass_criteria: + - make upgrade succeeds with the cache disconnected + - the loaded cas_cache is the new one, the loaded cas_bd the old one + - the cache connects with the new version + - I/O issued throughout reads back what it wrote + - dirty data is kept when disconnecting without flush, flushed otherwise + - the core data is intact + """ + with TestRun.step("Build both versions"): + old, new = (build_variant(name) for name in directions[direction]) + + with TestRun.step(f"Install {old.version}"): + TestRun.executor.run("rmmod cas_cache; rmmod cas_bd") + TestRun.executor.run_expect_success(f"cd {old.path} && make install") + TestRun.executor.run_expect_success(f"echo '{config_marker}' >> {config_file}") + + with TestRun.step(f"Build the fallback cas_cache of {new.version}"): + TestRun.executor.run_expect_success( + f"cd {new.path} && make -j$(nproc) FALLBACK_SYMVERS={installed_symvers}", + build_timeout, + ) + + with TestRun.step("Start a WB cache with two cores"): + cache_dev, core_dev = TestRun.disks["cache"], TestRun.disks["core"] + cache_dev.create_partitions([CACHE_SIZE]) + core_dev.create_partitions([CORE_SIZE] * 2) + cache_part = cache_dev.partitions[0] + + cache = casadm.start_cache(cache_part, cache_mode=CacheMode.WB, force=True) + data_core, io_core = [cache.add_core(part) for part in core_dev.partitions] + + with TestRun.step("Write data to the first core"): + TestRun.executor.run_expect_success( + f"head -c {int(CORE_SIZE.get_value())} /dev/urandom > {data_file}" + ) + TestRun.executor.run_expect_success( + f"dd if={data_file} of={data_core.path} bs=1M conv=fsync" + ) + + with TestRun.step("Start verified I/O on the second core"): + fio_pid = TestRun.executor.run_in_background( + f"fio --name=live --filename={io_core.path} --rw=randrw --bs=4k --direct=1 " + f"--ioengine=libaio --iodepth=8 --verify=crc32c --verify_backlog=1024 " + f"--verify_fatal=1 --verify_state_save=0 --time_based --runtime=7200", + stdout_redirect_path=fio_log, + stderr_redirect_path=fio_log, + ) + time.sleep(10) + + with TestRun.step(f"Disconnect the cache ({disconnect_mode})"): + casadm.disconnect_cache(cache.cache_id, **disconnect_modes[disconnect_mode]) + + with TestRun.step(f"Upgrade to {new.version}"): + TestRun.executor.run_expect_success(f"cd {new.path} && make upgrade", build_timeout) + + with TestRun.step("Connect the cache"): + cache = casadm.connect_cache(cache_part) + + with TestRun.step("Stop the I/O"): + time.sleep(10) + TestRun.executor.run(f"kill -INT {fio_pid}") + TestRun.executor.wait_cmd_finish(fio_pid) + TestRun.executor.run_expect_success(f"grep -q 'err= 0' {fio_log}") + + with TestRun.step("Check the installation"): + check_upgraded(old, new) + + with TestRun.step("Check the dirty data of the first core"): + # the second core keeps getting new dirty data, the first one does not + dirty = data_core.get_dirty_blocks().get_value() + if disconnect_mode == "no-flush" and dirty == 0: + TestRun.fail("Dirty data was lost across the upgrade") + if disconnect_mode != "no-flush" and dirty != 0: + TestRun.fail(f"Dirty data was not flushed on disconnect: {dirty}") + + with TestRun.step("Check the data of the first core"): + TestRun.executor.run_expect_success("echo 3 > /proc/sys/vm/drop_caches") + TestRun.executor.run_expect_success( + f"cmp -n {int(CORE_SIZE.get_value())} {data_file} {data_core.path}" + ) + + with TestRun.step("Stop the cache"): + cache.stop() + + +@pytest.mark.parametrize("direction", directions.keys()) +def test_make_upgrade_fallback_not_built(direction): + """ + title: 'make upgrade' requires the fallback cas_cache. + description: | + Run 'make upgrade' from sources built the regular way, without the + fallback cas_cache for the loaded cas_bd. + pass_criteria: + - make upgrade fails asking to build the fallback + - installed files and loaded modules are left as they were + """ + with TestRun.step("Build both versions"): + old, new = (build_variant(name) for name in directions[direction]) + TestRun.executor.run_expect_success(f"rm -rf {new.path}/modules/.fallback") + + with TestRun.step(f"Install {old.version}"): + TestRun.executor.run("rmmod cas_cache; rmmod cas_bd") + TestRun.executor.run_expect_success(f"cd {old.path} && make install") + files_before = TestRun.executor.run(installed_files_checksums).stdout + + with TestRun.step(f"Upgrade to {new.version}"): + output = TestRun.executor.run(f"cd {new.path} && make upgrade") + if output.exit_code == 0 or "Fallback cas_cache.ko not built" not in output.stderr: + TestRun.fail( + f"make upgrade was not refused as expected:\n{output.stdout}\n{output.stderr}" + ) + + with TestRun.step("Check nothing changed"): + check_not_upgraded(old, new, files_before) + TestRun.executor.run_expect_success( + f'test "$(cat /sys/module/cas_cache/version /sys/module/cas_bd/version | sort -u)" ' + f"= '{old.version}'" + ) + + +def test_make_upgrade_fallback_mismatch(): + """ + title: 'make upgrade' requires the fallback built for the loaded cas_bd. + description: | + Build the fallback cas_cache against symvers that do not match the loaded + cas_bd and run 'make upgrade'. + pass_criteria: + - make upgrade fails before changing anything + - installed files and loaded modules are left as they were + """ + foreign_symvers = "/var/tmp/cas_make_upgrade_foreign.symvers" + + with TestRun.step("Build both versions"): + old, new = (build_variant(name) for name in directions["upgrade"]) + + with TestRun.step(f"Install {old.version}"): + TestRun.executor.run("rmmod cas_cache; rmmod cas_bd") + TestRun.executor.run_expect_success(f"cd {old.path} && make install") + files_before = TestRun.executor.run(installed_files_checksums).stdout + + with TestRun.step(f"Build the fallback cas_cache of {new.version} for a different cas_bd"): + TestRun.executor.run_expect_success( + f"sed '1s/^0x[0-9a-f]*/0xdeadbeef/' {installed_symvers} > {foreign_symvers}" + ) + TestRun.executor.run_expect_success( + f"cd {new.path} && make -j$(nproc) FALLBACK_SYMVERS={foreign_symvers}", + build_timeout, + ) + + with TestRun.step(f"Upgrade to {new.version}"): + output = TestRun.executor.run(f"cd {new.path} && make upgrade") + if output.exit_code == 0 or "was not built against the loaded" not in output.stderr: + TestRun.fail( + f"make upgrade was not refused as expected:\n{output.stdout}\n{output.stderr}" + ) + + with TestRun.step("Check nothing changed"): + check_not_upgraded(old, new, files_before) + TestRun.executor.run_expect_success( + f'test "$(cat /sys/module/cas_cache/version /sys/module/cas_bd/version | sort -u)" ' + f"= '{old.version}'" + ) + + +@pytest.mark.parametrize( + "unloaded, expected_error", + [ + ("cas_cache", "Module cas_cache is not currently loaded"), + ("cas_cache cas_bd", "cas_bd not loaded"), + ], +) +def test_make_upgrade_modules_not_loaded(unloaded, expected_error): + """ + title: 'make upgrade' requires the old modules to be loaded. + description: | + Unload cas_cache, or both CAS modules, and run 'make upgrade'. Without + cas_cache there is nothing to replace, and without cas_bd nothing to + load the fallback cas_cache on top of. + pass_criteria: + - make upgrade fails before changing anything + - installed files are left as they were, and no module gets loaded + """ + with TestRun.step("Build both versions"): + old, new = (build_variant(name) for name in directions["upgrade"]) + + with TestRun.step(f"Install {old.version}"): + TestRun.executor.run("rmmod cas_cache; rmmod cas_bd") + TestRun.executor.run_expect_success(f"cd {old.path} && make install") + + with TestRun.step(f"Build the fallback cas_cache of {new.version}"): + TestRun.executor.run_expect_success( + f"cd {new.path} && make -j$(nproc) FALLBACK_SYMVERS={installed_symvers}", + build_timeout, + ) + + with TestRun.step(f"Unload {unloaded}"): + TestRun.executor.run_expect_success(f"rmmod {unloaded}") + files_before = TestRun.executor.run(installed_files_checksums).stdout + + with TestRun.step(f"Upgrade to {new.version}"): + output = TestRun.executor.run(f"cd {new.path} && make upgrade") + if output.exit_code == 0 or expected_error not in output.stderr: + TestRun.fail( + f"make upgrade was not refused as expected:\n{output.stdout}\n{output.stderr}" + ) + + with TestRun.step("Check nothing changed"): + check_not_upgraded(old, new, files_before) + for module in unloaded.split(): + if TestRun.executor.run(f"test -e /sys/module/{module}").exit_code == 0: + TestRun.fail(f"{module} got loaded by a refused upgrade") + + +@pytest.mark.require_disk("cache", DiskTypeSet([DiskType.nand, DiskType.optane])) +@pytest.mark.require_disk("core", DiskTypeLowerThan("cache")) +def test_make_upgrade_cache_running(): + """ + title: 'make upgrade' does not touch running caches. + description: | + Run 'make upgrade' while a cache is running. cas_cache cannot be + unloaded then, so the upgrade has to stop before changing anything. + pass_criteria: + - make upgrade fails before changing anything + - installed files and loaded modules are left as they were + - the cache keeps running with its dirty data and the core data is intact + """ + with TestRun.step("Build both versions"): + old, new = (build_variant(name) for name in directions["upgrade"]) + + with TestRun.step(f"Install {old.version}"): + TestRun.executor.run("rmmod cas_cache; rmmod cas_bd") + TestRun.executor.run_expect_success(f"cd {old.path} && make install") + + with TestRun.step(f"Build the fallback cas_cache of {new.version}"): + TestRun.executor.run_expect_success( + f"cd {new.path} && make -j$(nproc) FALLBACK_SYMVERS={installed_symvers}", + build_timeout, + ) + + with TestRun.step("Start a WB cache and write data to its core"): + cache_dev, core_dev = TestRun.disks["cache"], TestRun.disks["core"] + cache_dev.create_partitions([CACHE_SIZE]) + core_dev.create_partitions([CORE_SIZE]) + + cache = casadm.start_cache(cache_dev.partitions[0], cache_mode=CacheMode.WB, force=True) + core = cache.add_core(core_dev.partitions[0]) + TestRun.executor.run_expect_success( + f"head -c {int(CORE_SIZE.get_value())} /dev/urandom > {data_file}" + ) + TestRun.executor.run_expect_success(f"dd if={data_file} of={core.path} bs=1M conv=fsync") + files_before = TestRun.executor.run(installed_files_checksums).stdout + + with TestRun.step(f"Upgrade to {new.version}"): + output = TestRun.executor.run(f"cd {new.path} && make upgrade") + if output.exit_code == 0 or "Module cas_cache is in use" not in output.stderr: + TestRun.fail( + f"make upgrade was not refused as expected:\n{output.stdout}\n{output.stderr}" + ) + + with TestRun.step("Check nothing changed"): + check_not_upgraded(old, new, files_before) + TestRun.executor.run_expect_success( + f'test "$(cat /sys/module/cas_cache/version /sys/module/cas_bd/version | sort -u)" ' + f"= '{old.version}'" + ) + + with TestRun.step("Check the cache"): + if cache.get_status() != CacheStatus.running: + TestRun.fail(f"Cache is {cache.get_status()} after the refused upgrade") + if cache.get_dirty_blocks().get_value() == 0: + TestRun.fail("Cache lost its dirty data") + + TestRun.executor.run_expect_success("echo 3 > /proc/sys/vm/drop_caches") + TestRun.executor.run_expect_success( + f"cmp -n {int(CORE_SIZE.get_value())} {data_file} {core.path}" + ) + + with TestRun.step("Stop the cache"): + cache.stop() + + +def build_variant(name: str): + """Build a variant with the regular build, unless an earlier test did.""" + path = os.path.join(upgrade_root, name) + + if TestRun.executor.run(f"test -f {path}/modules/cas_cache/cas_cache.ko").exit_code != 0: + TestRun.LOGGER.info(f"Building the {name} variant in {path}") + remove(path, recursive=True, force=True, ignore_errors=True) + TestRun.executor.run_expect_success( + f"mkdir -p {path} && tar -C {TestRun.usr.working_dir} --exclude=./test -cf - . " + f"| tar -C {path} --no-same-owner -xf -" + ) + TestRun.executor.run_expect_success(f"cd {path} && ./configure", build_timeout) + + build = variants[name] + if build: + TestRun.executor.run_expect_success( + f"sed -i -e 's/^CAS_VERSION_BUILD=.*/CAS_VERSION_BUILD={build}/' " + f"-e 's/^\\(CAS_VERSION=[0-9]*\\.[0-9]*\\.[0-9]*\\.\\)[0-9]*/\\1{build}/' " + f"{path}/.metadata/cas_version" + ) + + TestRun.executor.run_expect_success(f"cd {path} && make -j$(nproc)", build_timeout) + + version = TestRun.executor.run_expect_success( + f"sed -n 's/^CAS_VERSION=//p' {path}/.metadata/cas_version" + ).stdout.strip() + return Variant(path, version) + + +def check_upgraded(old: Variant, new: Variant): + """State after a successful upgrade: new files, fallback cas_cache on the old cas_bd.""" + expected = { + "cat /sys/module/cas_cache/version": new.version, + "cat /sys/module/cas_bd/version": old.version, + f"modinfo -F version {modules_dir}/cas_cache.ko": new.version, + f"modinfo -F version {modules_dir}/cas_bd.ko": new.version, + f"modinfo -F version {fallback_dir}/cas_cache.ko": new.version, + f"modinfo -F version {fallback_dir}/cas_bd.ko": old.version, + "casadm -V -o csv | sed -n 's/^CAS CLI Utility,//p'": new.version, + } + for command, version in expected.items(): + actual = TestRun.executor.run(command).stdout.strip() + if actual != version: + TestRun.fail(f"'{command}' reports {actual}, expected {version}") + + TestRun.executor.run_expect_success(f"test -f {fallback_dir}/cas_bd.symvers") + TestRun.executor.run_expect_success(f"grep -qxF '{config_marker}' {config_file}") + TestRun.executor.run_expect_success( + "systemctl is-enabled --quiet open-cas.service open-cas-shutdown.service" + ) + TestRun.executor.run_expect_success(f"test ! -e {new.path}/.upgrade") + + # udev has to be processing events again + TestRun.executor.run("udevadm trigger --action=change /sys/class/mem/null") + TestRun.executor.run_expect_success("udevadm settle --timeout=10") + + +def check_not_upgraded(old: Variant, new: Variant, files_before: str): + """A refused upgrade must leave the installation as it was.""" + if TestRun.executor.run(installed_files_checksums).stdout != files_before: + TestRun.fail("A refused upgrade changed the installed files") + + TestRun.executor.run_expect_success( + f"modinfo -F version {modules_dir}/cas_cache.ko | grep -qx '{old.version}'" + ) + TestRun.executor.run_expect_success(f"test ! -e {new.path}/.upgrade") + + # udev has to be processing events again + TestRun.executor.run("udevadm trigger --action=change /sys/class/mem/null") + TestRun.executor.run_expect_success("udevadm settle --timeout=10") + + +def restore_installation(): + """Leave the DUT with the sources under test installed the regular way.""" + TestRun.executor.run("pkill -KILL -x fio") + TestRun.executor.run("udevadm control --start-exec-queue") + try: + casadm.stop_all_caches() + except Exception as e: + TestRun.LOGGER.warning(f"Failed to stop caches: {e}") + + # exported objects of disconnected caches stay with cas_bd + TestRun.executor.run( + "test -e /sys/module/cas_bd/delete && for dev in $(ls /dev | grep '^cas[0-9]'); do " + "echo $dev > /sys/module/cas_bd/delete; done" + ) + TestRun.executor.run(f"rmmod cas_cache; rmmod cas_bd; rm -rf {fallback_dir}") + TestRun.executor.run(f"rm -f {data_file} {fio_log} /var/tmp/cas_make_upgrade_foreign.symvers") + + current = os.path.join(upgrade_root, "current") + if TestRun.executor.run(f"test -f {current}/modules/cas_cache/cas_cache.ko").exit_code == 0: + output = TestRun.executor.run(f"cd {current} && rm -rf .upgrade && make install") + if output.exit_code != 0: + TestRun.LOGGER.error(f"Failed to reinstall CAS:\n{output.stdout}\n{output.stderr}") diff --git a/utils/Makefile b/utils/Makefile index 9059183a0..ccc6f4317 100644 --- a/utils/Makefile +++ b/utils/Makefile @@ -108,4 +108,25 @@ uninstall: $(call remove-file,$(DESTDIR)$(SYSTEMD_DIR)/open-cas.service) $(call remove-file,$(DESTDIR)$(SYSTEMD_DIR)/../system-shutdown/open-cas.shutdown) -.PHONY: install uninstall clean distclean +# Steps of the in-flight upgrade, driven by 'make upgrade' in the top +# directory. User configuration is preserved across uninstall/install; +# this is the place to migrate it if the format ever changes. +CONFIG_FILES = /etc/opencas/opencas.conf /etc/opencas/ioclass-config.csv + +upgrade_preserve: + @echo "Preserving Open-CAS configuration" + @for file in $(CONFIG_FILES); do \ + if [ -f $$file ]; then \ + install -m 644 -D $$file $(UPGRADE_DIR)$$file || exit 1; \ + fi; \ + done + +upgrade_restore: + @echo "Restoring Open-CAS configuration" + @for file in $(CONFIG_FILES); do \ + if [ -f $(UPGRADE_DIR)$$file ]; then \ + install -m 644 -D $(UPGRADE_DIR)$$file $$file || exit 1; \ + fi; \ + done + +.PHONY: install uninstall clean distclean upgrade_preserve upgrade_restore