diff --git a/src/client/pydaos/raw/daos_api.py b/src/client/pydaos/raw/daos_api.py index 3908168cfb4..922723e7942 100644 --- a/src/client/pydaos/raw/daos_api.py +++ b/src/client/pydaos/raw/daos_api.py @@ -874,6 +874,172 @@ def punch_array(self, dkey, akey, rec_idx, rec_count, raise DaosApiError("Array punch returned non-zero. RC: {0}" .format(ret)) + def insert_recx(self, dkey, akey, rec_size, rx_idx, rx_nr, data, + txn=daos_cref.DAOS_TX_NONE): + """Update one record extent from a single contiguous buffer. + + insert_array describes an extent with one scatter gather entry per record, which + caps the extent at the scatter gather list limit. This uses a single entry for + the whole extent, so a byte granular extent can span an erasure coded stripe. + + dkey --1st level key for the array value + akey --2nd level key for the array value + rec_size --size in bytes of a single record + rx_idx --index of the first record + rx_nr --how many records + data --buffer holding rx_nr * rec_size bytes + txn --which transaction to write in. + Default is independent transaction (DAOS_TX_NONE) + """ + buf_len = rx_nr * rec_size + + extent = daos_cref.Extent() + extent.rx_idx = rx_idx + extent.rx_nr = rx_nr + + self.iod.iod_name.iov_buf = ctypes.cast(akey, ctypes.c_void_p) + self.iod.iod_name.iov_buf_len = ctypes.sizeof(akey) + self.iod.iod_name.iov_len = ctypes.sizeof(akey) + self.iod.iod_type = 2 + self.iod.iod_size = rec_size + self.iod.iod_flags = 0 + self.iod.iod_nr = 1 + self.iod.iod_recxs = ctypes.pointer(extent) + + sgl_iov = daos_cref.IOV() + sgl_iov.iov_len = buf_len + sgl_iov.iov_buf_len = buf_len + sgl_iov.iov_buf = ctypes.cast(data, ctypes.c_void_p) + self.sgl.sg_iovs = ctypes.pointer(sgl_iov) + self.sgl.sg_nr = 1 + self.sgl.sg_nr_out = 0 + + dkey_iov = daos_cref.IOV() + dkey_iov.iov_buf = ctypes.cast(dkey, ctypes.c_void_p) + dkey_iov.iov_buf_len = ctypes.sizeof(dkey) + dkey_iov.iov_len = ctypes.sizeof(dkey) + + func = self.context.get_function('update-obj') + + ret = func(self.obj.obj_handle, txn, 0, ctypes.byref(dkey_iov), + 1, ctypes.byref(self.iod), ctypes.byref(self.sgl), None) + if ret != 0: + raise DaosApiError("Recx update returned non-zero. RC: {0}" + .format(ret)) + + def fetch_recx(self, dkey, akey, rec_size, rx_idx, rx_nr, + txn=daos_cref.DAOS_TX_NONE): + """Fetch one record extent into a single contiguous buffer. + + dkey --1st level key for the array value + akey --2nd level key for the array value + rec_size --size in bytes of a single record + rx_idx --index of the first record + rx_nr --how many records + txn --which transaction to read from. + Default is independent transaction (DAOS_TX_NONE) + + Returns: + bytes: rx_nr * rec_size bytes. The fetch leaves holes untouched, and the + buffer starts zeroed, so a hole reads back as zeros. + + """ + data, _ = self._fetch_recx(dkey, akey, rec_size, rx_idx, rx_nr, 0, txn) + return data + + def fetch_recx_map(self, dkey, akey, rec_size, rx_idx, rx_nr, max_extents=64, + txn=daos_cref.DAOS_TX_NONE): + """Fetch one record extent and the io map describing what really holds data. + + A hole and a range someone wrote zeros over read back the same, so the io map + is the only way to tell them apart. + + dkey --1st level key for the array value + akey --2nd level key for the array value + rec_size --size in bytes of a single record + rx_idx --index of the first record + rx_nr --how many records + max_extents --how many extents the io map can hold. Raises if the server has + more to report, since a truncated map would silently under + report what is there + txn --which transaction to read from. + Default is independent transaction (DAOS_TX_NONE) + + Returns: + tuple: (bytes, extents), where extents is a list of (rx_idx, rx_nr) + covering only the records that really hold data. + + """ + return self._fetch_recx(dkey, akey, rec_size, rx_idx, rx_nr, max_extents, txn) + + def _fetch_recx(self, dkey, akey, rec_size, rx_idx, rx_nr, max_extents, txn): + """Fetch one record extent, optionally asking for the io map. + + See fetch_recx and fetch_recx_map. + + Returns: + tuple: (bytes, extents), extents is None when max_extents is 0. + + """ + buf_len = rx_nr * rec_size + buf = ctypes.create_string_buffer(buf_len) + + extent = daos_cref.Extent() + extent.rx_idx = rx_idx + extent.rx_nr = rx_nr + + self.iod.iod_name.iov_buf = ctypes.cast(akey, ctypes.c_void_p) + self.iod.iod_name.iov_buf_len = ctypes.sizeof(akey) + self.iod.iod_name.iov_len = ctypes.sizeof(akey) + self.iod.iod_type = 2 + self.iod.iod_size = rec_size + self.iod.iod_flags = 0 + self.iod.iod_nr = 1 + self.iod.iod_recxs = ctypes.pointer(extent) + + sgl_iov = daos_cref.IOV() + sgl_iov.iov_len = buf_len + sgl_iov.iov_buf_len = buf_len + sgl_iov.iov_buf = ctypes.cast(buf, ctypes.c_void_p) + self.sgl.sg_iovs = ctypes.pointer(sgl_iov) + self.sgl.sg_nr = 1 + self.sgl.sg_nr_out = 0 + + dkey_iov = daos_cref.IOV() + dkey_iov.iov_buf = ctypes.cast(dkey, ctypes.c_void_p) + dkey_iov.iov_buf_len = ctypes.sizeof(dkey) + dkey_iov.iov_len = ctypes.sizeof(dkey) + + iom_ptr = None + iom = None + if max_extents: + iom = daos_cref.DaosIOMap() + iom_recxs = (daos_cref.Extent * max_extents)() + iom.iom_flags = daos_cref.DAOS_IOMF_DETAIL + iom.iom_nr = max_extents + iom.iom_recxs = ctypes.cast(iom_recxs, ctypes.POINTER(daos_cref.Extent)) + iom_ptr = ctypes.byref(iom) + + func = self.context.get_function('fetch-obj') + + ret = func(self.obj.obj_handle, txn, 0, ctypes.byref(dkey_iov), 1, + ctypes.byref(self.iod), ctypes.byref(self.sgl), iom_ptr, None) + if ret != 0: + raise DaosApiError("Recx fetch returned non-zero. RC: {0}" + .format(ret)) + + data = ctypes.string_at(buf, buf_len) + if iom is None: + return data, None + + if iom.iom_nr_out > iom.iom_nr: + raise DaosApiError( + "Io map holds {0} extents, need room for {1}".format( + iom.iom_nr, iom.iom_nr_out)) + + return data, [(iom.iom_recxs[i].rx_idx, iom.iom_recxs[i].rx_nr) + for i in range(iom.iom_nr_out)] + def fetch_array(self, dkey, akey, rec_count, rec_size, txn=daos_cref.DAOS_TX_NONE): """Retrieve an array data from a dkey/akey pair. diff --git a/src/client/pydaos/raw/daos_cref.py b/src/client/pydaos/raw/daos_cref.py index f86faaa937d..3e83407266b 100644 --- a/src/client/pydaos/raw/daos_cref.py +++ b/src/client/pydaos/raw/daos_cref.py @@ -209,6 +209,22 @@ class DaosIODescriptor(ctypes.Structure): ("iod_recxs", ctypes.POINTER(Extent))] +class DaosIOMap(ctypes.Structure): + """Represents struct: daos_iom_t""" + _fields_ = [("iom_type", ctypes.c_int), # enum + ("iom_nr", ctypes.c_uint32), + ("iom_nr_out", ctypes.c_uint32), + ("iom_flags", ctypes.c_uint32), + ("iom_size", ctypes.c_uint64), + ("iom_recx_lo", Extent), + ("iom_recx_hi", Extent), + ("iom_recxs", ctypes.POINTER(Extent))] + + +# daos_iom_t iom_flags, ask for the full list of extents rather than just lo/hi +DAOS_IOMF_DETAIL = 0x1 + + class Anchor(ctypes.Structure): """ Class to represent a C daos_anchor_t struct. """ _fields_ = [('da_type', ctypes.c_uint16), diff --git a/src/tests/ftest/datamover/obj_ec.py b/src/tests/ftest/datamover/obj_ec.py index cdae56f872d..99489e87887 100644 --- a/src/tests/ftest/datamover/obj_ec.py +++ b/src/tests/ftest/datamover/obj_ec.py @@ -11,15 +11,15 @@ class DmvrObjEcTest(DataMoverTestBase): # pylint: disable=too-many-ancestors - """Object Data Mover validation for cloning erasure coded containers. + """Object Data Mover validation for erasure coded containers. Test Class Description: - Cloning a container whose objects are erasure coded and hold punched - extents. Enumeration of an erasure coded object is served by a parity - shard, where one parity block stands for a whole stripe, so the listed - extents are an upper bound on the data and cover the punched records. - A clone that copies the listed extents instead of what the fetch - actually returned writes uninitialized bytes over those holes. + Copying a container whose objects are erasure coded and hold punched + stripes. Enumeration of an erasure coded object is served by a parity + shard, where one parity block stands for a whole stripe, so a stripe + that has been punched is still reported as holding data. A copy that + trusts the listed extents instead of the io map the fetch returned + writes over those holes. :avocado: recursive """ @@ -29,24 +29,57 @@ def setUp(self): self.num_objs = self.params.get("num_objs", "/run/dataset/*") self.num_dkeys = self.params.get("num_dkeys", "/run/dataset/*") - self.num_akeys_array = self.params.get("num_akeys_array", "/run/dataset/*") - self.num_akeys_single = self.params.get("num_akeys_single", "/run/dataset/*") - self.akey_sizes = self.params.get("akey_sizes", "/run/dataset/*") - self.akey_extents = self.params.get("akey_extents", "/run/dataset/*") - self.punch_extents = self.params.get("punch_extents", "/run/dataset/*") + self.num_akeys = self.params.get("num_akeys", "/run/dataset/*") + self.stripe_size = self.params.get("stripe_size", "/run/dataset/*") + self.num_stripes = self.params.get("num_stripes", "/run/dataset/*") + self.punch_stripes = self.params.get("punch_stripes", "/run/dataset/*") + self.full_punch_akeys = self.params.get("full_punch_akeys", "/run/dataset/*", 0) # an EC class is required, a replicated one would make the test vacuous self.obj_class = self.params.get("obj_class", "/run/dataset/*", "OC_EC_2P1G1") + def _gen_dataset(self, cont): + """Create the punched EC dataset and confirm the source reads back as expected. + + Args: + cont (TestContainer): the container to create the dataset in. + + Returns: + list: a list of DaosObj created. + + """ + obj_list = self.dataset_gen_ec( + cont, self.num_objs, self.num_dkeys, self.num_akeys, self.obj_class, + self.stripe_size, self.num_stripes, self.punch_stripes, + full_punch_akeys=self.full_punch_akeys) + + # the source itself must report the punched stripes as holes, otherwise + # the run below would pass without ever exercising the hole handling + self._verify_dataset(obj_list, cont) + + return obj_list + + def _verify_dataset(self, obj_list, cont): + """Verify a dataset created by _gen_dataset. + + Args: + obj_list (list): obj_list returned from _gen_dataset. + cont (TestContainer): the container to verify. + """ + self.dataset_verify_ec( + obj_list, cont, self.num_objs, self.num_dkeys, self.num_akeys, + self.stripe_size, self.num_stripes, self.punch_stripes, + full_punch_akeys=self.full_punch_akeys) + def run_dm_obj_ec(self, tool): """ Test Description: - Tests cloning a container of erasure coded objects with holes. + Tests copying a container of erasure coded objects with holes. Use Cases: Create pool1 and cont1. Create a dataset of erasure coded objects in cont1, where each - array akey spans whole stripes and then has its leading records + array akey spans whole stripes and then has alternate stripes punched back out. - Clone cont1 to a new cont2 and verify that the holes are still + Copy cont1 to a new cont2 and verify that the holes are still holes in cont2. Args: @@ -57,19 +90,7 @@ def run_dm_obj_ec(self, tool): pool1 = self.get_pool() cont1 = self.get_container(pool1) - obj_list = self.dataset_gen( - cont1, - self.num_objs, self.num_dkeys, self.num_akeys_single, - self.num_akeys_array, self.akey_sizes, self.akey_extents, - oclass=self.obj_class, punch_extents=self.punch_extents) - - # the source itself must report the punched records as holes, otherwise - # the run below would pass without ever exercising the hole handling - self.dataset_verify( - obj_list, cont1, - self.num_objs, self.num_dkeys, self.num_akeys_single, - self.num_akeys_array, self.akey_sizes, self.akey_extents, - punch_extents=self.punch_extents) + obj_list = self._gen_dataset(cont1) result = self.run_datamover( self.test_id + " (cont1->cont2) (same pool)", @@ -78,11 +99,35 @@ def run_dm_obj_ec(self, tool): cont2_label = self.parse_create_cont_label(result.stdout_text) cont2 = get_existing_container(self, pool1, cont2_label) - self.dataset_verify( - obj_list, cont2, - self.num_objs, self.num_dkeys, self.num_akeys_single, - self.num_akeys_array, self.akey_sizes, self.akey_extents, - punch_extents=self.punch_extents) + self._verify_dataset(obj_list, cont2) + + def run_dm_obj_ec_dsync(self): + """ + Test Description: + Tests syncing a container of erasure coded objects with holes. + Use Cases: + Create pool1, cont1 and an empty cont2. + Create a dataset of erasure coded objects with punched stripes in cont1. + Sync cont1 to cont2 and verify the holes are still holes in cont2. + Sync a second time, so the comparison of the destination runs against + a populated container rather than an empty one. + """ + self.set_tool("DSYNC") + + pool1 = self.get_pool() + cont1 = self.get_container(pool1) + + # dsync does not create the destination + cont2 = self.get_container(pool1) + + obj_list = self._gen_dataset(cont1) + + for run in ("first", "second"): + self.run_datamover( + self.test_id + " (cont1->cont2) ({} sync)".format(run), + "DAOS_UUID", None, pool1, cont1, + "DAOS_UUID", None, pool1, cont2) + self._verify_dataset(obj_list, cont2) @avocado.fail_on(DaosApiError) def test_dm_obj_ec_cont_clone(self): @@ -95,3 +140,39 @@ def test_dm_obj_ec_cont_clone(self): :avocado: tags=DmvrObjEcTest,test_dm_obj_ec_cont_clone """ self.run_dm_obj_ec("CONT_CLONE") + + @avocado.fail_on(DaosApiError) + def test_dm_obj_ec_dcp(self): + """ + Test Description: + Verify copying an erasure coded container with punched extents. + :avocado: tags=all,daily_regression + :avocado: tags=hw,medium + :avocado: tags=datamover,mfu,mfu_dcp + :avocado: tags=DmvrObjEcTest,test_dm_obj_ec_dcp + """ + self.run_dm_obj_ec("DCP") + + @avocado.fail_on(DaosApiError) + def test_dm_obj_ec_dserialize(self): + """ + Test Description: + Verify serializing an erasure coded container with punched extents. + :avocado: tags=all,daily_regression + :avocado: tags=hw,medium + :avocado: tags=datamover,mfu,mfu_serialize,mfu_deserialize,hdf5 + :avocado: tags=DmvrObjEcTest,test_dm_obj_ec_dserialize + """ + self.run_dm_obj_ec("DSERIAL") + + @avocado.fail_on(DaosApiError) + def test_dm_obj_ec_dsync(self): + """ + Test Description: + Verify syncing an erasure coded container with punched extents. + :avocado: tags=all,daily_regression + :avocado: tags=hw,medium + :avocado: tags=datamover,mfu,mfu_dsync + :avocado: tags=DmvrObjEcTest,test_dm_obj_ec_dsync + """ + self.run_dm_obj_ec_dsync() diff --git a/src/tests/ftest/datamover/obj_ec.yaml b/src/tests/ftest/datamover/obj_ec.yaml index b1543e6d424..27b2543c4bc 100644 --- a/src/tests/ftest/datamover/obj_ec.yaml +++ b/src/tests/ftest/datamover/obj_ec.yaml @@ -4,6 +4,12 @@ hosts: timeout: 300 +timeouts: + test_dm_obj_ec_cont_clone: 300 + test_dm_obj_ec_dcp: 300 + test_dm_obj_ec_dsync: 400 + test_dm_obj_ec_dserialize: 400 + server_config: name: daos_server engines_per_host: 2 @@ -25,18 +31,36 @@ pool: size: 95% container: - # the smallest cell size keeps a full stripe at 8KiB for OC_EC_2P1G1 - properties: ec_cell_sz:4KiB + # OC_EC_2P1G1 tolerates one failure, the inherited default demands more + properties: rd_fac:1,rd_lvl:rank + +dcp: + client_processes: + np: 3 + +dsync: + client_processes: + np: 3 + +dserialize: + client_processes: + np: 3 + +ddeserialize: + client_processes: + np: 3 dataset: obj_class: OC_EC_2P1G1 - num_objs: 4 + num_objs: 2 num_dkeys: 2 - num_akeys_single: 2 - num_akeys_array: 2 - # 16 records of 1KiB is 16KiB, two whole stripes, so parity is written at - # update time without waiting for aggregation - akey_sizes: [1024] - akey_extents: [16] - # punch the first 4KiB, one whole cell, leaving stale parity over the hole - punch_extents: 4 + num_akeys: 2 + # OC_EC_2P1G1 is 2 data cells, and the default cell size is 128KiB, so a full + # stripe is 256KiB. Whole stripes are written so parity is computed at update + # time, then alternate stripes are punched, leaving that parity behind. + stripe_size: 262144 + num_stripes: 4 + punch_stripes: [0, 2] + # one akey per dkey has every stripe punched, so a listed extent returns no data + # at all. That is a harder case than an akey that keeps some live stripes. + full_punch_akeys: 1 diff --git a/src/tests/ftest/datamover/obj_small.py b/src/tests/ftest/datamover/obj_small.py index 4e3a4d1fbb2..f22a4b53756 100644 --- a/src/tests/ftest/datamover/obj_small.py +++ b/src/tests/ftest/datamover/obj_small.py @@ -1,5 +1,6 @@ ''' (C) Copyright 2020-2024 Intel Corporation. + (C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent ''' @@ -38,6 +39,47 @@ def setUp(self): "akey_sizes", "/run/dataset/*") self.akey_extents = self.params.get( "akey_extents", "/run/dataset/*") + self.punch_extents = self.params.get( + "punch_extents", "/run/dataset/*", 0) + self.punch_tail_extents = self.params.get( + "punch_tail_extents", "/run/dataset/*", 0) + + def _gen_dataset(self, cont): + """Create the dataset and confirm the source reads back as expected. + + Args: + cont (TestContainer): the container to create the dataset in. + + Returns: + list: a list of DaosObj created. + + """ + obj_list = self.dataset_gen( + cont, + self.num_objs, self.num_dkeys, self.num_akeys_single, + self.num_akeys_array, self.akey_sizes, self.akey_extents, + punch_extents=self.punch_extents, + punch_tail_extents=self.punch_tail_extents) + + # the source itself must report the punched records as holes, otherwise + # the runs below would pass without ever exercising the hole handling + self._verify_dataset(obj_list, cont) + + return obj_list + + def _verify_dataset(self, obj_list, cont): + """Verify a dataset created by _gen_dataset. + + Args: + obj_list (list): obj_list returned from _gen_dataset. + cont (TestContainer): the container to verify. + """ + self.dataset_verify( + obj_list, cont, + self.num_objs, self.num_dkeys, self.num_akeys_single, + self.num_akeys_array, self.akey_sizes, self.akey_extents, + punch_extents=self.punch_extents, + punch_tail_extents=self.punch_tail_extents) def run_dm_obj_small(self, tool): """ @@ -64,10 +106,7 @@ def run_dm_obj_small(self, tool): cont1 = self.get_container(pool1) # Create dataset in cont1 - obj_list = self.dataset_gen( - cont1, - self.num_objs, self.num_dkeys, self.num_akeys_single, - self.num_akeys_array, self.akey_sizes, self.akey_extents) + obj_list = self._gen_dataset(cont1) # Clone cont1 to a new cont2 in pool1 result = self.run_datamover( @@ -78,10 +117,7 @@ def run_dm_obj_small(self, tool): # Verify data in cont2 cont2 = get_existing_container(self, pool1, cont2_label) - self.dataset_verify( - obj_list, cont2, - self.num_objs, self.num_dkeys, self.num_akeys_single, - self.num_akeys_array, self.akey_sizes, self.akey_extents) + self._verify_dataset(obj_list, cont2) # Create pool2 pool2 = self.get_pool() @@ -94,10 +130,36 @@ def run_dm_obj_small(self, tool): cont3_label = self.parse_create_cont_label(result.stdout_text) # Verify data in cont3 cont3 = get_existing_container(self, pool2, cont3_label) - self.dataset_verify( - obj_list, cont3, - self.num_objs, self.num_dkeys, self.num_akeys_single, - self.num_akeys_array, self.akey_sizes, self.akey_extents) + self._verify_dataset(obj_list, cont3) + + def run_dm_obj_small_dsync(self): + """ + Test Description: + Tests syncing a small container at the object level. + Use Cases: + Create pool1. + Create cont1 and an empty cont2 in pool1. + Create a small dataset in cont1. + Sync cont1 to cont2. + Sync a second time, so the comparison of the destination runs + against a populated container rather than an empty one. + """ + self.set_tool("DSYNC") + + pool1 = self.get_pool() + cont1 = self.get_container(pool1) + + # dsync does not create the destination + cont2 = self.get_container(pool1) + + obj_list = self._gen_dataset(cont1) + + for run in ("first", "second"): + self.run_datamover( + self.test_id + " (cont1->cont2) ({} sync)".format(run), + "DAOS_UUID", None, pool1, cont1, + "DAOS_UUID", None, pool1, cont2) + self._verify_dataset(obj_list, cont2) @avocado.fail_on(DaosApiError) def test_dm_obj_small_dcp(self): @@ -122,3 +184,15 @@ def test_dm_obj_small_cont_clone(self): :avocado: tags=DmvrObjSmallTest,test_dm_obj_small_cont_clone """ self.run_dm_obj_small("CONT_CLONE") + + @avocado.fail_on(DaosApiError) + def test_dm_obj_small_dsync(self): + """ + Test Description: + Verify syncing a small container at the object level. + :avocado: tags=all,daily_regression + :avocado: tags=vm + :avocado: tags=datamover,mfu,mfu_dsync + :avocado: tags=DmvrObjSmallTest,test_dm_obj_small_dsync + """ + self.run_dm_obj_small_dsync() diff --git a/src/tests/ftest/datamover/obj_small.yaml b/src/tests/ftest/datamover/obj_small.yaml index fb7f4f57292..115abbc91a4 100644 --- a/src/tests/ftest/datamover/obj_small.yaml +++ b/src/tests/ftest/datamover/obj_small.yaml @@ -4,6 +4,7 @@ hosts: timeouts: test_dm_obj_small_dcp: 120 test_dm_obj_small_cont_clone: 90 + test_dm_obj_small_dsync: 180 server_config: name: daos_server engines_per_host: 1 @@ -21,6 +22,9 @@ pool: dcp: client_processes: np: 3 +dsync: + client_processes: + np: 3 dataset: num_objs: 10 num_dkeys: 10 @@ -28,3 +32,8 @@ dataset: num_akeys_array: 5 akey_sizes: [1, 512, 1024] akey_extents: [32, 16, 8, 4, 2] + # punch records back out so the arrays are sparse. A trailing hole is the one + # case that shortens the reply of a fetch, which is what a copy driven by the + # enumerated extents rather than the io map trips over. + punch_extents: 1 + punch_tail_extents: 1 diff --git a/src/tests/ftest/util/data_mover_test_base.py b/src/tests/ftest/util/data_mover_test_base.py index 52f74e52a77..6d5bbf199cb 100644 --- a/src/tests/ftest/util/data_mover_test_base.py +++ b/src/tests/ftest/util/data_mover_test_base.py @@ -310,9 +310,10 @@ def parse_create_cont_label(self, output): self.fail("Failed to parse container label") return label_search.group(1).strip() + # pylint: disable=too-many-arguments,too-many-locals def dataset_gen(self, cont, num_objs, num_dkeys, num_akeys_single, num_akeys_array, akey_sizes, akey_extents, oclass="OC_SX", - punch_extents=0): + punch_extents=0, punch_tail_extents=0): """Generate a dataset with some number of objects, dkeys, and akeys. Expects the container to be created with the API control method. @@ -327,8 +328,10 @@ def dataset_gen(self, cont, num_objs, num_dkeys, num_akeys_single, akey_extents (list): varying number of akey extents to iterate. oclass (str, optional): object class for the objects. Defaults to "OC_SX". punch_extents (int, optional): number of leading records to punch back out of - each array akey. Defaults to 0. Always leaves at least one record intact, - so the akey keeps a hole followed by data. + each array akey. Defaults to 0. + punch_tail_extents (int, optional): number of trailing records to punch back out + of each array akey. Defaults to 0. Only a trailing hole shortens the reply of + a fetch, so this is what catches a copy that trusts the enumerated extents. Returns: list: a list of DaosObj created. @@ -382,9 +385,13 @@ def dataset_gen(self, cont, num_objs, num_dkeys, num_akeys_single, c_data.append([create_string_buffer(data), data_size]) ioreq.insert_array(c_dkey, c_akey, c_data) - punch_nr = self._dataset_punch_nr(punch_extents, num_extents) - if punch_nr: - ioreq.punch_array(c_dkey, c_akey, 0, punch_nr) + punch_lead, punch_tail = self._dataset_punch_nr( + punch_extents, punch_tail_extents, num_extents) + if punch_lead: + ioreq.punch_array(c_dkey, c_akey, 0, punch_lead) + if punch_tail: + ioreq.punch_array( + c_dkey, c_akey, num_extents - punch_tail, punch_tail) obj.close() cont.close() @@ -392,25 +399,225 @@ def dataset_gen(self, cont, num_objs, num_dkeys, num_akeys_single, return obj_list @staticmethod - def _dataset_punch_nr(punch_extents, num_extents): - """Get how many leading records of an array akey are punched. + def _dataset_punch_nr(punch_extents, punch_tail_extents, num_extents): + """Get how many leading and trailing records of an array akey are punched. Args: - punch_extents (int): number of records the caller asked to punch. + punch_extents (int): number of leading records the caller asked to punch. + punch_tail_extents (int): number of trailing records the caller asked to punch. num_extents (int): number of records in the akey. Returns: - int: the number to punch, always leaving at least one record. + tuple: (leading, trailing), clamped to always leave at least one record. """ - if not punch_extents: - return 0 - return min(punch_extents, num_extents - 1) + lead = min(punch_extents or 0, max(num_extents - 1, 0)) + tail = min(punch_tail_extents or 0, max(num_extents - 1 - lead, 0)) + return lead, tail - # pylint: disable=too-many-locals + @staticmethod + def _ec_stripe_fill(stripe_idx): + """Get the byte an erasure coded stripe is filled with. + + Args: + stripe_idx (int): index of the stripe. + + Returns: + bytes: a single byte, distinct per stripe so a misplaced copy is visible. + + """ + return bytes([ord("a") + (stripe_idx % 26)]) + + # pylint: disable=too-many-arguments + def dataset_gen_ec(self, cont, num_objs, num_dkeys, num_akeys, oclass, + stripe_size, num_stripes, punch_stripes, full_punch_akeys=0): + """Generate erasure coded objects whose array akeys have whole stripes punched. + + Each akey is written one full stripe at a time so parity is computed at update + time, then whole stripes are punched. The parity of a punched stripe is left + behind, and enumeration of an erasure coded object is served by a parity shard, + so daos_obj_list_recx() reports those stripes as if they still held data. Only + the io map returned by a fetch says what is really there. + + The extents are byte granular on purpose. With a record size larger than a byte + the enumeration reports the live extents exactly and none of this is exercised. + + Args: + cont (TestContainer): the container. + num_objs (int): number of objects to create in the container. + num_dkeys (int): number of dkeys to create per object. + num_akeys (int): number of array akeys per dkey. + oclass (str): erasure coded object class, for example OC_EC_2P1G1. + stripe_size (int): full stripe size in bytes, data cells times the cell size. + num_stripes (int): number of stripes to write per akey. + punch_stripes (list): indices of the stripes to punch back out. + full_punch_akeys (int, optional): how many of the trailing akeys have every + stripe punched. Enumeration still reports those akeys as holding a full + stripe of data while the fetch returns nothing at all, which is a + separate case from an akey that keeps some live stripes. Defaults to 0. + + Returns: + list: a list of DaosObj created. + + """ + self.log.info("Creating erasure coded dataset in %s/%s", str(cont.pool), str(cont)) + + cont.open() + obj_list = [] + + for obj_idx in range(num_objs): + obj = DaosObj(cont.pool.context, cont.container) + obj_list.append(obj) + obj.create(rank=obj_idx, objcls=oclass) + obj.open() + + ioreq = IORequest(cont.pool.context, cont.container, obj) + for dkey_idx in range(num_dkeys): + c_dkey = create_string_buffer("dkey {}".format(dkey_idx)) + + for akey_idx in range(num_akeys): + c_akey = create_string_buffer("akey array {}".format(akey_idx)) + + for stripe_idx in range(num_stripes): + data = self._ec_stripe_fill(stripe_idx) * stripe_size + c_data = create_string_buffer(data, stripe_size) + ioreq.insert_recx( + c_dkey, c_akey, 1, stripe_idx * stripe_size, stripe_size, c_data) + + # punching a whole stripe leaves its parity behind + for stripe_idx in self._ec_punched_stripes( + akey_idx, num_akeys, num_stripes, punch_stripes, + full_punch_akeys): + ioreq.punch_array( + c_dkey, c_akey, stripe_idx * stripe_size, stripe_size) + + obj.close() + cont.close() + + return obj_list + + @staticmethod + def _ec_punched_stripes(akey_idx, num_akeys, num_stripes, punch_stripes, + full_punch_akeys): + """Get which stripes of an array akey are punched. + + Args: + akey_idx (int): index of the akey. + num_akeys (int): number of array akeys per dkey. + num_stripes (int): number of stripes per akey. + punch_stripes (list): stripes to punch in a normal akey. + full_punch_akeys (int): how many of the trailing akeys have every stripe + punched instead. + + Returns: + list: indices of the stripes to punch. + + """ + if akey_idx >= num_akeys - full_punch_akeys: + return list(range(num_stripes)) + return list(punch_stripes) + + @staticmethod + def _ec_live_extents(stripe_size, num_stripes, punch_stripes): + """Get the extents an akey should really hold data in. + + Args: + stripe_size (int): full stripe size in bytes. + num_stripes (int): number of stripes written per akey. + punch_stripes (list): indices of the stripes that were punched. + + Returns: + list: (rx_idx, rx_nr) tuples, with neighboring stripes merged the way + an io map reports them. + + """ + extents = [] + for stripe_idx in range(num_stripes): + if stripe_idx in punch_stripes: + continue + start = stripe_idx * stripe_size + if extents and extents[-1][0] + extents[-1][1] == start: + extents[-1] = (extents[-1][0], extents[-1][1] + stripe_size) + else: + extents.append((start, stripe_size)) + return extents + + # pylint: disable=too-many-arguments + def dataset_verify_ec(self, obj_list, cont, num_objs, num_dkeys, num_akeys, + stripe_size, num_stripes, punch_stripes, full_punch_akeys=0): + """Verify a dataset generated with dataset_gen_ec. + + Checks the io map as well as the bytes. A punched stripe and a stripe someone + wrote zeros over both read back as zeros, so comparing bytes alone would pass + against a copy that filled every hole in. + + Args: + obj_list (list): obj_list returned from dataset_gen_ec. + cont (TestContainer): the container. + num_objs (int): number of objects created in the container. + num_dkeys (int): number of dkeys created per object. + num_akeys (int): number of array akeys per dkey. + stripe_size (int): full stripe size in bytes. + num_stripes (int): number of stripes written per akey. + punch_stripes (list): indices of the stripes that were punched. + full_punch_akeys (int, optional): the value passed to dataset_gen_ec. + Defaults to 0. + + """ + self.log.info("Verifying erasure coded dataset in %s/%s", str(cont.pool), str(cont)) + + cont.open() + + for obj_idx in range(num_objs): + c_oid = obj_list[obj_idx].c_oid + obj = DaosObj(cont.pool.context, cont.container, c_oid=c_oid) + obj.open() + + ioreq = IORequest(cont.pool.context, cont.container, obj) + for dkey_idx in range(num_dkeys): + dkey = "dkey {}".format(dkey_idx) + c_dkey = create_string_buffer(dkey) + + for akey_idx in range(num_akeys): + akey = "akey array {}".format(akey_idx) + c_akey = create_string_buffer(akey) + punched = self._ec_punched_stripes( + akey_idx, num_akeys, num_stripes, punch_stripes, full_punch_akeys) + expect_extents = self._ec_live_extents( + stripe_size, num_stripes, punched) + where = "\nobj: {}.{}\ndkey: {}\nakey: {}".format( + obj.c_oid.hi, obj.c_oid.lo, dkey, akey) + actual, extents = ioreq.fetch_recx_map( + c_dkey, c_akey, 1, 0, num_stripes * stripe_size) + + for stripe_idx in range(num_stripes): + if stripe_idx in punched: + expect = b"\0" * stripe_size + else: + expect = self._ec_stripe_fill(stripe_idx) * stripe_size + start = stripe_idx * stripe_size + got = actual[start:start + stripe_size] + if got != expect: + self.log.info( + "Expected stripe %s to be %r but got %r", + stripe_idx, expect[:16], got[:16]) + self.log.info("For:%s", where) + self.fail("Erasure coded stripe verification failed.") + + if extents != expect_extents: + self.log.info( + "Expected the io map to hold %s but it holds %s", + expect_extents, extents) + self.log.info("For:%s", where) + self.fail("Punched stripes were filled in rather than kept.") + + obj.close() + cont.close() + + # pylint: disable=too-many-arguments,too-many-locals def dataset_verify(self, obj_list, cont, num_objs, num_dkeys, num_akeys_single, num_akeys_array, akey_sizes, - akey_extents, punch_extents=0): + akey_extents, punch_extents=0, punch_tail_extents=0): """Verify a dataset generated with dataset_gen. Args: @@ -424,6 +631,7 @@ def dataset_verify(self, obj_list, cont, num_objs, num_dkeys, akey_extents (list): varying number of akey extents to iterate. punch_extents (int, optional): the value passed to dataset_gen. Punched records must read back as zeros. Defaults to 0. + punch_tail_extents (int, optional): the value passed to dataset_gen. Defaults to 0. """ self.log.info("Verifying dataset in %s/%s", str(cont.pool), str(cont)) @@ -475,9 +683,10 @@ def dataset_verify(self, obj_list, cont, num_objs, num_dkeys, c_num_extents = ctypes.c_uint(num_extents) c_data_size = ctypes.c_size_t(data_size) actual_data = ioreq.fetch_array(c_dkey, c_akey, c_num_extents, c_data_size) - punch_nr = self._dataset_punch_nr(punch_extents, num_extents) + punch_lead, punch_tail = self._dataset_punch_nr( + punch_extents, punch_tail_extents, num_extents) for data_idx in range(num_extents): - if data_idx < punch_nr: + if data_idx < punch_lead or data_idx >= num_extents - punch_tail: # a punched record is a hole and reads back as zeros data = data_size * "\0" else: