diff --git a/.gitignore b/.gitignore index 0b9d4c1d..e6a08b07 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ docs/build/* docs/source/_build/* src/idvc/version.py build/* +*.profraw diff --git a/README.md b/README.md index 1f40cf93..62cb4c07 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,21 @@ iDVC depends on code initially developed by Prof. Brian K. Bay and collaborators [1] B. K. Bay, T. S. Smith, D. P. Fyhrie, M. Saad "Digital volume correlation: Three-dimensional strain mapping using x-ray tomography", *Experimental Mechanics* **39** 217–226, 1999. [DOI: 10.1007/BF02323555](https://doi.org/10.1007/BF02323555) [2] B. K. Bay "Methods and applications of digital volume correlation", *J. Strain Analysis* **43** 745-760, 2008. [DOI: 10.1243/03093247JSA436](https://doi.org/10.1243/03093247JSA436) + + +# MacOS + Qt + VTK + +`conda-forge`'s VTK package depends on Qt6 and we only tested this app with Qt5. On MacOS this seems to be a problem as the OS finds 2 files which provide the same functionality so it raises an error and nothing works. The suggested workaround is to install the CILViewer (from the repo), `PySide2` (`conda-forge`) and [VTK](https://docs.vtk.org/en/latest/advanced/available_python_wheels.html) (from Kitware's wheels) manually. + + +```bash +pip install --extra-index-url https://wheels.vtk.org vtk +``` + +```yaml +dependencies: + - pip + - pip: + - --extra-index-url https://wheels.vtk.org + - vtk +``` \ No newline at end of file diff --git a/recipe/dev_environment.yml b/recipe/dev_environment.yml index 53242c25..88a5bfbc 100644 --- a/recipe/dev_environment.yml +++ b/recipe/dev_environment.yml @@ -11,10 +11,11 @@ dependencies: - pyside2 - scipy - ccpi::ccpi-viewer >=25.0.0 - - ccpi::ccpi-dvc >=22.0.0 + - ccpi/label/dev::ccpi-dvc >=25.0.0 - natsort - docopt - matplotlib - openmp # [osx] - qdarkstyle - vtk + - filetype diff --git a/recipe/dev_environment_osx.yml b/recipe/dev_environment_osx.yml new file mode 100644 index 00000000..252a0213 --- /dev/null +++ b/recipe/dev_environment_osx.yml @@ -0,0 +1,29 @@ +name: idvc_dev +channels: + - conda-forge + - ccpi +dependencies: + # - eqt >=1.0.1 + - openpyxl + - python + - numpy + - pandas + - pyside2 + - scipy + # - ccpi::ccpi-viewer >=25.0.0 + - ccpi/label/dev::ccpi-dvc >=25.0.0 + - natsort + - docopt + - matplotlib + - llvm-openmp + - qdarkstyle + - h5py + - pyyaml + - schema + - qtpy + - filetype + - pip: + - --extra-index-url https://wheels.vtk.org + - vtk + - git+https://github.com/TomographicImaging/CILViewer.git#subdirectory=Wrappers/Python + - eqt >=1.0.1 diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 49a3b47a..312c2179 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -41,8 +41,8 @@ requirements: - pandas - pyside2 - scipy - - ccpi-viewer >=25.0.0 - - ccpi-dvc >=22.0.0 + - ccpi::ccpi-viewer >=25.0.0 + - ccpi/label/dev::ccpi-dvc >=25.0.0 - natsort - docopt - matplotlib diff --git a/src/idvc/dvc_config_template.txt b/src/idvc/dvc_config_template.txt index edf58a75..a36bd906 100644 --- a/src/idvc/dvc_config_template.txt +++ b/src/idvc/dvc_config_template.txt @@ -42,8 +42,8 @@ min_vol_fract {min_vol_fract} ### only search if subvol fraction is greater th disp_max {disp_max} ### in voxels, used for range checking and global search limits num_srch_dof {num_srch_dof} ### 3, 6, or 12 obj_function {obj_function} ### sad, ssd, zssd, nssd, znssd -interp_type {interp_type} ### trilinear, tricubic - +interp_type {interp_type} ### trilinear, tricubic, tri_bspline_3, tri_bspline_5, tri_bspline_7 +{bspline_order} ### optional parameters tuning and refining the search process rigid_trans {rigid_trans} ### rigid body offset of target volume, in voxels diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 0c5c9e8d..bcddc185 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -54,7 +54,7 @@ from ccpi.viewer.utils.colormaps import CILColorMaps from natsort import natsorted -import imghdr +import filetype import glob # from vtk.numpy_interface import algorithms as algs @@ -762,8 +762,9 @@ def copy_file(self, start_location, end_location, image_var, **kwargs): N = len(start_location) for i,fs in enumerate(zip(start_location, end_location)): if N > 1: - ftype = imghdr.what(fs[0]) - if ftype != 'tiff': + kind = filetype.guess(fs[0]) + ftype = kind.extension if kind is not None else "" + if ftype not in ['tiff', 'tif']: raise ValueError('File type not supported: {}'.format(fs[0])) progress_callback.emit(i/N*100) shutil.copyfile(fs[0], fs[1]) @@ -4384,6 +4385,9 @@ def CreateRunDVCPanel(self): rdvc_widgets['run_iterp_type_entry'].addItem('Nearest') rdvc_widgets['run_iterp_type_entry'].addItem('Trilinear') rdvc_widgets['run_iterp_type_entry'].addItem('Tricubic') + rdvc_widgets['run_iterp_type_entry'].addItem('tri_bspline_3') + rdvc_widgets['run_iterp_type_entry'].addItem('tri_bspline_5') + rdvc_widgets['run_iterp_type_entry'].addItem('tri_bspline_7') rdvc_widgets['run_iterp_type_entry'].setCurrentIndex(2) rdvc_widgets['run_iterp_type_entry'].setToolTip(interp_text) formLayout.setWidget(widgetno, QFormLayout.FieldRole, rdvc_widgets['run_iterp_type_entry']) @@ -4753,15 +4757,15 @@ def create_run_config(self, **kwargs): run_config['subvol_geom'] = self.pointcloud_parameters['pointcloud_volume_shape_entry'].currentText().lower() run_config['subvol_npts'] = self.subvolume_points - run_config['disp_max'] = self.rdvc_widgets['run_max_displacement_entry'].value(), #38 for test image + run_config['step_max'] = self.rdvc_widgets['run_max_displacement_entry'].value(), #38 for test image run_config['dof'] = self.rdvc_widgets['run_ndof_entry'].currentText() run_config['obj'] = self.rdvc_widgets['run_objf_entry'].currentText() run_config['interp_type'] = self.rdvc_widgets['run_iterp_type_entry'].currentText().lower() if (hasattr(self, 'translate')): - run_config['rigid_trans'] = str(self.translate.GetTranslation()[0]*-1) + " " + str(self.translate.GetTranslation()[1]*-1) + " " + str(self.translate.GetTranslation()[2]*-1) + run_config['start_estimate'] = str(self.translate.GetTranslation()[0]*-1) + " " + str(self.translate.GetTranslation()[1]*-1) + " " + str(self.translate.GetTranslation()[2]*-1) else: - run_config['rigid_trans']= "0.0 0.0 0.0" + run_config['start_estimate']= "0.0 0.0 0.0" self.run_folder = "Results/" + folder_name run_config['run_folder'] = self.run_folder @@ -5006,7 +5010,7 @@ def CreateViewDVCResultsPanel(self): scroll_area = dockWidget.widget() scroll_area.apply_qdarkstyle_to_buttons(self.dvc_results_panel[1]) - + def show_run_pcs(self): #show pointcloud files in list self.result_widgets['pc_entry'].clear() @@ -5040,16 +5044,16 @@ def LoadResultsOnViewer(self): try: subvol_size = int(self.result_widgets['pc_entry'].currentText()) except ValueError as ve: - self.warningDialog("Invalid input at Subvolume Size", "Error") + self.warningDialog(message="Invalid input at Subvolume Size", window_title="Parse Error", detailed_text=f"{ve}") return try: subvol_points = int(self.result_widgets['subvol_entry'].currentText()) except ValueError as ve: - self.warningDialog("Invalid input at Subvolume Size", "Error") + self.warningDialog(message="Invalid input at Subvolume Entry", window_title="Parse Error", detailed_text=f"{ve}") return if(subvol_points == ""): - self.warningDialog("An error occurred with this run so the results could not be displayed.", "Error") + self.warningDialog(message="An error occurred with this run so the results could not be displayed.", window_title="Error") return diff --git a/src/idvc/dvc_runner.py b/src/idvc/dvc_runner.py index 386c2a4f..a0810584 100644 --- a/src/idvc/dvc_runner.py +++ b/src/idvc/dvc_runner.py @@ -32,25 +32,16 @@ def emit(self, *args, **kwargs): count = 0 runs_completed = 0 -blank_config = '''############################################################################### -# -# -# example dvc process control file -# -# -############################################################################### - -# all lines beginning with a # character are ignored -# some parameters are conditionally required, depending on the setting of other parameters -# for example, if subvol_thresh is off, the threshold description parameters are not required - +blank_config = ''' ### file names -reference_filename {reference_filename} ### reference tomography image volume -correlate_filename {correlate_filename} ### correlation tomography image volume +reference_filename {reference_filename} ### single raster file (raw) of same dimensions as the correlate_volume +correlate_filename {correlate_filename} ### single raster file (raw) of same dimensions as the reference_volume + +point_cloud_filename {point_cloud_filename} ### tab/space/comma delimited text file with point cloud labels (n) and (xyz) locations +output_filename {output_filename} ### base name for run status (.stat) and displacement data (.disp) output files -point_cloud_filename {point_cloud_filename} ### file of search point locations -output_filename {output_filename} ### base name for output files +num_points_to_process {num_points_to_process} ### Number of points in the point cloud to process ### description of the image data files, all must be the same size and structure @@ -59,33 +50,35 @@ def emit(self, *args, **kwargs): vol_wide {vol_wide} ### width in pixels of each slice vol_high {vol_high} ### height in pixels of each slice vol_tall {vol_tall} ### number of slices in the stack -vol_endian {vol_endian} ### big or little endian byte ordering +vol_endian {vol_endian} ### big or little endian byte ordering, ignored if vol_bit_depth is 8 ### parameters defining the subvolumes that will be created at each search point -subvol_geom {subvol_geom} ### cube, sphere +subvol_geom {subvol_geom} ### cube, sphere: geometry of the subvolumes subvol_size {subvol_size} ### side length or diameter, in voxels subvol_npts {subvol_npts} ### number of points to distribute within the subvol +subvol_aspect {subvol_aspect} ### stretch or contract the subvolume in any coordinate direction between 0.5 and 2.0 -subvol_thresh {subvol_thresh} ### on or off, evaluate subvolumes based on threshold -gray_thresh_min {gray_thresh_min} ### lower limit of a gray threshold range if subvol_thresh is on -gray_thresh_max {gray_thresh_max} ### upper limit of a gray threshold range if subvol_thresh is on -min_vol_fract {min_vol_fract} ### only search if subvol fraction is greater than - -### required parameters defining the basic the search process +### +### Keywords in group opt_mthd +### -disp_max {disp_max} ### in voxels, used for range checking and global search limits -num_srch_dof {num_srch_dof} ### 3, 6, or 12 -obj_function {obj_function} ### sad, ssd, zssd, nssd, znssd -interp_type {interp_type} ### trilinear, tricubic +num_srch_dof {num_srch_dof} ### 3 (translation), 6 (+rotation), or 12 (+strain) +obj_function {obj_function} ### ssd, zssd, nssd, znssd +interp_type {interp_type} ### trilinear tricubic tri_bspline_3 tri_bspline_5 tri_bspline_7 +start_position {starting_point} ### x,y,z location of starting point for DVC analysisstart_estimate +start_estimate {start_estimate} ### x,y,z displacement (u,v,w) in voxels at the start_position +step_max {step_max} ### maximum parameter step allowed during optimization, see also step_tol ### optional parameters tuning and refining the search process -rigid_trans {rigid_trans} ### rigid body offset of target volume, in voxels -basin_radius {basin_radius} ### coarse-search resolution, in voxels, 0.0 = none -subvol_aspect {subvol_aspect} ### subvolume aspect ratio -num_points_to_process {num_points_to_process} ### Number of points in the point cloud to process -starting_point {starting_point} ### x,y,z location of starting point for DVC analysis +### +### Keywords in group opt_tune +### + +# cost_tol 1.0e-06 ### optional tuning of objective function convergence tolerance +# step_tol 1.0e-04 ### optional tuning of parameter vector convergence tolerance +# max_iter 20 ### optional change in the maximum number of optimization iterations ''' def update_progress(main_window, process, total_points, required_runs, run_succeeded, start_time, num_points_to_process): @@ -232,7 +225,8 @@ def set_up(self, *args, **kwargs): - disp_max: maximum displacement in voxels - dof: number of parameters in the optimisation, 3,6, or 12 - obj: objective function, either sad, ssd, zssd, nssd, or znssd - - interp_type: interpolation type, either trilinear or tricubic + - interp_type: interpolation type, either trilinear, tricubic, or bspline + - bspline_order: order of the B-spline interpolation, only used if interp_type is bspline - run_folder: folder to save the run results and configurations - rigid_trans: rigid translation between the reference and correlation volumes - point0_world_coordinate: world coordinates of the starting point for the DVC analysis @@ -314,13 +308,13 @@ def set_up(self, *args, **kwargs): subvol_npts = config['subvol_npts'] - disp_max = int(config['disp_max'][0]) #TODO: need to change + step_max = int(config['step_max'][0]) #TODO: need to change dof = int(config['dof']) obj = config['obj'] interp_type = config['interp_type'] - rigid_trans = config['rigid_trans'] - starting_point = config['point0_world_coordinate'] + start_estimate = config['start_estimate'] + start_position = config['point0_world_coordinate'] # Change directory into the folder where the run will be saved: os.chdir(self.session_folder) @@ -394,33 +388,29 @@ def set_up(self, *args, **kwargs): return + config = blank_config.format( - reference_filename= reference_file, # reference tomography image volume - correlate_filename= correlate_file, # correlation tomography image volume + reference_filename= reference_file, # reference tomography image volume + correlate_filename= correlate_file, # correlation tomography image volume point_cloud_filename = grid_roi_fname, - output_filename= output_filename, + output_filename= output_filename, vol_bit_depth= vol_bit_depth, # get from ref, 8 or 16 - vol_hdr_lngth=vol_hdr_lngth,# get from ref, fixed-length header size, may be zero - vol_wide= dims[0], # number of x slices - vol_high= dims[1], # number of y slices - vol_tall= dims[2], #number of z slices - vol_endian = endian, - subvol_geom= subvol_geom, - subvol_size= subvolume_size, - subvol_npts= subvolume_point, - subvol_thresh='off', - gray_thresh_min='27', - gray_thresh_max='127', - min_vol_fract='0.2', - disp_max= disp_max, #38 for test image - num_srch_dof= dof, #6 for test image - obj_function= obj, - interp_type= interp_type, #tricubic for test image - rigid_trans= rigid_trans, #translation between ref and cor - determined from image registration - basin_radius='0.0', + vol_hdr_lngth= vol_hdr_lngth,# get from ref, fixed-length header size, may be zero + vol_wide= dims[0], # number of x slices + vol_high= dims[1], # number of y slices + vol_tall= dims[2], #number of z slices + vol_endian = endian, + subvol_geom= subvol_geom, + subvol_size= subvolume_size, + subvol_npts= subvolume_point, + step_max= step_max, #38 for test image + num_srch_dof= dof, #6 for test image + obj_function= obj, + interp_type= interp_type, #tricubic for test image + start_estimate= start_estimate, #translation between ref and cor - determined from image registration subvol_aspect='1.0 1.0 1.0',# image spacing num_points_to_process=num_points_to_process, - starting_point='{} {} {}'.format(*starting_point)) + starting_point='{} {} {}'.format(*start_position)) time.sleep(1) with open(config_filename,"w") as config_file: config_file.write(config) diff --git a/src/idvc/io.py b/src/idvc/io.py index 92aa4158..683f6ca8 100644 --- a/src/idvc/io.py +++ b/src/idvc/io.py @@ -13,7 +13,7 @@ # Author: Laura Murgatroyd (UKRI-STFC) # Author: Edoardo Pasca (UKRI-STFC) -import imghdr +import filetype import os import shutil import sys @@ -116,7 +116,8 @@ def getFileExtension(self): file_extension = os.path.splitext(self.image)[1] else: for image in self.image_files: - file_extension = imghdr.what(image) + kind = filetype.guess(image) + file_extension = kind.extension if kind is not None else "" if file_extension.lower() not in ['tiff', 'tif']: error_title = "Read Error" error_text = "Error reading file: ({filename})".format( diff --git a/src/idvc/utilities.py b/src/idvc/utilities.py index a50916a4..6fad6940 100644 --- a/src/idvc/utilities.py +++ b/src/idvc/utilities.py @@ -7,7 +7,114 @@ import os from eqt.ui.NoBorderScrollArea import NoBorderScrollArea +import re +import sys +def parse_stat_file(filepath): + data = {} + + with open(filepath, 'r') as f: + lines = f.readlines() + + in_echo = False + + for line in lines: + line = line.strip() + if not line: + continue + + if line == '### echo of the input file for this run': + in_echo = True + continue + elif line == '### end of input file echo': + in_echo = False + continue + + if in_echo: + parts = line.split() + if not parts: + continue + key = parts[0] + values = parts[1:] + + parsed_values = [] + for v in values: + try: + if '.' in v or 'e' in v.lower(): + parsed_values.append(float(v)) + else: + parsed_values.append(int(v)) + except ValueError: + parsed_values.append(v) + + if len(parsed_values) == 1: + data[key] = parsed_values[0] + elif len(parsed_values) > 1: + data[key] = parsed_values + + else: + # Parse the post-echo summary stats + try: + if line.startswith("Point Cloud contains"): + m = re.search(r'contains (\d+) points', line) + if m: data['point_cloud_points'] = int(m.group(1)) + + elif line.startswith("bounding box min"): + m = re.search(r'\[(.*?)\]', line) + if m: data['bounding_box_min'] = [int(x) for x in m.group(1).split()] + + elif line.startswith("bounding box max"): + m = re.search(r'\[(.*?)\]', line) + if m: data['bounding_box_max'] = [int(x) for x in m.group(1).split()] + + elif line.startswith("running under dvc code version:"): + parts = line.split(": ") + if len(parts) > 1: + data['dvc_code_version'] = parts[1].strip() + + elif line.startswith("Run start:"): + data['run_start'] = line.split(":\t")[1].strip() if ":\t" in line else line.replace("Run start:", "").strip() + + elif line.startswith("Run finish:"): + data['run_finish'] = line.split(":\t")[1].strip() if ":\t" in line else line.replace("Run finish:", "").strip() + + elif "points processed in" in line: + m = re.search(r'(\d+) points processed in (\d+) seconds', line) + if m: + data['points_processed'] = int(m.group(1)) + data['time_seconds'] = int(m.group(2)) + + elif line.endswith("sec/pt"): + parts = line.split() + if parts: + data['sec_per_pt'] = float(parts[0]) + + elif line.endswith("pt/sec"): + parts = line.split() + if parts: + data['pt_per_sec'] = float(parts[0]) + + elif line.startswith("number successful"): + m = re.search(r'number successful = (\d+)\s+\((.*?)\%\)', line) + if m: + data['num_successful'] = int(m.group(1)) + data['pct_successful'] = float(m.group(2)) + + elif line.startswith("number range fail"): + m = re.search(r'number range fail = (\d+)\s+\((.*?)\%\)', line) + if m: + data['num_range_fail'] = int(m.group(1)) + data['pct_range_fail'] = float(m.group(2)) + + elif line.startswith("number convg fail"): + m = re.search(r'number convg fail = (\d+)\s+\((.*?)\%\)', line) + if m: + data['num_convg_fail'] = int(m.group(1)) + data['pct_convg_fail'] = float(m.group(2)) + except (ValueError, IndexError, AttributeError) as e: + print(f"Warning: Failed to parse line '{line}'. Error: {e}", file=sys.stderr) + + return data class RunResults(object): def __init__(self, folder): @@ -16,36 +123,17 @@ def __init__(self, folder): disp_file_name = file_name + ".disp" stat_file_name = file_name + ".stat" - with open(stat_file_name,"r") as stat_file: - - count = 0 - offset = 0 - for line in stat_file: - if count == 9: - if line.split('\t')[0] == "vol_endian": - offset = 1 - - if count == 14 + offset: - self.subvol_geom = str(line.split('\t')[1]) - if count == 15 + offset: - self.subvol_size = round(int(line.split('\t')[1])) - if count == 16 +offset: - self.subvol_points = int(line.split('\t')[1]) - if count == 20 + offset: - self.disp_max = int(line.split('\t')[1]) - if count == 21 + offset: - self.num_srch_dof = int(line.split('\t')[1]) - if count == 22 + offset: - self.obj_function = str(line.split('\t')[1]) - if count == 23 + offset: - self.interp_type = str(line.split('\t')[1]) - if count == 25 + offset: - self.rigid_trans = [int(line.split('\t')[1]),int(line.split('\t')[2]), int(line.split('\t')[3])] - # if count == 26 + offset: - # self.basin_radius = int(line.split('\t')[1]) - # if count == 27 + offset: - # self.subvol_aspect = [int(line.split('\t')[1]),int(line.split('\t')[2]), int(line.split('\t')[3])] - count+=1 + stat_data = parse_stat_file(stat_file_name) + self.stat_data = stat_data + self.subvol_geom = stat_data.get('subvol_geom') + self.subvol_size = stat_data.get('subvol_size') + self.subvol_points = stat_data.get('subvol_npts') + self.disp_max = stat_data.get('disp_max') + self.num_srch_dof = stat_data.get('num_srch_dof') + self.obj_function = stat_data.get('obj_function') + self.interp_type = stat_data.get('interp_type') + self.rigid_trans = stat_data.get('rigid_trans') + data_label_dict = { 'objmin': "Objective minimum", 'u': "Displacement x component", 'v':"Displacement y component", 'w':"Displacement z component",