From 9e7d609f4281b96cb6875a21cbf2fce104c006ab Mon Sep 17 00:00:00 2001 From: Clemens Vonrhein Date: Fri, 12 Jun 2020 15:57:14 +0100 Subject: [PATCH 1/7] allow for image offset via DURIN_IMAGE_NUMBER_OFFSET environment variable --- src/file.c | 6 ++++-- src/file.h | 1 + src/plugin.c | 7 +++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/file.c b/src/file.c index be7b5ee..aa8bb71 100644 --- a/src/file.c +++ b/src/file.c @@ -257,11 +257,12 @@ int get_frame_from_chunk(const struct ds_desc_t *desc, int get_nxs_frame( const struct ds_desc_t *desc, - const int n, + const int nin, void *buffer) { /* detector data are the two inner most indices */ /* TODO: handle ndims > 3 and select appropriately */ int retval = 0; + int n = nin - desc->image_number_offset; hsize_t frame_idx[3] = {n, 0, 0}; hsize_t frame_size[3] = {1, desc->dims[1], desc->dims[2]}; if (n < 0 || n >= desc->dims[0]) { @@ -280,10 +281,11 @@ int get_nxs_frame( int get_dectris_eiger_frame( const struct ds_desc_t *desc, - int n, + int nin, void *buffer) { int retval = 0; + int n = nin - desc->image_number_offset; int block, frame_count, idx; struct eiger_ds_desc_t *eiger_desc = (struct eiger_ds_desc_t*) desc; char data_name[16] = {0}; diff --git a/src/file.h b/src/file.h index a7e0ae9..23d923b 100644 --- a/src/file.h +++ b/src/file.h @@ -17,6 +17,7 @@ struct ds_desc_t { hid_t data_g_id; hsize_t dims[3]; int data_width; + int image_number_offset; int (*get_pixel_properties)(const struct ds_desc_t*, double*, double*); int (*get_pixel_mask)(const struct ds_desc_t*, int*); int (*get_data_frame)(const struct ds_desc_t*, const int, void*); diff --git a/src/plugin.c b/src/plugin.c index c1c53c4..8bb4118 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -125,6 +125,13 @@ void plugin_open( ERROR_JUMP(-4, done, ""); } + data_desc->image_number_offset = 0; + char *cenv; + cenv = getenv("DURIN_IMAGE_NUMBER_OFFSET"); + if (cenv!=NULL) { + data_desc->image_number_offset = atoi(cenv); + } + mask_buffer = malloc(data_desc->dims[1] * data_desc->dims[2] * sizeof(int)); if (mask_buffer) { retval = data_desc->get_pixel_mask(data_desc, mask_buffer); From 77d2b8495725c300598ab3ceb415160d0533a9ca Mon Sep 17 00:00:00 2001 From: Clemens Vonrhein Date: Thu, 8 Apr 2021 14:46:48 +0100 Subject: [PATCH 2/7] Better handling of (usually) unsigned data arrays when calling routine to convert to (signed) int and apply pixel mask. The environment variable DURIN_RESET_UNMASKED_PIXEL can now be used to set non-masked saturated pixels in order to process them correctly with e.g. XDS. --- src/file.c | 14 ++++- src/plugin.c | 157 +++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 127 insertions(+), 44 deletions(-) diff --git a/src/file.c b/src/file.c index 8722518..f941a12 100644 --- a/src/file.c +++ b/src/file.c @@ -104,7 +104,13 @@ int get_nxs_dataset_dims(struct ds_desc_t *desc) { ERROR_JUMP(-1, close_space, "Error getting dataset dimensions"); } - desc->data_width = width; + if ( H5Tequal(t_id,H5T_NATIVE_CHAR)>0 || H5Tequal(t_id,H5T_NATIVE_INT)>0 || H5Tequal(t_id,H5T_NATIVE_SHORT)>0 || H5Tequal(t_id,H5T_NATIVE_LONG)>0 || H5Tequal(t_id,H5T_NATIVE_LLONG)>0 ) { + // signed + desc->data_width = -width; + } else { + // unsigned + desc->data_width = width; + } close_space: H5Sclose(s_id); @@ -233,7 +239,7 @@ int get_frame_from_chunk(const struct ds_desc_t *desc, const char *ds_name, if (o_eiger_desc->bs_applied) { if (bslz4_decompress(o_eiger_desc->bs_params, c_bytes, c_buffer, - desc->data_width * frame_size[1] * frame_size[2], + abs(desc->data_width) * frame_size[1] * frame_size[2], buffer) < 0) { char message[128]; sprintf(message, @@ -351,6 +357,10 @@ int get_dectris_eiger_dataset_dims(struct ds_desc_t *desc) { if (data_width <= 0) { ERROR_JUMP(-1, close_space, "Unable to get type size"); } + if ( H5Tequal(t_id,H5T_NATIVE_CHAR)>0 || H5Tequal(t_id,H5T_NATIVE_INT)>0 || H5Tequal(t_id,H5T_NATIVE_SHORT)>0 || H5Tequal(t_id,H5T_NATIVE_LONG)>0 || H5Tequal(t_id,H5T_NATIVE_LLONG)>0 ) { + // signed + data_width = -data_width; + } ndims = H5Sget_simple_extent_ndims(s_id); if (ndims != 3) { diff --git a/src/plugin.c b/src/plugin.c index 676c38b..f45a1d9 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -17,24 +17,41 @@ /* mask bits loosely based on what Neggia does and what NeXus says should be done basically - anything in the low byte (& 0xFF) means "ignore this" Neggia uses the value -2 if bit 1, 2 or 3 are set */ -#define COPY_AND_MASK(in, out, size, mask) \ +/* CV-GPhL-20210408: we want more control over the value non-masked + pixels should be set to. */ +#define COPY_AND_MASK(in, value, setValue, out, size, mask) \ { \ int i; \ - if (mask) { \ - for (i = 0; i < size; ++i) { \ - out[i] = in[i]; \ - if (mask[i] & 0xFF) \ - out[i] = -1; \ - if (mask[i] & 30) \ - out[i] = -2; \ + if (value!=0) { \ + if (mask) { \ + for (i = 0; i < size; ++i) { \ + out[i] = (in[i] == value) ? setValue : in[i]; \ + if (mask[i] & 0xFF) \ + out[i] = -1; \ + if (mask[i] & 30) \ + out[i] = -2; \ + } \ + } else { \ + for (i = 0; i < size; i++) { \ + out[i] = (in[i] == value) ? setValue : in[i]; \ + } \ } \ } else { \ - for (i = 0; i < size; i++) { \ - out[i] = in[i]; \ + if (mask) { \ + for (i = 0; i < size; ++i) { \ + out[i] = in[i]; \ + if (mask[i] & 0xFF) \ + out[i] = -1; \ + if (mask[i] & 30) \ + out[i] = -2; \ + } \ + } else { \ + for (i = 0; i < size; i++) { \ + out[i] = in[i]; \ + } \ } \ } \ } - #define APPLY_MASK(buffer, mask, size) \ { \ int i; \ @@ -58,36 +75,97 @@ void fill_info_array(int info[1024]) { info[2] = VERSION_MINOR; info[3] = VERSION_PATCH; info[4] = VERSION_TIMESTAMP; + info[5] = 0; // image number offset + info[6] = -1; // marked pixels not already in pixel_mask: reset to this value + + char *cenv; + cenv = getenv("DURIN_IMAGE_NUMBER_OFFSET"); + if (cenv!=NULL) { + info[5] = atoi(cenv); + } + cenv = getenv("DURIN_RESET_UNMASKED_PIXEL"); + if (cenv!=NULL) { + info[6] = atoi(cenv); + } + } -int convert_to_int_and_mask(void *in_buffer, int d_width, int *out_buffer, +int convert_to_int_and_mask(void *in_buffer, int width, int setValue, int *out_buffer, int length, int *mask) { /* transfer data to output buffer, performing data conversion as required */ int retval = 0; /* TODO: decide how conversion of data should work */ /* Should we sign extend? Neggia doesn't (casts from uint*), but may be more * intuitive */ - if (d_width == sizeof(signed char)) { - signed char *in = in_buffer; - COPY_AND_MASK(in, out_buffer, length, mask); - } else if (d_width == sizeof(short)) { - short *in = in_buffer; - COPY_AND_MASK(in, out_buffer, length, mask); - } else if (d_width == sizeof(int)) { - int *in = in_buffer; - COPY_AND_MASK(in, out_buffer, length, mask); - } else if (d_width == sizeof(long int)) { - long int *in = in_buffer; - COPY_AND_MASK(in, out_buffer, length, mask); - } else if (d_width == sizeof(long long int)) { - long long int *in = in_buffer; - COPY_AND_MASK(in, out_buffer, length, mask); - } else { - char message[128]; - sprintf(message, "Unsupported conversion of data width %d to %ld (int)", - d_width, sizeof(int)); - ERROR_JUMP(-1, done, message); + + int d_width = abs(width); + + // CV-20210407 + // Dealing with a signed data array: no extra check for marker + // value needed (since data can already take advantage of the + // negative data range). It is unclear though why/when data would + // come in as a signed array in the first place ... + if (width<0) { + if (d_width == sizeof(signed char)) { + // 8-bit + signed char *in = in_buffer; + COPY_AND_MASK(in, 0, setValue, out_buffer, length, mask); + } else if (d_width == sizeof(short)) { + // 16-bit + short *in = in_buffer; + COPY_AND_MASK(in, 0, setValue, out_buffer, length, mask); + } else if (d_width == sizeof(int)) { + // 16-bit + int *in = in_buffer; + COPY_AND_MASK(in, 0, setValue, out_buffer, length, mask); + } else if (d_width == sizeof(long int)) { + // 32-bit + long int *in = in_buffer; + COPY_AND_MASK(in, 0, setValue, out_buffer, length, mask); + } else if (d_width == sizeof(long long int)) { + // 64-bit + long long int *in = in_buffer; + COPY_AND_MASK(in, 0, setValue, out_buffer, length, mask); + } else { + char message[128]; + sprintf(message, "Unsupported conversion of data width %d to %ld (int)", + d_width, sizeof(int)); + ERROR_JUMP(-1, done, message); + } + } + // CV-20210407 + // Dealing with an unsigned data array: extra check for marker + // value required (to handle overloaded pixels correctly if wanted + // - see also DURIN_RESET_UNMASKED_PIXEL environment variable). + else { + if (d_width == sizeof(unsigned char)) { + // 8-bit + unsigned char *in = in_buffer; + COPY_AND_MASK(in, UINT8_MAX, setValue, out_buffer, length, mask); + } else if (d_width == sizeof(unsigned short)) { + // 16-bit + unsigned short *in = in_buffer; + COPY_AND_MASK(in, UINT16_MAX, setValue, out_buffer, length, mask); + } else if (d_width == sizeof(unsigned int)) { + // 16-bit + unsigned int *in = in_buffer; + COPY_AND_MASK(in, UINT16_MAX, setValue, out_buffer, length, mask); + } else if (d_width == sizeof(unsigned long)) { + // 32-bit + unsigned long *in = in_buffer; + COPY_AND_MASK(in, UINT32_MAX, setValue, out_buffer, length, mask); + } else if (d_width == sizeof(unsigned long long)) { + // 64-bit + unsigned long long *in = in_buffer; + COPY_AND_MASK(in, UINT32_MAX, setValue, out_buffer, length, mask); + } else { + char message[128]; + sprintf(message, "Unsupported conversion of data width %d to %ld (int)", + d_width, sizeof(int)); + ERROR_JUMP(-1, done, message); + } } + done: return retval; } @@ -124,12 +202,7 @@ void plugin_open(const char *filename, int info[1024], int *error_flag) { ERROR_JUMP(-4, done, ""); } - data_desc->image_number_offset = 0; - char *cenv; - cenv = getenv("DURIN_IMAGE_NUMBER_OFFSET"); - if (cenv!=NULL) { - data_desc->image_number_offset = atoi(cenv); - } + data_desc->image_number_offset = info[5]; mask_buffer = malloc(data_desc->dims[1] * data_desc->dims[2] * sizeof(int)); if (mask_buffer) { @@ -172,7 +245,7 @@ void plugin_get_header(int *nx, int *ny, int *nbytes, float *qx, float *qy, *nx = data_desc->dims[2]; *ny = data_desc->dims[1]; - *nbytes = data_desc->data_width; + *nbytes = abs(data_desc->data_width); *number_of_frames = data_desc->dims[0]; *qx = (float)x_pixel_size; *qy = (float)y_pixel_size; @@ -193,10 +266,10 @@ void plugin_get_data(int *frame_number, int *nx, int *ny, int *data_array, fill_info_array(info); void *buffer = NULL; - if (sizeof(*data_array) == data_desc->data_width) { + if (sizeof(*data_array) == abs(data_desc->data_width)) { buffer = data_array; } else { - buffer = malloc(data_desc->data_width * frame_size_px); + buffer = malloc(abs(data_desc->data_width) * frame_size_px); if (!buffer) { ERROR_JUMP(-1, done, "Unable to allocate data buffer"); } @@ -209,7 +282,7 @@ void plugin_get_data(int *frame_number, int *nx, int *ny, int *data_array, } if (buffer != data_array) { - if (convert_to_int_and_mask(buffer, data_desc->data_width, data_array, + if (convert_to_int_and_mask(buffer, data_desc->data_width, info[6], data_array, frame_size_px, mask_buffer) < 0) { char message[64]; sprintf(message, "Error converting data for frame %d", *frame_number); From 163d5bdfb1d3f3549be3c7c60426a04361a6da2e Mon Sep 17 00:00:00 2001 From: CV-GPhL Date: Thu, 14 Mar 2024 16:42:52 +0100 Subject: [PATCH 3/7] Update plugin.c to write compilation time if available/set --- src/plugin.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugin.c b/src/plugin.c index f45a1d9..8afa637 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -218,6 +218,10 @@ void plugin_open(const char *filename, int info[1024], int *error_flag) { } retval = 0; +#ifdef GPHL_COMPILE_DATE + fprintf(ERROR_OUTPUT, "\n XDS HDF5/Durin plugin %d.%d.%d (DLS, 2018-2023; GPhL, 2020-2024 - built %d)\n", info[1], info[2], info[3], GPHL_COMPILE_DATE); +#endif + done: *error_flag = retval; if (retval < 0) { From 44ba3cd278322eefbc2da3cb7d46a7c5d9768501 Mon Sep 17 00:00:00 2001 From: Clemens Vonrhein Date: Mon, 16 Dec 2024 14:57:32 +0000 Subject: [PATCH 4/7] Provide a mechanism to distinguish between an image number (as understood on the XDS side) and the image ordinal (as stored in the HDF data). The 2D data arrays are not necessarily at image_nr_low=1 ;-) --- src/file.h | 1 + src/plugin.c | 130 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 128 insertions(+), 3 deletions(-) diff --git a/src/file.h b/src/file.h index 0bcc36e..6208d4c 100644 --- a/src/file.h +++ b/src/file.h @@ -20,6 +20,7 @@ struct ds_desc_t { int (*get_pixel_mask)(const struct ds_desc_t *, int *); int (*get_data_frame)(const struct ds_desc_t *, const int, void *); void (*free_desc)(struct ds_desc_t *); + int i2i[]; // array to hold a translation from the image number requested by XDS and the actual position in the HDF5 file }; struct nxs_ds_desc_t { diff --git a/src/plugin.c b/src/plugin.c index 8afa637..0f5c3e9 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -5,6 +5,7 @@ #include #include +#include #include "file.h" #include "filters.h" @@ -69,6 +70,17 @@ static hid_t file_id = 0; static struct ds_desc_t *data_desc = NULL; static int *mask_buffer = NULL; +// CV-20240605: potentially provide a mapping from frame number (as +// requested by caller) to actual 2D slice within 3D data +// array. +// +// This is defined by the environment variable +// DURIN_IMAGE2ORDINAL (see below). +int *image2ordinal = NULL; +int image2ordinal_debug = 0; +int image2ordinal_imin = 0; +int image2ordinal_imax = 0; + void fill_info_array(int info[1024]) { info[0] = DLS_CUSTOMER_ID; info[1] = VERSION_MAJOR; @@ -88,6 +100,92 @@ void fill_info_array(int info[1024]) { info[6] = atoi(cenv); } + cenv = getenv("DURIN_IMAGE2ORDINAL"); + if (cenv!=NULL&&(!image2ordinal)) { + + char *denv = getenv("DURIN_IMAGE2ORDINAL_DEBUG"); + if (denv!=NULL) { + image2ordinal_debug=1; + } + + // ,-,-,..,- + if (image2ordinal_debug) printf("DURIN_IMAGE2ORDINAL = \"%s\"\n",cenv); + + const char outer_delimiters[] = ","; + const char inner_delimiters[] = "-"; + + char *found; + + char *outer_saveptr; + char *inner_saveptr; + + int ordinal_start = 0; + int ordinal = 0; + int ntt = -1; + found = strtok_r(cenv,outer_delimiters, &outer_saveptr); + if (found!=NULL) { + int tt[1000][2]; + while(found) { + if (ordinal_start==0) { + ordinal_start = atoi(found); + ordinal = ordinal_start - 1; + } + else { + char* s = strtok_r(found, inner_delimiters, &inner_saveptr); + if (s!=NULL) { + int i1 = atoi(s); + s = strtok_r(NULL,inner_delimiters, &inner_saveptr); + if (s!=NULL) { + int i2 = atoi(s); + ntt++; + if (ntt<=1000) { + tt[ntt][0] = i1; + tt[ntt][1] = i2; + for(int i=i1; i<=i2; ++i) { + ordinal++; + if (ordinal==1) { + image2ordinal_imin=i; + image2ordinal_imax=i; + } + else { + if (iimage2ordinal_imax) image2ordinal_imax=i; + } + } + } + } + } + } + found = strtok_r(NULL,outer_delimiters,&outer_saveptr); + } + + if (ordinal_start>0) { + if (image2ordinal_debug) { + printf("ordinal_start, end = %d %d\n",ordinal_start, ordinal); + printf("imin, imax = %d %d\n",image2ordinal_imin,image2ordinal_imax); + } + + // allocate array to go from image number/id to ordinal: + image2ordinal = malloc((image2ordinal_imax-image2ordinal_imin+1) * sizeof(image2ordinal_imin)); + int ordinal = ordinal_start - 1; + for(int i=0; i<=ntt; i++) { + for(int j=tt[i][0];j<=tt[i][1];j++) { + ordinal++; + //printf(" %d -> %d\n",ordinal,j); + image2ordinal[j] = ordinal; + } + } + if (image2ordinal&&image2ordinal_debug) { + for(int i=image2ordinal_imin; i<=image2ordinal_imax; i++) { + if (image2ordinal[i]>0) { + printf(" %d -> %d\n",i,image2ordinal[i]); + } + } + } + } + } + } + } int convert_to_int_and_mask(void *in_buffer, int width, int setValue, int *out_buffer, @@ -279,9 +377,26 @@ void plugin_get_data(int *frame_number, int *nx, int *ny, int *data_array, } } - if (data_desc->get_data_frame(data_desc, (*frame_number) - 1, buffer) < 0) { + int ordinal = *frame_number; + if (image2ordinal) { + if (ordinal < image2ordinal_imin || ordinal>image2ordinal_imax) { + char message[64] = {0}; + sprintf(message, "Failed to map frame %d to ordinals since outside of range %d - %d", ordinal,image2ordinal_imin,image2ordinal_imax); + ERROR_JUMP(-2, done, message); + } + ordinal = image2ordinal[ordinal]; + if (ordinal!=*frame_number) { + if (image2ordinal_debug) printf("fetching data from ordinal %d for frame %d\n",ordinal,*frame_number); + } + } + + if (data_desc->get_data_frame(data_desc, ordinal - 1, buffer) < 0) { char message[64] = {0}; - sprintf(message, "Failed to retrieve data for frame %d", *frame_number); + if (image2ordinal) { + sprintf(message, "Failed to retrieve data for frame %d (ordinal %d)", *frame_number, ordinal); + } else { + sprintf(message, "Failed to retrieve data for frame %d", *frame_number); + } ERROR_JUMP(-2, done, message); } @@ -289,7 +404,11 @@ void plugin_get_data(int *frame_number, int *nx, int *ny, int *data_array, if (convert_to_int_and_mask(buffer, data_desc->data_width, info[6], data_array, frame_size_px, mask_buffer) < 0) { char message[64]; - sprintf(message, "Error converting data for frame %d", *frame_number); + if (image2ordinal) { + sprintf(message, "Error converting data for frame %d (ordinal %d)", *frame_number, ordinal); + } else { + sprintf(message, "Error converting data for frame %d", *frame_number); + } ERROR_JUMP(-2, done, message); } } else { @@ -314,6 +433,11 @@ void plugin_close(int *error_flag) { } file_id = 0; + if (image2ordinal) { + free(image2ordinal); + image2ordinal = NULL; + } + if (mask_buffer) { free(mask_buffer); mask_buffer = NULL; From 4cf18ceff4f8395ebfeaed927d7f30cdb2c7d461 Mon Sep 17 00:00:00 2001 From: Clemens Vonrhein Date: Mon, 16 Dec 2024 15:05:38 +0000 Subject: [PATCH 5/7] Mechanism to read bitshuffle-compressed pixel-mask data - by loading/using the normal bitshuffle source from https://github.com/kiyo-masui/bitshuffle (see Makefile). --- Makefile | 20 ++++++++++++++++++-- src/file.c | 26 +++++++++++++++++++++++++- src/plugin.c | 10 ++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4387348..0a6b257 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,19 @@ BSLZ4_BUILD_DIR = ./bslz4/build BSLZ4_INC_DIR = $(BSLZ4_SRC_DIR) CC=h5cc -CFLAGS=-DH5_USE_110_API -Wall -g -O2 -fpic -I$(INC_DIR) -I$(BSLZ4_INC_DIR) -std=c99 -shlib +# -std=gnu99 provides for strtok_r +CFLAGS=-DH5_USE_110_API -Wall -g -O2 -fpic -I$(INC_DIR) -I$(BSLZ4_INC_DIR) -std=gnu99 -shlib + +# include https://github.com/kiyo-masui/bitshuffle (to handle +# e.g. bitshuffle compressed pixel_masks) +use_BITSHUFFLE = +ifeq ($(use_BITSHUFFLE),) +else +BITSHUFFLE_SRC_DIR = ../bitshuffle-master/src/ +BITSHUFFLE_INC_DIR = ../bitshuffle-master/src/ +BITSHUFFLE_OBJS = $(BUILD_DIR)/bshuf_h5filter.o +CFLAGS += -DUSE_BITSHUFFLE -I$(BITSHUFFLE_INC_DIR) +endif .PHONY: plugin plugin: $(BUILD_DIR)/durin-plugin.so @@ -26,6 +38,10 @@ $(BUILD_DIR)/test_plugin: $(TEST_DIR)/generic_data_plugin.f90 $(TEST_DIR)/test_g mkdir -p $(BUILD_DIR) gfortran -O -g -fopenmp -ldl $(TEST_DIR)/generic_data_plugin.f90 $(TEST_DIR)/test_generic_host.f90 -o $@ -J$(BUILD_DIR) +$(BUILD_DIR)/bshuf_h5filter.o: $(BITSHUFFLE_SRC_DIR)/bshuf_h5filter.c + mkdir -p $(BUILD_DIR) + $(CC) $(CFLAGS) -c $< -o $@ + $(BUILD_DIR)/%.o: $(SRC_DIR)/%.c mkdir -p $(BUILD_DIR) $(CC) $(CFLAGS) -c $< -o $@ @@ -39,7 +55,7 @@ $(BSLZ4_BUILD_DIR)/bitshuffle_core.o $(BSLZ4_BUILD_DIR)/iochain.o mkdir -p $(BUILD_DIR) ar rcs $@ $^ -$(BUILD_DIR)/durin-plugin.so: $(BUILD_DIR)/plugin.o $(BUILD_DIR)/file.o $(BUILD_DIR)/err.o $(BUILD_DIR)/filters.o \ +$(BUILD_DIR)/durin-plugin.so: $(BUILD_DIR)/plugin.o $(BUILD_DIR)/file.o $(BUILD_DIR)/err.o $(BUILD_DIR)/filters.o $(BITSHUFFLE_OBJS) \ $(BUILD_DIR)/bslz4.a mkdir -p $(BUILD_DIR) $(CC) $(CFLAGS) -shared -noshlib $^ -o $(BUILD_DIR)/durin-plugin.so diff --git a/src/file.c b/src/file.c index 7a6017d..85bf2f8 100644 --- a/src/file.c +++ b/src/file.c @@ -573,9 +573,33 @@ int get_dectris_eiger_pixel_mask(const struct ds_desc_t *desc, int *buffer) { ERROR_JUMP(-1, done, "Error opening detectorSpecific/pixel_mask"); } + // what if this is compressed? + hid_t dcpl = H5Dget_create_plist(ds_id); + int n_filters = H5Pget_nfilters(dcpl); + H5Z_filter_t filter_id; + if (n_filters>0) { + unsigned int flags; + size_t nelmts = 1; + unsigned int values_out[1] = {99}; + char filter_name[80]; + filter_id = H5Pget_filter(dcpl, (unsigned) 0, &flags, &nelmts, values_out, sizeof(filter_name), filter_name, NULL); + if (filter_id>=0) { + fprintf(stderr," filter name =\"%s\"\n",filter_name); + } + } + err = H5Dread(ds_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer); if (err < 0) { - ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask"); + if (n_filters>0) { + ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask with filter(s)"); + } + else { + ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask"); + } + } + + if (H5Zfilter_avail(BS_H5_FILTER_ID)) { + fprintf(stderr," bitshuffle filter is available now since H5Dread (of pixel-mask) triggered loading of the filter.\n"); } close_dataset: diff --git a/src/plugin.c b/src/plugin.c index 0f5c3e9..9c2e3b3 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -11,6 +11,10 @@ #include "filters.h" #include "plugin.h" +#ifdef USE_BITSHUFFLE +#include "bshuf_h5filter.h" +#endif + /* XDS does not provide an error callback facility, so just write to stderr for now - generally regarded as poor practice */ #define ERROR_OUTPUT stderr @@ -286,6 +290,12 @@ void plugin_open(const char *filename, int info[1024], int *error_flag) { ERROR_JUMP(-2, done, "Failed to configure HDF5 error handling"); } +#ifdef USE_BITSHUFFLE + if (bshuf_register_h5filter() < 0 ) { + ERROR_JUMP(-2, done, "Failed to register bitshuffle filter"); + } +#endif + fill_info_array(info); file_id = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); if (file_id < 0) { From d7801cdb549c6ed85713df3b52fe08387c8c2bf5 Mon Sep 17 00:00:00 2001 From: Clemens Vonrhein Date: Tue, 17 Dec 2024 09:59:55 +0000 Subject: [PATCH 6/7] modified reading of NX_class attribute - to accommodate both DATATYPE H5T_STRING { STRSIZE 14; STRPAD H5T_STR_NULLTERM; CSET H5T_CSET_ASCII; CTYPE H5T_C_S1; } and DATATYPE H5T_STRING { STRSIZE H5T_VARIABLE; STRPAD H5T_STR_NULLTERM; CSET H5T_CSET_UTF8; CTYPE H5T_C_S1; } --- src/file.c | 97 ++++++++++++++++++++---------------------------------- 1 file changed, 36 insertions(+), 61 deletions(-) diff --git a/src/file.c b/src/file.c index 85bf2f8..7533566 100644 --- a/src/file.c +++ b/src/file.c @@ -578,28 +578,28 @@ int get_dectris_eiger_pixel_mask(const struct ds_desc_t *desc, int *buffer) { int n_filters = H5Pget_nfilters(dcpl); H5Z_filter_t filter_id; if (n_filters>0) { - unsigned int flags; - size_t nelmts = 1; - unsigned int values_out[1] = {99}; - char filter_name[80]; - filter_id = H5Pget_filter(dcpl, (unsigned) 0, &flags, &nelmts, values_out, sizeof(filter_name), filter_name, NULL); - if (filter_id>=0) { - fprintf(stderr," filter name =\"%s\"\n",filter_name); - } + unsigned int flags; + size_t nelmts = 1; + unsigned int values_out[1] = {99}; + char filter_name[80]; + filter_id = H5Pget_filter(dcpl, (unsigned) 0, &flags, &nelmts, values_out, sizeof(filter_name), filter_name, NULL); + if (filter_id>=0) { + fprintf(stderr," filter name =\"%s\"\n",filter_name); + } } err = H5Dread(ds_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer); if (err < 0) { - if (n_filters>0) { - ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask with filter(s)"); - } - else { - ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask"); - } + if (n_filters>0) { + ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask with filter(s)"); + } + else { + ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask"); + } } if (H5Zfilter_avail(BS_H5_FILTER_ID)) { - fprintf(stderr," bitshuffle filter is available now since H5Dread (of pixel-mask) triggered loading of the filter.\n"); + fprintf(stderr," bitshuffle filter is available now since H5Dread (of pixel-mask) triggered loading of the filter.\n"); } close_dataset: @@ -630,9 +630,9 @@ herr_t det_visit_callback(hid_t root_id, const char *name, /* check for an "NX_class" attribute */ { - int str_size = 0; - void *buffer = NULL; - hid_t a_id, t_id, mt_id; + char* buffer = (char*)malloc(1); + buffer[0] = '\0'; + hid_t a_id, t_id; if (H5Aexists(g_id, "NX_class") <= 0) { /* not an error - just close group and allow continuation */ retval = 0; @@ -652,70 +652,45 @@ herr_t det_visit_callback(hid_t root_id, const char *name, if (t_id < 0) { ERROR_JUMP(-1, close_attr, "Error getting datatype"); } - if (H5Tis_variable_str(t_id) > 0) { - str_size = -1; - buffer = malloc(sizeof(char *)); - } else { - str_size = H5Tget_size(t_id); - buffer = malloc(str_size + 1); + + H5A_info_t a_info; + herr_t err = H5Aget_info(a_id, &a_info); + if (err<0) { + ERROR_JUMP(-1, close_type, + "Unable to get attribute info for NX_class"); } - if (!buffer) { - ERROR_JUMP(-1, close_type, "Error allocating string buffer"); + else { + if (a_info.cset != H5T_CSET_ASCII && a_info.cset != H5T_CSET_UTF8) { + fprintf(stderr," %s : NX_class attribute info cset = unknown with size %d\n",name,(int) a_info.data_size); + } } - mt_id = H5Tcopy(H5T_C_S1); - if (mt_id < 0) { - ERROR_JUMP(-1, free_buffer, "Error creating HDF5 String datatype"); - } - // set the target string type to be one longer than the recorded string - // in keeping with the malloc'd buffer - if this is already a null - // terminated string then we will just have two nulls, if it is not - // then we won't clobber the last char in the buffer with a null in the - // H5Aread call - if (H5Tset_size(mt_id, str_size == -1 ? H5T_VARIABLE : str_size + 1) < 0) { - char message[64]; - sprintf(message, "Error setting string datatype to size %d", str_size); - ERROR_JUMP(-1, close_mtype, message); + buffer = (char *)malloc(sizeof(char)*(H5Tget_size(t_id)+1)); + if (!buffer) { + ERROR_JUMP(-1, close_type, "Error allocating string buffer"); } - if (H5Aread(a_id, mt_id, buffer) < 0) { + if (H5Aread(a_id, t_id, buffer) < 0) { char message[256]; sprintf( message, "H5OVisit callback: Error reading NX_class attribute on group %.128s", name); - ERROR_JUMP(-1, close_mtype, message); + ERROR_JUMP(-1, free_buffer, message); } - /* at least one file has been seen where the NX_class attribute was not null - * terminated and extraneous bytes where being read by strcmp - set the end - * byte to null - */ - - if (str_size > 0) - ((char *)buffer)[str_size] = '\0'; /* test for NXdata or NXdetector */ { - char *nxclass = str_size > 0 ? (char *)buffer : *((char **)buffer); - if (strcmp("NXdata", nxclass) == 0) { + if (strcmp("NXdata", buffer) == 0) { hid_t out_id = H5Gopen(root_id, name, H5P_DEFAULT); output_data->nxdata = out_id; - } else if (strcmp("NXdetector", nxclass) == 0) { + } + else if (strcmp("NXdetector", buffer) == 0) { hid_t out_id = H5Gopen(root_id, name, H5P_DEFAULT); output_data->nxdetector = out_id; } } - if (str_size == -1) { - hsize_t dims[1] = {1}; - hid_t s_id = H5Screate_simple(1, dims, NULL); - H5Sselect_all(s_id); - H5Dvlen_reclaim(mt_id, s_id, H5P_DEFAULT, buffer); - H5Sclose(s_id); - } - - close_mtype: - H5Tclose(mt_id); free_buffer: free(buffer); close_type: From cfe0e783199ff60ad765a2513b148cbfafbae616 Mon Sep 17 00:00:00 2001 From: Clemens Vonrhein Date: Wed, 18 Dec 2024 16:46:14 +0000 Subject: [PATCH 7/7] loop over all filters registered when printing --- src/file.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/file.c b/src/file.c index 7533566..a279768 100644 --- a/src/file.c +++ b/src/file.c @@ -582,12 +582,16 @@ int get_dectris_eiger_pixel_mask(const struct ds_desc_t *desc, int *buffer) { size_t nelmts = 1; unsigned int values_out[1] = {99}; char filter_name[80]; - filter_id = H5Pget_filter(dcpl, (unsigned) 0, &flags, &nelmts, values_out, sizeof(filter_name), filter_name, NULL); - if (filter_id>=0) { - fprintf(stderr," filter name =\"%s\"\n",filter_name); + for ( int i_filt = 0; i_filt < n_filters; i_filt++) { + filter_id = H5Pget_filter(dcpl, i_filt, &flags, &nelmts, values_out, sizeof(filter_name), filter_name, NULL); + if (filter_id>=0) { + fprintf(stderr," filter #%d name =\"%s\"\n",(i_filt+1),filter_name); + } } } + int i0 = H5Zfilter_avail(BS_H5_FILTER_ID); + err = H5Dread(ds_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer); if (err < 0) { if (n_filters>0) { @@ -598,7 +602,7 @@ int get_dectris_eiger_pixel_mask(const struct ds_desc_t *desc, int *buffer) { } } - if (H5Zfilter_avail(BS_H5_FILTER_ID)) { + if (!i0 && H5Zfilter_avail(BS_H5_FILTER_ID)) { fprintf(stderr," bitshuffle filter is available now since H5Dread (of pixel-mask) triggered loading of the filter.\n"); }