Skip to content

Three modifications: Handling of overloaded pixels, signed data arrays and image number offsets #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
CV-GPhL wants to merge 10 commits into
base: master
Choose a base branch
from
Open

Three modifications: Handling of overloaded pixels, signed data arrays and image number offsets #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9e7d609
allow for image offset via DURIN_IMAGE_NUMBER_OFFSET environment
CV-GPhL Jun 12, 2020
0f27832
Resolved merge conflicts
CV-GPhL Apr 6, 2021
77d2b84
Better handling of (usually) unsigned data arrays when calling routine
CV-GPhL Apr 8, 2021
ca2c0ee
Merge pull request #1 from DiamondLightSource/master
CV-GPhL Feb 23, 2024
163d5bd
Update plugin.c to write compilation time if available/set
CV-GPhL Mar 14, 2024
ca8806c
Merge pull request #2 from CV-GPhL/CV-GPhL-patch-1
CV-GPhL Mar 14, 2024
44ba3cd
Provide a mechanism to distinguish between an image number (as
CV-GPhL Dec 16, 2024
4cf18ce
Mechanism to read bitshuffle-compressed pixel-mask data - by
CV-GPhL Dec 16, 2024
d7801cd
modified reading of NX_class attribute - to accommodate both
CV-GPhL Dec 17, 2024
cfe0e78
loop over all filters registered when printing
CV-GPhL Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,19 @@ BSLZ4_BUILD_DIR = ./bslz4/build
BSLZ4_INC_DIR = $(BSLZ4_SRC_DIR)

CC=h5cc
CFLAGS=-DH5_USE_110_API -Wall -g -O2 -fpic -I$(INC_DIR) -I$(BSLZ4_INC_DIR) -std=c99 -shlib
# -std=gnu99 provides for strtok_r
CFLAGS=-DH5_USE_110_API -Wall -g -O2 -fpic -I$(INC_DIR) -I$(BSLZ4_INC_DIR) -std=gnu99 -shlib

# include https://github.com/kiyo-masui/bitshuffle (to handle
# e.g. bitshuffle compressed pixel_masks)
use_BITSHUFFLE =
ifeq ($(use_BITSHUFFLE),)
else
BITSHUFFLE_SRC_DIR = ../bitshuffle-master/src/
BITSHUFFLE_INC_DIR = ../bitshuffle-master/src/
BITSHUFFLE_OBJS = $(BUILD_DIR)/bshuf_h5filter.o
CFLAGS += -DUSE_BITSHUFFLE -I$(BITSHUFFLE_INC_DIR)
endif

.PHONY: plugin
plugin: $(BUILD_DIR)/durin-plugin.so
Expand All @@ -26,6 +38,10 @@ $(BUILD_DIR)/test_plugin: $(TEST_DIR)/generic_data_plugin.f90 $(TEST_DIR)/test_g
mkdir -p $(BUILD_DIR)
gfortran -O -g -fopenmp -ldl $(TEST_DIR)/generic_data_plugin.f90 $(TEST_DIR)/test_generic_host.f90 -o $@ -J$(BUILD_DIR)

$(BUILD_DIR)/bshuf_h5filter.o: $(BITSHUFFLE_SRC_DIR)/bshuf_h5filter.c
mkdir -p $(BUILD_DIR)
$(CC) $(CFLAGS) -c $< -o $@

$(BUILD_DIR)/%.o: $(SRC_DIR)/%.c
mkdir -p $(BUILD_DIR)
$(CC) $(CFLAGS) -c $< -o $@
Expand All @@ -39,7 +55,7 @@ $(BSLZ4_BUILD_DIR)/bitshuffle_core.o $(BSLZ4_BUILD_DIR)/iochain.o
mkdir -p $(BUILD_DIR)
ar rcs $@ $^

$(BUILD_DIR)/durin-plugin.so: $(BUILD_DIR)/plugin.o $(BUILD_DIR)/file.o $(BUILD_DIR)/err.o $(BUILD_DIR)/filters.o \
$(BUILD_DIR)/durin-plugin.so: $(BUILD_DIR)/plugin.o $(BUILD_DIR)/file.o $(BUILD_DIR)/err.o $(BUILD_DIR)/filters.o $(BITSHUFFLE_OBJS) \
$(BUILD_DIR)/bslz4.a
mkdir -p $(BUILD_DIR)
$(CC) $(CFLAGS) -shared -noshlib $^ -o $(BUILD_DIR)/durin-plugin.so
Expand Down
117 changes: 66 additions & 51 deletions src/file.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,13 @@ int get_nxs_dataset_dims(struct ds_desc_t *desc) {
ERROR_JUMP(-1, close_space, "Error getting dataset dimensions");
}

desc->data_width = width;
if ( H5Tequal(t_id,H5T_NATIVE_CHAR)>0 || H5Tequal(t_id,H5T_NATIVE_INT)>0 || H5Tequal(t_id,H5T_NATIVE_SHORT)>0 || H5Tequal(t_id,H5T_NATIVE_LONG)>0 || H5Tequal(t_id,H5T_NATIVE_LLONG)>0 ) {
// signed
desc->data_width = -width;
} else {
// unsigned
desc->data_width = width;
}

close_space:
H5Sclose(s_id);
Expand Down Expand Up @@ -233,7 +239,7 @@ int get_frame_from_chunk(const struct ds_desc_t *desc, const char *ds_name,

if (o_eiger_desc->bs_applied) {
if (bslz4_decompress(o_eiger_desc->bs_params, c_bytes, c_buffer,
desc->data_width * frame_size[1] * frame_size[2],
abs(desc->data_width) * frame_size[1] * frame_size[2],
buffer) < 0) {
char message[128];
sprintf(message,
Expand All @@ -251,10 +257,11 @@ int get_frame_from_chunk(const struct ds_desc_t *desc, const char *ds_name,
return retval;
}

int get_nxs_frame(const struct ds_desc_t *desc, const int n, void *buffer) {
int get_nxs_frame(const struct ds_desc_t *desc, const int nin, void *buffer) {
/* detector data are the two inner most indices */
/* TODO: handle ndims > 3 and select appropriately */
int retval = 0;
int n = nin - desc->image_number_offset;
hsize_t frame_idx[3] = {n, 0, 0};
hsize_t frame_size[3] = {1, desc->dims[1], desc->dims[2]};
if (n < 0 || n >= desc->dims[0]) {
Expand All @@ -271,9 +278,10 @@ int get_nxs_frame(const struct ds_desc_t *desc, const int n, void *buffer) {
return retval;
}

int get_dectris_eiger_frame(const struct ds_desc_t *desc, int n, void *buffer) {
int get_dectris_eiger_frame(const struct ds_desc_t *desc, int nin, void *buffer) {

int retval = 0;
int n = nin - desc->image_number_offset;
int block, frame_count, idx;
struct eiger_ds_desc_t *eiger_desc = (struct eiger_ds_desc_t *)desc;
char data_name[16] = {0};
Expand Down Expand Up @@ -349,6 +357,10 @@ int get_dectris_eiger_dataset_dims(struct ds_desc_t *desc) {
if (data_width <= 0) {
ERROR_JUMP(-1, close_space, "Unable to get type size");
}
if ( H5Tequal(t_id,H5T_NATIVE_CHAR)>0 || H5Tequal(t_id,H5T_NATIVE_INT)>0 || H5Tequal(t_id,H5T_NATIVE_SHORT)>0 || H5Tequal(t_id,H5T_NATIVE_LONG)>0 || H5Tequal(t_id,H5T_NATIVE_LLONG)>0 ) {
// signed
data_width = -data_width;
}

ndims = H5Sget_simple_extent_ndims(s_id);
if (ndims != 3) {
Expand Down Expand Up @@ -561,9 +573,37 @@ int get_dectris_eiger_pixel_mask(const struct ds_desc_t *desc, int *buffer) {
ERROR_JUMP(-1, done, "Error opening detectorSpecific/pixel_mask");
}

// what if this is compressed?
hid_t dcpl = H5Dget_create_plist(ds_id);
int n_filters = H5Pget_nfilters(dcpl);
H5Z_filter_t filter_id;
if (n_filters>0) {
unsigned int flags;
size_t nelmts = 1;
unsigned int values_out[1] = {99};
char filter_name[80];
for ( int i_filt = 0; i_filt < n_filters; i_filt++) {
filter_id = H5Pget_filter(dcpl, i_filt, &flags, &nelmts, values_out, sizeof(filter_name), filter_name, NULL);
if (filter_id>=0) {
fprintf(stderr," filter #%d name =\"%s\"\n",(i_filt+1),filter_name);
}
}
}

int i0 = H5Zfilter_avail(BS_H5_FILTER_ID);

err = H5Dread(ds_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer);
if (err < 0) {
ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask");
if (n_filters>0) {
ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask with filter(s)");
}
else {
ERROR_JUMP(-1, close_dataset, "Error reading detectorSpecific/pixel_mask");
}
}

if (!i0 && H5Zfilter_avail(BS_H5_FILTER_ID)) {
fprintf(stderr," bitshuffle filter is available now since H5Dread (of pixel-mask) triggered loading of the filter.\n");
}

close_dataset:
Expand Down Expand Up @@ -594,9 +634,9 @@ herr_t det_visit_callback(hid_t root_id, const char *name,

/* check for an "NX_class" attribute */
{
int str_size = 0;
void *buffer = NULL;
hid_t a_id, t_id, mt_id;
char* buffer = (char*)malloc(1);
buffer[0] = '\0';
hid_t a_id, t_id;
if (H5Aexists(g_id, "NX_class") <= 0) {
/* not an error - just close group and allow continuation */
retval = 0;
Expand All @@ -616,70 +656,45 @@ herr_t det_visit_callback(hid_t root_id, const char *name,
if (t_id < 0) {
ERROR_JUMP(-1, close_attr, "Error getting datatype");
}
if (H5Tis_variable_str(t_id) > 0) {
str_size = -1;
buffer = malloc(sizeof(char *));
} else {
str_size = H5Tget_size(t_id);
buffer = malloc(str_size + 1);

H5A_info_t a_info;
herr_t err = H5Aget_info(a_id, &a_info);
if (err<0) {
ERROR_JUMP(-1, close_type,
"Unable to get attribute info for NX_class");
}
if (!buffer) {
ERROR_JUMP(-1, close_type, "Error allocating string buffer");
else {
if (a_info.cset != H5T_CSET_ASCII && a_info.cset != H5T_CSET_UTF8) {
fprintf(stderr," %s : NX_class attribute info cset = unknown with size %d\n",name,(int) a_info.data_size);
}
}

mt_id = H5Tcopy(H5T_C_S1);
if (mt_id < 0) {
ERROR_JUMP(-1, free_buffer, "Error creating HDF5 String datatype");
}
// set the target string type to be one longer than the recorded string
// in keeping with the malloc'd buffer - if this is already a null
// terminated string then we will just have two nulls, if it is not
// then we won't clobber the last char in the buffer with a null in the
// H5Aread call
if (H5Tset_size(mt_id, str_size == -1 ? H5T_VARIABLE : str_size + 1) < 0) {
char message[64];
sprintf(message, "Error setting string datatype to size %d", str_size);
ERROR_JUMP(-1, close_mtype, message);
buffer = (char *)malloc(sizeof(char)*(H5Tget_size(t_id)+1));
if (!buffer) {
ERROR_JUMP(-1, close_type, "Error allocating string buffer");
}

if (H5Aread(a_id, mt_id, buffer) < 0) {
if (H5Aread(a_id, t_id, buffer) < 0) {
char message[256];
sprintf(
message,
"H5OVisit callback: Error reading NX_class attribute on group %.128s",
name);
ERROR_JUMP(-1, close_mtype, message);
ERROR_JUMP(-1, free_buffer, message);
}

/* at least one file has been seen where the NX_class attribute was not null
* terminated and extraneous bytes where being read by strcmp - set the end
* byte to null
*/

if (str_size > 0)
((char *)buffer)[str_size] = '\0';
/* test for NXdata or NXdetector */
{
char *nxclass = str_size > 0 ? (char *)buffer : *((char **)buffer);
if (strcmp("NXdata", nxclass) == 0) {
if (strcmp("NXdata", buffer) == 0) {
hid_t out_id = H5Gopen(root_id, name, H5P_DEFAULT);
output_data->nxdata = out_id;
} else if (strcmp("NXdetector", nxclass) == 0) {
}
else if (strcmp("NXdetector", buffer) == 0) {
hid_t out_id = H5Gopen(root_id, name, H5P_DEFAULT);
output_data->nxdetector = out_id;
}
}

if (str_size == -1) {
hsize_t dims[1] = {1};
hid_t s_id = H5Screate_simple(1, dims, NULL);
H5Sselect_all(s_id);
H5Dvlen_reclaim(mt_id, s_id, H5P_DEFAULT, buffer);
H5Sclose(s_id);
}

close_mtype:
H5Tclose(mt_id);
free_buffer:
free(buffer);
close_type:
Expand Down
2 changes: 2 additions & 0 deletions src/file.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@ struct ds_desc_t {
hid_t data_g_id;
hsize_t dims[3];
int data_width;
int image_number_offset;
int (*get_pixel_properties)(const struct ds_desc_t *, double *, double *);
int (*get_pixel_mask)(const struct ds_desc_t *, int *);
int (*get_data_frame)(const struct ds_desc_t *, const int, void *);
void (*free_desc)(struct ds_desc_t *);
int i2i[]; // array to hold a translation from the image number requested by XDS and the actual position in the HDF5 file
};

struct nxs_ds_desc_t {
Expand Down
Loading