| import pickle | |
| import h5py | |
| def save_pkl(filename, save_object): | |
| writer = open(filename,'wb') | |
| pickle.dump(save_object, writer) | |
| writer.close() | |
| def load_pkl(filename): | |
| loader = open(filename,'rb') | |
| file = pickle.load(loader) | |
| loader.close() | |
| return file | |
| def save_hdf5(output_path, asset_dict, attr_dict= None, mode='a'): | |
| file = h5py.File(output_path, mode) | |
| for key, val in asset_dict.items(): | |
| data_shape = val.shape | |
| if key not in file: | |
| data_type = val.dtype | |
| chunk_shape = (1, ) + data_shape[1:] | |
| maxshape = (None, ) + data_shape[1:] | |
| dset = file.create_dataset(key, shape=data_shape, maxshape=maxshape, chunks=chunk_shape, dtype=data_type) | |
| dset[:] = val | |
| if attr_dict is not None: | |
| if key in attr_dict.keys(): | |
| for attr_key, attr_val in attr_dict[key].items(): | |
| dset.attrs[attr_key] = attr_val | |
| else: | |
| dset = file[key] | |
| dset.resize(len(dset) + data_shape[0], axis=0) | |
| dset[-data_shape[0]:] = val | |
| file.close() | |
| return output_path |