#include #include #include #include #include #include "nagg.h" #define GEO_Filename_size 79 #define GEO_FilePath_size 255 #define GEOPRODUCT_UNINIT "Uninitialized GEO Product " #define Group_path_size 79 #define GEO_Filename_max_matches 16 typedef struct { char *filepath; const char *dpid; /* the group name is also a product type */ char product_name[NAGG_Product_Type_size]; int granule_input_index; hid_t grpid; char *geofile; granule_t **granArr; /* number of granules added to the granule table */ int gran_num; } GranF_data; /* global variable to track total number of all granules. Used as granule index. */ static int g_gran_num; static int get_n_geo_ref_value(hid_t file, char *geofile_buf); static int get_group_granules (hid_t loc_id, const char *name, void *operator_data); static int get_granule_info (hid_t loc_id, const char *name, void *operator_data); static int get_string_attribute_value(hid_t loc_id, const char * dsetname, const char *attrname, void *buf, hid_t aapl_id); static int find_geo_granules(char **geofile_list, int number_of_files, char *geoproduct, geolocation_t geofiles_arg, granule_p_t *granule_info_p[]); static int get_geo_granules(hid_t file, char *geofile, char *geoproduct, granule_p_t *granule_info_p[]); static int set_granule_pattern(granule_p_t granule); static int group_finder (hid_t loc_id, const char *name, const H5L_info_t *info, void *operator_data); /*------------------------------------------------------------------------- * Function: set_granule_pattern * * Purpose: Add a pattern granule for a product to the granule_pattern_array. * * Parameter: * granule: granule structure to be entered into granule table * * Notes: Selected information from the first granule found for each product * is saved in an granule_pattern structure to insure that information * is available for writing fill granules. * * Keep track of whether a granule pattern has been saved for a product. * If it has, return 0 but don't save the new one. If it hasn't save * the needed information in the structure. * * Return: Success: 0 * Failure: -1 * * Programmer: Larry Knox * March 22, 2012 *------------------------------------------------------------------------- */ static int set_granule_pattern(granule_p_t granule) { int status = SUCCEED; int i, found = 0; static int last_found; char *prod_set_id; granule_pattern_p_t grpat; static char *products_set[NAGG_Product_list_max + 1]; /* arugment check */ assert(granule->product_id!=NULL); /* If the granule is the same product as last time the pattern is already set */ if (products_set[last_found] && HDstreq(granule->product_id, products_set[last_found])) { goto done; } /* The product wasn't the same; check the list of product with patterns. */ for (i=0; iproduct_id, products_set[i])) goto done; } /* The product for this granule doesn't have a pattern set, so set it. */ if (NULL == (grpat = (granule_pattern_t *)HDmalloc(sizeof(granule_pattern_t)))) { status = FAIL; goto done; } /* Check that all the required information is there in the granule. */ if (!granule->product_id || !granule->product_name || !granule->file_in) { status = FAIL; } else { HDstrcpy(grpat->product_id, granule->product_id); HDstrcpy(grpat->product_name, granule->product_name); HDstrcpy(grpat->file_in, granule->file_in); if(granule->geofile) HDstrcpy(grpat->geofile, granule->geofile); else HDstrcpy(grpat->geofile, ""); } granule_pattern_array[i] = grpat; if (NULL == (prod_set_id = HDmalloc(DPID_size+1))) { status = FAIL; goto done; } /* mark product "set" */ HDstrcpy(prod_set_id, granule->product_id); products_set[i] = prod_set_id; last_found = i; done: return status; } /*------------------------------------------------------------------------- * Function: get_granule_pattern * * * Purpose: Return the pattern granule for a product from the granule_pattern_array. * * Parameter: * product_id: DPID of product or GEO product * * Notes: Selected information from the first granule found for each product * is saved in an granule_pattern structure to insure that information * is available for writing fill granules. * * Keep track of whether a granule pattern has been saved for a product. * If it has, return 0 but don't save the new one. If it hasn't save * the needed information in the structure. * * Return: gran_pattern_t granule_pattern or NULL * * Programmer: Larry Knox * March 31, 2012 *------------------------------------------------------------------------- */ granule_pattern_p_t get_granule_pattern(const char *product_id) { int i = 0; while(i < NAGG_Product_list_max + 1 && granule_pattern_array[i] != NULL ) { if (HDstreq(granule_pattern_array[i]->product_id, product_id)) /* Found in the list. */ return granule_pattern_array[i]; ++i; } /* Not found in the list. */ return NULL; } /*------------------------------------------------------------------------- * Function: nagg_get_granules * * Purpose: Get granule information from a list of JPSS files. * * Parameters (IN): * file_list: names of input files to process * number_of_files number of filenames in file_list * products_list product_ids for which to get granules * nproducts number of product_ids in products_list * geofiles_arg indicates no geo, geo (default), filenames that exactly * match the N_GEO_Ref value (strict) or geo only * geoproduct DPID of the GEO product that matches the products in * products_list or the only product if aggregating * GEO only. NULL if products only (-g no) * Parameters (OUT): granule_info_p and number_of_granules_p are * out parameters to return values: * granule_info_p: pointer to the array of granule_info_t * structures (the granule table). * number_of_granules_p: pointer to the variable containing * the number of granules in the table. * * Notes: Input files in the file_list are required. * An empty products_list with 0 nproducts and a valid GEO DPID * will search for the GEO product in the input files. Any * N_GEO_Ref attributes will be ignored. * If geoproduct is NULL, no GEO granule searching will be attempted * nor will any GEO granules be added to the granule table. * If one or more products and a geoproduct are supplied, an error * will be returned unless at least one granule of each product * and the geoproduct are found. * * The function will get granules for each group matching a requested * product. The get_group_granules function will process the matching * subgroups of /Data_Products, calling get_granule_info for each of the * datasets in the subgroup to add the group's granules to the table. * * GEO granules are searched for if geoproduct is defined. The GEO * granules will be found in the GEO subgroup of /Data_Products if * the input file is packaged, or in the N_GEO_Ref file if the files * are unpackaged. * * Return: Success: 0 * Failure: -1 * * Programmer: Larry Knox * March 23, 2012 *------------------------------------------------------------------------- */ int nagg_get_granules(char **file_list, int number_of_files, char **products_list, int nproducts, geolocation_t geofiles_arg, char *geoproduct, granule_p_t *granule_info_p[], int *number_of_granules_p) { int status = SUCCEED; int i, j; hid_t file = -1; hid_t grp = -1; hid_t subgrp = -1; /* File and group identifiers */ char *geofilepath = NULL; char *geofile_buf = NULL; char **geofile_list = NULL; int num_geo_files = 0; htri_t retval; char *pos = NULL; GranF_data grp_dat; int num_patterns; int gran_num_before; int duplicate_file = 0; H5E_auto2_t func; void *client_data; g_gran_num = 0; if (geoproduct) { /* there should be at most the same number of geolocation files as there are data files. */ if (NULL==(geofile_list = HDmalloc(sizeof(char*)*number_of_files))){ NAGG_ERROR("nagg_get_granules(): malloc failed\n"); } } for (i=0; i 0) { fprintf(stderr, "%s is an HDF5 file, but H5Fopen failed.\n", file_list[i]); } else if (!retval) { fprintf(stderr, "%s is not an HDF5 file.\n", file_list[i]); } else { fprintf(stderr, "%s does not exist or otherwise cannot be accessed for reading.\n", file_list[i]); } status = FAIL; goto done; } /* * Open the /Data_Products group, */ if((grp = H5Gopen2(file, "/Data_Products", H5P_DEFAULT)) < 0) { fprintf(stderr, "Failed to open /Data_Products group in file %s.\n", file_list[i]); status = FAIL; goto done; } grp_dat.filepath = file_list[i]; grp_dat.granArr = granule_info_array; grp_dat.grpid = grp; grp_dat.granule_input_index = -1; /* * get the geofile from the N_GEO_Ref attribute if the attribute exists * and geoproduct is not NULL (-g no was not invoked). */ if (geoproduct) { /* The file should either have a group for the geoproduct or a /N_GE)_Ref attribute. It should not have both. */ if(H5Lexists(grp, get_product_sname_by_id(geoproduct), H5P_DEFAULT)) { grp_dat.geofile = NULL; } else if(NULL == (geofile_buf = (char *)malloc(GEO_Filename_size))) { NAGG_ERROR("nagg_get_granules(): malloc failed\n"); status = FAIL; goto done; } else if ((status = get_n_geo_ref_value(file, geofile_buf) < 0)) { NAGG_ERROR("nagg_get_granules(): Failed to get geo file\n"); goto done; } } /* geofile_buf was initialized to NULL. If the preceding code found a geofile, grp_dat.geofile should be set to that, otherwise to NULL. */ grp_dat.geofile = geofile_buf; /* for debugging, add all granules. This will add all product and geoproduct granules in the input file */ if(get_option_debug) { if((status = H5Literate(grp, H5_INDEX_NAME, H5_ITER_NATIVE, NULL, group_finder, &grp_dat)) < 0) { NAGG_ERROR("nagg_get_granules(): Failed to get granules iterating over product groups.\n"); goto done; } /* Get geo granules, too if there is a geoproduct and a geofile_buf. If no geoproduct we were told to ignore geo granules, and if no geofile_buf the geo is in this file, not a separate file (and the strncat will seg fault).*/ if (geoproduct && geofile_buf) { if (pos) { strncat(geofilepath, geofile_buf, strlen(geofile_buf)); geofile_list[num_geo_files++] = geofilepath; } else { geofile_list[num_geo_files++] = geofile_buf; } } } else { /* We don't want to add geogranules to the table if they don't correspond to the products requested. Track whether any product granules are added. */ gran_num_before = g_gran_num; /* Trying to open the group for each requested product can produce a lot of failed H5Gopen calls, making debugging difficult. To clean up these error messages and possibly reduce running time, try getting a list of /Data_Products subgroups and open any that are found in the products_list. */ for (j=0; j -1) { grp_dat.grpid = subgrp; grp_dat.dpid = products_list[j]; HDstrcpy(grp_dat.product_name, get_product_sname_by_id(products_list[j])); if((status = get_group_granules(subgrp, grp_dat.product_name, &grp_dat)) < 0) { if (g_gran_num >= NAGG_Granule_info_max){ /* Maximum granule limit exceeded. Error message already printed. */ ; } else { NAGG_ERROR("nagg_get_granules(): Failed to get granules\n"); fprintf(stderr, "for product %s.\n", get_product_sname_by_id(get_product_sname_by_id(products_list[j]))); } if (subgrp > -1) { H5Gclose(subgrp); subgrp = -1; } goto done; } H5Gclose(subgrp); subgrp = -1; } } /* Get granules for the geoproduct group in the input file if there is no geofile, there is a geoproduct, and either product granules were added from the file or nproducts is 0. */ if ( geoproduct && (g_gran_num > gran_num_before || nproducts == 0)) { if (!grp_dat.geofile) { subgrp = H5Gopen2(grp, get_product_sname_by_id(geoproduct), H5P_DEFAULT); grp_dat.dpid = geoproduct; grp_dat.grpid = subgrp; HDstrcpy(grp_dat.product_name, get_product_sname_by_id(geoproduct)); if((status = get_group_granules(subgrp, grp_dat.product_name, &grp_dat)) < 0) { if (g_gran_num >= NAGG_Granule_info_max){ /* Maximum granule limit exceeded. Error message already printed. */ ; } else { NAGG_ERROR("nagg_get_granules(): Failed to get granules\n"); fprintf(stderr, "for product %s.\n", get_product_sname_by_id(geoproduct)); } if (subgrp > -1) { H5Gclose(subgrp); subgrp = -1; } goto done; } if (subgrp > -1) H5Gclose(subgrp); subgrp = -1; } else { /* check to see if the file is already in the geofile_list. If not, add it. */ duplicate_file = 0; j=0; if (pos) { strncat(geofilepath, geofile_buf, strlen(geofile_buf)); while(j -1) { H5Gclose(grp); grp = -1; } if (file > -1) { H5Fclose(file); file = -1; } } *granule_info_p = granule_info_array; if(num_geo_files > 0) { /* * Get attributes of each Gran_n dataset in the GEO file * and add each granule to the table. */ if((status = find_geo_granules(geofile_list, num_geo_files, geoproduct, geofiles_arg, granule_info_p)) < 0) { NAGG_ERROR("nagg_get_granules(): Failed to get geo-location granules\n"); } /* There are geo files, but they're all in the input file list. We don't want to add the granules twice, but we need the GEO product ID. Both parts of this have changed. The GEO product is now determined in the product table, and the GEO product may not be allowed in the product list. This code may no longer be needed. */ } *number_of_granules_p = g_gran_num; for (i=0; i -1) H5Gclose(subgrp); if (grp > -1) H5Gclose(grp); if (file > -1) H5Fclose(file); return status; } /*------------------------------------------------------------------------- * Function: get_n_geo_ref_value * * Purpose: get name of geo file in N_GEO_Ref attribute to add to * geofile_list if not already present * * Parameters: * file: IN: id of file containing N_GEO_Ref attribute * geofile_buf: OUT: pre-allocated buffer to return value * * Return: 0 if successful, -1 otherwise * * Programmer: Larry Knox * April 2, 2012 *------------------------------------------------------------------------- */ static int get_n_geo_ref_value(hid_t file, char *geofile_buf) { int status = SUCCEED; int i, j; hid_t attr; hid_t atype, atype_mem; /* Attribute type */ H5T_class_t type_class; char *pos = NULL; char *duplicate_geo_file = NULL; if ( H5Aexists_by_name(file, "/", "N_GEO_Ref", H5P_DEFAULT)) { if((attr = H5Aopen(file, "N_GEO_Ref", H5P_DEFAULT)) < 0) { fprintf(stderr, "Failed to open attribute /N_GEO_Ref.\n"); status = FAIL; goto done; } else if(( atype = H5Aget_type(attr))< 0) { fprintf(stderr, "Failed to get type for attribute /N_GEO_Ref.\n"); status = FAIL; } else if((type_class = H5Tget_class(atype)) < 0) { fprintf(stderr, "Failed to get type class for attribute /N_GEO_Ref.\n"); status = FAIL; } else if ((atype_mem = H5Tget_native_type(atype, H5T_DIR_ASCEND)) < 0) { fprintf(stderr, "Failed to get type class for attribute /N_GEO_Ref.\n"); status = FAIL; } else if ((status = H5Aread(attr, atype_mem, geofile_buf)) < 0) { fprintf(stderr, "Failed to read attribute /N_GEO_Ref.\n"); status = FAIL; } status = H5Aclose(attr); } done: return status; } /*------------------------------------------------------------------------- * Function: get_last_created_geofile * * Purpose: Get the filename with the latest creation time from multiple * matches to the filename with * replacing creation time, origin * and domain fields. * * Return: Success: 0 * Failure: -1 * * Programmer: Larry Knox * February 14, 2012 *------------------------------------------------------------------------- */ get_last_created_geofile(char **geofile_list, char *geofile, int count) { int i, status = SUCCEED; int largest_index = 0; /* compare pairs of names, indices i and i+1; greatest index is count-1 */ for (i=0; i<(count-1); ++i) { if (geofile_list[i+1]) { if(strcmp(geofile_list[largest_index], geofile_list[i+1]) < 0) largest_index = i + 1; } else { NAGG_ERROR("get_last_created_geofile(): Internal error.\n"); status = FAIL; goto done; } } strcat(geofile, geofile_list[largest_index]); done: /* for (i=0; i 255) ? NAME_MAX : 255; # else return (size_t)(-1); # endif # else # if defined(NAME_MAX) name_max = (NAME_MAX > 255) ? NAME_MAX : 255; # else # error "buffer size for readdir_r cannot be determined" # endif # endif name_end = (size_t)offsetof(struct dirent, d_name) + name_max + 1; return (name_end > sizeof(struct dirent) ? name_end : sizeof(struct dirent)); } /*------------------------------------------------------------------------ * Function: get_approximate_match_geofile * * Purpose: Get a list of geolocation files (usually 1) that match the * filename in N_GEO_Ref except for the file creation time field. * Note: Files from the CLASS server currently use a slightly different * file creation time for the names of the geolocation files than * the time used in the filename stored in the sensor data file's * N_GEO_Ref attribute. It should be safe to do this, since the * granule IDs for sensor data and geolocation granules will match if the two correspond, regardless of the file names. * * According to the control books, chars 45 - 66 of the geolocation * filename are derived from the time the file was created. For * files whose name doesn't match the name for the file in * N_GEO_Ref, we will check in the same location for files matching * the name with * in place of the creation time field, and then get * the granules from any files found. Usually there will be just * one. If geofilename contains a prefix the creation time field * will be 45 characters after the last '/', so we check for the * last '/' and start counting from there. If there is no '/', * count from the beginning. * * Return: Success: 0 * Failure: -1 * * Programmer: Larry Knox * January 12, 2012 *------------------------------------------------------------------------ */ int get_approximate_match_geofile(char *geofilename, char *geofile) { int status = SUCCEED; char *geofile_list[GEO_Filename_max_matches]; char dirname[GEO_FilePath_size + 1]; char *gfbuf; char *pos = NULL; DIR *dirp; struct dirent *buf; struct dirent *ent; int i, count, errno, error; hbool_t local; size_t len; count = 0; errno = 0; pos = strrchr(geofilename, '/'); if (!pos) { strcpy(dirname, "."); pos = geofilename; local = TRUE; } else { strncpy(dirname, geofilename, pos - geofilename); strncpy(geofile, geofilename, pos - geofilename + 1); geofile[pos - geofilename + 1] = '\0'; dirname[pos - geofilename] = '\0'; pos += 1; local = FALSE; } dirp = opendir(dirname); if (dirp == NULL) { perror("opendir"); status = FAIL; goto done; } len = dirent_buf_size(dirp); if (len == -1) { perror("dirent_buf_size"); status = FAIL; goto done; } buf = (struct dirent *)malloc(len); if (buf == NULL) { perror("malloc"); status = FAIL; goto done; } while ((error = readdir_r(dirp, buf, &ent)) == 0 && ent != NULL) { int duplicated = 0; if (strncmp(ent->d_name, pos, 46) == 0) { if (count > 0) { for (i=0; id_name)) { duplicated = 1; break; } } } if (!duplicated) { if (NULL==(gfbuf = (char *)HDmalloc(sizeof(char)*(GEO_FilePath_size + 1)))){ NAGG_ERROR("get_near_match_geofile_list(): malloc failed\n"); status = FAIL; goto done; } HDstrcpy(gfbuf, ent->d_name); geofile_list[count++] = gfbuf; } } } if (error) { errno = error; perror("readdir_r"); status = FAIL; goto done; } status = closedir(dirp); if (status == -1) { perror("closedir"); } if (count == 0) { status = FAIL; }else if (count == 1) { if (local) HDstrcpy(geofile, geofile_list[0]); else strcat(geofile, geofile_list[0]); HDfree(geofile_list[0]); } else { status = get_last_created_geofile(geofile_list, geofile, count); } done: return status; } /*------------------------------------------------------------------------ * Function: find_geo_granules * * Purpose: Get granule information from a list of geolocation files. This * function is a copy of nagg_get_granules, minus the part that looks up the * geolocation files in the /N_GEO_Ref attribute and then calls this function * to process them. After the initial prototype is done, the common parts * should be extracted to a function called by both to process the files. * * Return: Success: 0 * Failure: -1 * * Programmer: Larry Knox * December, 20, 2011 *------------------------------------------------------------------------ */ int find_geo_granules(char **geofile_list, int number_of_files, char *geoproduct, geolocation_t geofiles_arg, granule_p_t *granule_info_p[]) { int status = SUCCEED; int i, j; hid_t file = -1; hid_t newfile = -1; hid_t grp; /* File, group and dataset identifiers */ char geofile_buf[GEO_FilePath_size]; char *duplicate_geo_file = NULL; int num_geo_files = 0; int num_geo_granules; htri_t retval; H5E_auto2_t func; void *client_data; for (i=0; i 0) { fprintf(stderr, "%s is an HDF5 file, but H5Fopen failed.\n", geofile_list[i]); /* no */ } else if (!retval) { fprintf(stderr, "%s is not an HDF5 file.\n", geofile_list[i]); status = FAIL; goto done; /* The original file didn't exist or can't be accessed. Geolocation files sometimes have names with different creation times, producers, or domains than in the N_GEO_Ref attribute. If command was -g strict fail now, otherwise check for approximate matches. */ } else if (geofiles_arg == GEOFILE_STRICT) { NAGG_ERROR("find_geo_granules(): Failed to get geolocation granules.\n"); fprintf(stderr, "%s does not exist or otherwise cannot be accessed for reading.\n", geofile_list[i]); /* with both --debug and -g strict options we'll print the error message but not return FAIL. That way we print all the granules, but only geo granules from exact filename matches. */ if (!get_option_debug) { status = FAIL; goto done; } /* check for approximate matches. Print error if none are found. */ } else if((status = get_approximate_match_geofile(geofile_list[i], geofile_buf)) < 0) { fprintf(stderr, "Warning! find_geo_granules(): Failed to find approximate match for geolocation file.\n"); fprintf(stderr, "%s does not exist or otherwise cannot be accessed for reading.\n", geofile_list[i]); fprintf(stderr, "Fill granules will be added for missing granules.\n"); if(H5Eclear2(H5E_DEFAULT) < 0) NAGG_ERROR("nagg_get_granules(): Failed to clear error stack.\n"); /* Similarly here we'll print the error message but not return FAIL. That way we print all the granules, that we find but print error messages for the geo files that aren't found. */ if (get_option_debug) { status = SUCCEED; } /* Match found - try to open it. */ } else { if((newfile = H5Fopen(geofile_buf, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) { retval = H5Fis_hdf5(geofile_buf); if (retval > 0) { fprintf(stderr, "%s is an HDF5 file, but H5Fopen failed.\n", geofile_buf); } else if (!retval) { fprintf(stderr, "%s is not an HDF5 file.\n", geofile_buf); } else { NAGG_ERROR("find_geo_granules(): Failed to open approximate match for geolocation file.\n"); fprintf(stderr, "%s does not exist or otherwise cannot be accessed for reading.\n", geofile_buf); status = FAIL; goto done; } } /* copy contents of geofile_buf to geofile_list[i], since geofile_list[i] is persistent while geofile_buf is local. Maybe geofile_list[i] could be replaced by get_approximate_match_geofile() to cut down on the strcpys. */ strcpy(geofile_list[i], geofile_buf); /* clear geobuf for the next file. */ strcpy(geofile_buf, ""); /* get geo granules from the approximate match file. */ get_geo_granules(newfile, geofile_list[i], geoproduct, granule_info_p); if (newfile > -1) { H5Fclose(newfile); newfile = -1; } } /* The original geo file was opened, so get granules from it. */ } else { if (H5Eset_auto2(H5E_DEFAULT, func, client_data) < 0) { fprintf(stderr, "find_geo_granules(): failed to reset automatic error printing parameters.\n"); } get_geo_granules(file, geofile_list[i], geoproduct, granule_info_p); } if (file > -1) { H5Fclose(file); file = -1; } } done: if (file > -1) H5Fclose(file); if (newfile > -1) H5Fclose(newfile); return status; } /*------------------------------------------------------------------------- * Function: get_geo_granules * * Purpose: Get granule information from a list of geolocation files. * * Note: This function is a copy of nagg_get_granules, minus the part * that looks up the geolocation files in the /N_GEO_Ref attribute * and then calls this function to process them. After the initial * prototype is done, the common parts should be extracted to a * function called by both to process the files. * * Return: Success: 0 * Failure: -1 * * Programmer: Larry Knox * December 20, 2011 *------------------------------------------------------------------------- */ int get_geo_granules(hid_t file, char *geofile, char *geoproduct, granule_p_t *granule_info_p[]) { int status = SUCCEED; hid_t grp, subgrp; /* group identifiers */ GranF_data grp_dat; /* * Open each subgroup of /Data_Products, get attributes of each Gran_n dataset * and add each granule to the list. */ if((grp = H5Gopen2(file, "/Data_Products", H5P_DEFAULT)) < 0) { fprintf(stderr, "Failed to open /Data_Products group in file %s.\n", geofile); status = FAIL; goto done; } grp_dat.filepath = geofile; grp_dat.granArr = granule_info_array; grp_dat.dpid = geoproduct; grp_dat.grpid = grp; grp_dat.geofile = NULL; grp_dat.granule_input_index = -1; subgrp = H5Gopen2(grp, get_product_sname_by_id(geoproduct), H5P_DEFAULT); grp_dat.grpid = subgrp; HDstrcpy(grp_dat.product_name, get_product_sname_by_id(geoproduct)); if((status = get_group_granules(subgrp, grp_dat.product_name, &grp_dat)) < 0) { NAGG_ERROR("get_geo_granules(): Failed to get GEO granules.\n"); status = FAIL; goto done; } if (!HDstreq(geoproduct, get_product_id_by_sname(grp_dat.product_name))) { NAGG_ERROR("get_geo_granules(): Products to be packaged use multiple geolocation products\n"); fprintf(stderr, "geoproduct: %s\nProduct found: %s\n", geoproduct, get_product_id_by_sname(grp_dat.product_name)); status = FAIL; } done: if (subgrp > -1) H5Gclose(subgrp); if (grp > -1) H5Gclose(grp); return status; } /*------------------------------------------------------------------------- * Function: group_finder * * Purpose: Function for iterator used to iterate over the subgroups of * the /Data_Products group in the JPSS files in order to find * all granules in the file for the --debug option. * * Note: There will be one subgroup for each data product in a packaged * file. The subgroups have one dataset with a name ending in _Aggr * with attributes describing the aggregation and references to the * raw data datasets, and 1 to n datasets with names ending in * _Gran_n. This function gets information from the attributes to * the _Aggr dataset and calls get_granule_info for each of the n * _Gran_n datasets to get their attribute information and add the * granule to the granule_info table. * * Return: Success: 0 * Failure: -1 * * Programmer: Larry Knox * May, 11, 2012 * *------------------------------------------------------------------------- */ static int group_finder (hid_t loc_id, const char *name, const H5L_info_t *info, void *operator_data) { int status = 0; H5O_info_t infobuf; hid_t grpid; /* group identifier */ GranF_data *granf_dat = (GranF_data *)operator_data; if((status = H5Oget_info_by_name (loc_id, name, &infobuf, H5P_DEFAULT)) < 0) { fprintf(stderr, "Failed to get info for %s.\n", name); goto done; } switch (infobuf.type) { case H5O_TYPE_GROUP: grpid = H5Gopen2(loc_id, name, H5P_DEFAULT); /* the group name matches the product name. */ HDstrcpy(granf_dat->product_name, name); granf_dat->dpid = get_product_id_by_sname(name); if((status = get_group_granules(grpid , name, granf_dat)) < 0) { NAGG_ERROR("group_finder(): Failed to get granules\n"); fprintf(stderr, "for product %s.\n", name); } if (grpid > -1) H5Gclose(grpid); break; default: fprintf(stderr, "Object %s is not a group.\n", name); break; } done: return status; } /*------------------------------------------------------------------------- * Function: get_group_granules * * Purpose: Function for iterator used to iterate over the subgroups of * the /Data_Products group in the JPSS files. * * Note: There will be one subgroup for each data product in a packaged * file. The subgroups have one dataset with a name ending in _Aggr * with attributes describing the aggregation and references to the * raw data datasets, and 1 to n datasets with names ending in * _Gran_n. This function gets information from the attributes to * the _Aggr dataset and calls get_granule_info for each of the n * _Gran_n datasets to get their attribute information and add the * granule to the granule_info table. * * Return: Success: 0 * Failure: -1 * * Programmer: Larry Knox * November, 29, 2011 * *------------------------------------------------------------------------- */ static int get_group_granules (hid_t loc_id, const char *name, void *operator_data) { int status; hid_t grpid; /* File, group and dataset identifiers */ hid_t dset = -1; hid_t attr = -1; char linkName[Group_path_size + 1]; int aggr_num_granules, i; char *pos = NULL; GranF_data *granf_dat = (GranF_data *)operator_data; /* get the number of granules in the aggregation from the _Aggr dataset. */ snprintf(linkName, strlen(name) + 6, "%s_Aggr", name); if((dset = H5Dopen2(loc_id, linkName, H5P_DEFAULT))<0) { fprintf(stderr, "Failed to open dataset %s in file %s.\n", linkName, granf_dat->filepath); status = FAIL; goto done; } if((attr = H5Aopen(dset, "AggregateNumberGranules", H5P_DEFAULT)) < 0) { fprintf(stderr, "Failed to open attribute AggregateNumberGranules on dataset %s.\n", linkName); status = FAIL; goto done; } else if((status = H5Aread(attr, H5T_NATIVE_INT, &aggr_num_granules)) < 0) { fprintf(stderr, "Failed to read attribute N_Ending_Time_IET on dataset %s.\n", name); goto done; } /* this for loop supplies a granule number from 0 to the number of granules in the file according to the AggregateNumberGranules attribute. Each number is used to construct a granule dataset name which is then processed. */ for(i=0; igranule_input_index = i; /* 8 is number of added characters + 1 for i + 1. Add characters for each additional digit in the granule index */ if (i < 10) snprintf(linkName, strlen(name) + 8, "%s_Gran_%d", name, i); else if (i < 100) snprintf(linkName, strlen(name) + 9, "%s_Gran_%d", name, i); else if (i < 999) snprintf(linkName, strlen(name) + 10, "%s_Gran_%d", name, i); else if (i < 9999) snprintf(linkName, strlen(name) + 10, "%s_Gran_%d", name, i); status = get_granule_info (loc_id, linkName, granf_dat); if (status < 0) { break; } } done: if (attr > -1) H5Aclose(attr); if (dset > -1) H5Dclose(dset); return status; } /*------------------------------------------------------------------------- * Function: get_granule_info * * Purpose: Get attribute information for 1 of 1 to n datasets with names * ending in _Gran_n. * * Note: This function allocates memory for a granule_t struct, gets * information from the attributes to the dataset and adds the * granule to the granule table. The pointer to the array and * information gathered from file and group attributes are passed * in through the operator_data parameter in a GranF_data struct. * * Return: Success: 0 * Failure: -1 * * Programmer: Larry Knox * November, 29, 2011 * *------------------------------------------------------------------------- */ static int get_granule_info (hid_t loc_id, const char *name, void *operator_data) { int status; H5O_info_t infobuf; char *pos = NULL; hid_t dset = -1; hid_t attr = -1; hid_t atype, atype_mem; /* Attribute type */ H5T_class_t type_class; unsigned long long beginning_time_IET, ending_time_IET; char *pt; #define NAGG_Granule_status_string_max 32 char gran_status[NAGG_Granule_status_string_max+1]; GranF_data *granf_dat = (GranF_data *)operator_data; if (g_gran_num >= NAGG_Granule_info_max){ NAGG_ERROR("get_granule_info(): Number of input granules exceed maximum granules allowed for nagg.\n"); fprintf(stderr, "Max = %d\n", NAGG_Granule_info_max); status = FAIL; goto done; } granule_info_array[g_gran_num] = (granule_t *)HDmalloc(sizeof(granule_t)); if (granule_info_array[g_gran_num] == NULL) { fprintf(stderr, "Error allocating memory for granule_info_array[%d].\n", g_gran_num); status = FAIL; goto done; } if((dset = H5Dopen2(loc_id, name, H5P_DEFAULT)) < 0) { fprintf(stderr, "Failed to open dataset %s.\n", name); status = FAIL; goto done; } if((status = get_string_attribute_value(dset, name, "N_Granule_ID", granf_dat->granArr[g_gran_num]->granule_id, H5P_DEFAULT))< 0) goto done; /*((GranF_data *)operator_data)->granArr[g_gran_num]->granule_id, H5P_DEFAULT))< 0)*/ /* This block of code skips the rest of the function, so that granules that don't have valid granule IDs are not added to the granule table. If --debug sets the debug option, skip this block and continue through the function to add the granule and subsequently print it. */ if(!get_option_debug) { if(HDstreq(granf_dat->granArr[g_gran_num]->granule_id, "N/A")) { if((status = get_string_attribute_value(dset, name, "N_Granule_Status", gran_status, H5P_DEFAULT))< 0) goto done; if(HDstreq(gran_status, "Missing at delivery time")) { /* This granule doesn't have a valid granule ID and it's a fill granule. Don't put it in the granule table. */ goto done; } else { /* The ID is invalid but according to N_Granule_Status it's not a fill granule. For now, fail. */ NAGG_ERROR("get_granule_info(): N_Granule_ID is N/A but the granule is not a fill granule according to N_Granule_Status.\n"); status = FAIL; goto done; } } } if((status = get_string_attribute_value(dset, name, "N_Granule_Version", granf_dat->granArr[g_gran_num]->granule_version, H5P_DEFAULT)) < 0) goto done; if((status = get_string_attribute_value(dset, name, "Beginning_Date", granf_dat->granArr[g_gran_num]->beginning_date, H5P_DEFAULT)) < 0) goto done; if((status = get_string_attribute_value(dset, name, "Beginning_Time", granf_dat->granArr[g_gran_num]->beginning_time, H5P_DEFAULT)) < 0) goto done; if((status = get_string_attribute_value(dset, name, "Ending_Time", granf_dat->granArr[g_gran_num]->ending_time, H5P_DEFAULT)) < 0) goto done; if (0 == strcmp("N/A", ((GranF_data *)operator_data)->granArr[g_gran_num]->granule_version)) { granf_dat->granArr[g_gran_num]->granule_version_number = -1; } else if ('A' == granf_dat->granArr[g_gran_num]->granule_version[0] && isdigit(granf_dat->granArr[g_gran_num]->granule_version[1])) { granf_dat->granArr[g_gran_num]->granule_version_number = atoi(granf_dat->granArr[g_gran_num]->granule_version + 1); } else { fprintf(stderr, "Granule version number could not be determined for granule %s in %s.\n", name, granf_dat->filepath); status = FAIL; goto done; } if((attr = H5Aopen(dset, "N_Beginning_Time_IET", H5P_DEFAULT)) < 0) { fprintf(stderr, "Failed to open attribute N_Beginning_Time_IET on dataset %s.\n", name); status = FAIL; goto done; } else if((status = H5Aread(attr, H5T_NATIVE_ULLONG, &granf_dat->granArr[g_gran_num]->granule_start_time_IET)) < 0) { fprintf(stderr, "Failed to read attribute N_Beginning_Time_IET on dataset %s.\n", name); } status = H5Aclose(attr); attr = -1; if (status < 0) { goto done; } if((attr = H5Aopen(dset, "N_Ending_Time_IET", H5P_DEFAULT)) < 0) { fprintf(stderr, "Failed to open attribute N_Ending_Time_IET on dataset %s.\n", name); status = FAIL; goto done; } else if(( status = H5Aread(attr, H5T_NATIVE_ULLONG, &granf_dat->granArr[g_gran_num]->granule_end_time_IET)) < 0) { fprintf(stderr, "Failed to read attribute N_Ending_Time_IET on dataset %s.\n", name); } status = H5Aclose(attr); attr = -1; if (status < 0) { goto done; } if((attr = H5Aopen(dset, "N_Beginning_Orbit_Number", H5P_DEFAULT)) < 0) { fprintf(stderr, "Failed to open attribute N_Beginning_Orbit_Number on dataset %s.\n", name); status = FAIL; goto done; } else if((status = H5Aread(attr, H5T_NATIVE_ULLONG, &granf_dat->granArr[g_gran_num]->orbit_number)) < 0) { fprintf(stderr, "Failed to read attribute N_Beginning_Orbit_Number on dataset %s.\n", name); } status = H5Aclose(attr); attr = -1; if (status < 0) { goto done; } HDstrncpy(granf_dat->granArr[g_gran_num]->product_name, granf_dat->product_name, NAGG_Product_Type_size+1); granf_dat->granArr[g_gran_num]->granule_input_index = granf_dat->granule_input_index; HDstrcpy(granf_dat->granArr[g_gran_num]->product_id, granf_dat->dpid); granf_dat->granArr[g_gran_num]->file_in = granf_dat->filepath; /* initialize granule->geofile to NULL. */ /* granf_dat->granArr[g_gran_num]->geofile = NULL; */ granf_dat->granArr[g_gran_num]->geofile = granf_dat->geofile; status = set_granule_pattern(granf_dat->granArr[g_gran_num]); /* increment g_gran_num at the end, otherwise it will be necessary to subtract 1 when using it as the array subscript. */ ++g_gran_num; done: if (attr > -1) H5Aclose(attr); if (dset > -1) H5Dclose(dset); return status; } /*------------------------------------------------------------------------- * Function: get_string_attribute_value * * Purpose: This function makes the several HDF5 calls to get the value of a * string attribute for the get_granule_info function. * * Return: Success: 0 * Failure: -1 * * Programmer: Larry Knox * November, 29, 2011 * *------------------------------------------------------------------------- */ static int get_string_attribute_value(hid_t loc_id, const char * dsetname, const char *attrname, void *buf, hid_t aapl_id) { hid_t attr; hid_t atype, atype_mem; /* Attribute type */ H5T_class_t type_class; int status; if((attr = H5Aopen(loc_id, attrname, aapl_id))< 0) { fprintf(stderr, "Failed to open attribute %s on dataset %s.\n", attrname, dsetname); status = FAIL; } else if ((atype = H5Aget_type(attr))< 0) { fprintf(stderr, "Failed to get type for attribute %s on dataset %s.\n", attrname, dsetname); status = FAIL; } else if((type_class = H5Tget_class(atype)) < 0) { fprintf(stderr, "Failed to get type class for attribute %s on dataset %s.\n", attrname, dsetname); status = FAIL; } else if((atype_mem = H5Tget_native_type(atype, H5T_DIR_ASCEND)) < 0) { fprintf(stderr, "Failed to get type class for attribute %s on dataset %s.\n", attrname, dsetname); status = FAIL; } else if((status = H5Aread(attr, atype_mem, buf))< 0) { fprintf(stderr, "Failed to read attribute %s on dataset %s.\n", attrname, dsetname); } H5Aclose(attr); return status; } /*------------------------------------------------------------------------- * Function: nagg_print_granules * * Purpose: Print granule information from JPSS files contained in an array of granule_t structures * in csv form * * Return: none * * Programmer: Larry Knox * November, 29, 2011 * *------------------------------------------------------------------------- */ void nagg_print_granules(granule_t *granArr[], int number_of_granules) { int i = 0; const char * geofile; printf("Printing Granules Read\n"); printf("Granule ID, DPID, Product Name, Gran Index, Granule Version, Granule Version Number, Granule Start Time IET,Gran End Time IET, Beginning Date, Beginning Time, Ending Time, Beginning Orbit Number, Filename, Geo Filename\n"); for(i = 0; igranule_id, granArr[i]->product_id, granArr[i]->product_name, granArr[i]->granule_input_index, granArr[i]->granule_version, granArr[i]->granule_version_number, granArr[i]->granule_start_time_IET, granArr[i]->granule_end_time_IET, granArr[i]->beginning_date, granArr[i]->beginning_time, granArr[i]->ending_time, granArr[i]->orbit_number, granArr[i]->file_in, stringval(granArr[i]->geofile)); } printf("Finished Printing Granules\n"); }