visir_util_join.c

00001 /* $Id: visir_util_join.c,v 1.2 2011/07/28 15:02:10 llundin Exp $
00002  *
00003  * This file is part of the VISIR Pipeline
00004  * Copyright (C) 2011 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: llundin $
00023  * $Date: 2011/07/28 15:02:10 $
00024  * $Revision: 1.2 $
00025  * $Name: visir-3_5_0 $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033                                 Includes
00034  -----------------------------------------------------------------------------*/
00035 
00036 #include "visir_recipe.h"
00037 
00038 /*-----------------------------------------------------------------------------
00039                                 Defines
00040  -----------------------------------------------------------------------------*/
00041 
00042 #define RECIPE_STRING   "visir_util_join"
00043 
00044 /*-----------------------------------------------------------------------------
00045                             Private Functions prototypes
00046  -----------------------------------------------------------------------------*/
00047 
00048 static
00049 cpl_error_code visir_util_join_one(cpl_frameset *,
00050                                    const irplib_framelist *,
00051                                    const irplib_framelist *,
00052                                    const irplib_framelist *,
00053                                    const irplib_framelist *,
00054                                    const irplib_framelist *,
00055                                    int, cpl_boolean, cpl_boolean, 
00056                                    cpl_boolean, cpl_boolean,
00057                                    const cpl_parameterlist *);
00058 
00059 VISIR_RECIPE_DEFINE(visir_util_join, 0,
00060                     "Extend the final product with extensions containing its "
00061                     "error map, bad pixel map and weight or contribution map",
00062                     "The files listed in the Set Of Frames (SOF-file) "
00063                     "must be tagged:\n"
00064                     "VISIR-data.fits "          VISIR_UTIL_DATA "\n"
00065                     "\nOptionally, the SOF may also contain one or more of "
00066                     "the following:\n"
00067                     "VISIR-bad-pixel-map.fits " VISIR_CALIB_BPM "\n"
00068                     "VISIR-error.fits "         VISIR_UTIL_WEIGHT2ERROR_PROCATG
00069                       "\n"
00070                     "VISIR-contribution-map.fits " VISIR_IMG_CLIPPED_MAP_PROCATG
00071                       "\n"
00072                     "VISIR-weight-map.fits "    VISIR_UTIL_WEIGHT2ERROR
00073                       "\n"
00074                     "\nThe product(s) will have a FITS card\n"
00075                     "'HIERARCH " CPL_DFS_PRO_CATG "' with a value of:\n"
00076                     VISIR_UTIL_JOIN_PROCATG);
00077 
00078 /*----------------------------------------------------------------------------*/
00082 /*----------------------------------------------------------------------------*/
00083 
00084 /*-----------------------------------------------------------------------------
00085                                 Functions code
00086  -----------------------------------------------------------------------------*/
00087 
00088 /*----------------------------------------------------------------------------*/
00095 /*----------------------------------------------------------------------------*/
00096 static int visir_util_join(cpl_frameset            * framelist,
00097                            const cpl_parameterlist * parlist)
00098 {
00099 #ifdef _OPENMP
00100     cpl_errorstate     cleanstate = cpl_errorstate_get();
00101 #endif
00102     cpl_error_code     didfail   = CPL_ERROR_NONE;
00103     irplib_framelist * allframes = NULL;
00104     irplib_framelist * rawframes = NULL;
00105     irplib_framelist * bpmframes = NULL;
00106     irplib_framelist * errframes = NULL;
00107     irplib_framelist * conframes = NULL;
00108     irplib_framelist * wgtframes = NULL;
00109     int                i, n;
00110     int                nbad = 0, nerr = 0, ncon = 0, nwgt = 0;
00111     
00112 
00113     /* Identify the RAW and TAB frames in the input frameset */
00114     skip_if (visir_dfs_set_groups(framelist));
00115 
00116     /* Objects observation */
00117     allframes = irplib_framelist_cast(framelist);
00118     skip_if(allframes == NULL);
00119 
00120     rawframes = irplib_framelist_extract(allframes, VISIR_UTIL_DATA);
00121     skip_if(rawframes == NULL);
00122 
00123     n = irplib_framelist_get_size(rawframes);
00124 
00125     if (cpl_frameset_find_const(framelist, VISIR_CALIB_BPM)) {
00126         bpmframes = irplib_framelist_extract(allframes, VISIR_CALIB_BPM);
00127         skip_if (bpmframes == NULL);
00128 
00129         nbad = irplib_framelist_get_size(bpmframes);
00130         error_if(nbad != n && nbad != 1, CPL_ERROR_INCOMPATIBLE_INPUT,
00131                  "%d raw-frames <=> %d bpm frames", n, nbad);
00132     }
00133 
00134     if (cpl_frameset_find_const(framelist, VISIR_UTIL_WEIGHT2ERROR_PROCATG)) {
00135         errframes = irplib_framelist_extract(allframes,
00136                                              VISIR_UTIL_WEIGHT2ERROR_PROCATG);
00137         skip_if (errframes == NULL);
00138 
00139         nerr = irplib_framelist_get_size(errframes);
00140         error_if(nerr != n && nerr != 1, CPL_ERROR_INCOMPATIBLE_INPUT,
00141                  "%d raw-frames <=> %d error frames", n, nerr);
00142 
00143     }
00144 
00145     if (cpl_frameset_find_const(framelist, VISIR_IMG_CLIPPED_MAP_PROCATG)) {
00146         conframes = irplib_framelist_extract(allframes,
00147                                              VISIR_IMG_CLIPPED_MAP_PROCATG);
00148         skip_if (conframes == NULL);
00149 
00150         ncon = irplib_framelist_get_size(conframes);
00151         error_if(ncon % n != 0 && ncon != 1, CPL_ERROR_INCOMPATIBLE_INPUT,
00152                  "%d raw-frames <=> %d contribution frames", n, ncon);
00153     }
00154 
00155     if (cpl_frameset_find_const(framelist, VISIR_UTIL_WEIGHT2ERROR)) {
00156         wgtframes = irplib_framelist_extract(allframes,
00157                                              VISIR_UTIL_WEIGHT2ERROR);
00158         skip_if (wgtframes == NULL);
00159 
00160         nwgt = irplib_framelist_get_size(wgtframes);
00161         error_if(nwgt != n && nwgt != 1, CPL_ERROR_INCOMPATIBLE_INPUT,
00162                  "%d raw-frames <=> %d weight frames", n, nwgt);
00163     }
00164 
00165 #ifdef _OPENMP
00166 #pragma omp parallel for private(i)
00167 #endif
00168     for (i = 0; i < n; i++) {
00169         if (!didfail) {
00170 
00171             /* The total number of iterations must be pre-determined for the
00172                parallelism to work. In case of an error we can therefore not
00173                break, so instead we skip immediately to the next iteration.
00174                FIXME: This check on didfail does not guarantee that only one
00175                iteration can cause an error to be dumped, but it is not
00176                worse than checking on a thread-local state, e.g. errori. */
00177 
00178             cpl_msg_info(cpl_func, "Joining frame %d/%d", 1+i, n);
00179 
00180             if (visir_util_join_one(framelist, rawframes, bpmframes, errframes,
00181                                     conframes, wgtframes, i, nbad == 1,
00182                                     nerr == 1, ncon == 1, nwgt == 1,
00183                                     parlist)) {
00184             const cpl_error_code errori = cpl_error_set_where(cpl_func);
00185 #ifdef _OPENMP
00186                 /* Cannot access these errors after the join,
00187                    so dump them now. :-(((((((((((((((((((( */
00188                 cpl_errorstate_dump(cleanstate, CPL_FALSE, NULL);
00189                 cpl_errorstate_set(cleanstate);
00190 #pragma omp critical(visir_util_join)
00191 #endif
00192                 didfail = errori;
00193             }
00194         }
00195     }
00196 
00197     error_if(didfail, didfail, "Failed to process %d frame(s)", n);
00198 
00199     end_skip;
00200 
00201     irplib_framelist_delete(allframes);
00202     irplib_framelist_delete(rawframes);
00203     irplib_framelist_delete(bpmframes);
00204     irplib_framelist_delete(errframes);
00205     irplib_framelist_delete(conframes);
00206     irplib_framelist_delete(wgtframes);
00207 
00208     return cpl_error_get_code();
00209 }
00210 
00211 
00212 /*----------------------------------------------------------------------------*/
00229 /*----------------------------------------------------------------------------*/
00230 static
00231 cpl_error_code visir_util_join_one(cpl_frameset * framelist,
00232                                    const irplib_framelist * rawframes,
00233                                    const irplib_framelist * bpmframes,
00234                                    const irplib_framelist * errframes,
00235                                    const irplib_framelist * conframes,
00236                                    const irplib_framelist * wgtframes,
00237                                    int i,
00238                                    cpl_boolean bshared, cpl_boolean eshared,
00239                                    cpl_boolean cshared, cpl_boolean wshared,
00240                                    const cpl_parameterlist * parlist)
00241 {
00242 
00243     const int          n = irplib_framelist_get_size(rawframes);
00244     const cpl_frame  * frame;
00245     cpl_frameset     * products   = cpl_frameset_new();
00246     cpl_frameset     * usedframes = cpl_frameset_new();
00247     const char       * filename;
00248     cpl_propertylist * plist      = NULL;
00249     char             * proname    = cpl_sprintf(RECIPE_STRING "_%d"
00250                                                 CPL_DFS_FITS, 1+i);
00251     cpl_image        * image      = NULL;
00252     cpl_image        * csum       = NULL;
00253     cpl_mask         * bpm        = NULL; 
00254 
00255     bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate
00256                                (irplib_framelist_get_const(rawframes, i))));
00257 
00258     if (bpmframes != NULL) {
00259         frame = irplib_framelist_get_const(bpmframes, bshared ? 0 : i);
00260         bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(frame)));
00261     }
00262 
00263     if (errframes != NULL) {
00264         frame = irplib_framelist_get_const(errframes, eshared ? 0 : i);
00265         bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(frame)));
00266     }
00267 
00268     if (conframes != NULL && (irplib_framelist_get_size(conframes) == n ||
00269                               cshared)) {
00270         frame = irplib_framelist_get_const(conframes, cshared ? 0 : i);
00271         bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(frame)));
00272     }
00273 
00274     if (wgtframes != NULL) {
00275         frame = irplib_framelist_get_const(wgtframes, wshared ? 0 : i);
00276         bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(frame)));
00277     }
00278 
00279     /* 1) The data frame */
00280     frame = irplib_framelist_get_const(rawframes, i);
00281     filename = cpl_frame_get_filename(frame);
00282 
00283     plist = cpl_propertylist_load(filename, 0);
00284     skip_if(plist == NULL);
00285 
00286     /* FIXME: Change to CPL_TYPE_UNSPECIFIED when supported */
00287     image = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, 0);
00288     skip_if(image == NULL);
00289 
00290     /* FIXME: Change to CPL_TYPE_UNSPECIFIED when supported */
00291     skip_if(irplib_dfs_save_image(products, parlist, usedframes,
00292                                   image, CPL_BPP_IEEE_FLOAT,
00293                                   RECIPE_STRING,
00294                                   VISIR_UTIL_JOIN_PROCATG, plist,
00295                                   NULL, visir_pipe_id, proname));
00296 
00297     if (bpmframes != NULL) {
00298         /* 2) The bpm frame */
00299 
00300         frame = irplib_framelist_get_const(bpmframes, bshared ? 0 : i);
00301 
00302         filename = cpl_frame_get_filename(frame);
00303 
00304         cpl_propertylist_delete(plist);
00305         plist = cpl_propertylist_load_regexp(filename, 0, " ESO QC | ESO PRO ",
00306                                              0);
00307         skip_if(plist == NULL);
00308 
00309         bug_if(cpl_propertylist_append_string(plist, "EXTNAME",
00310                                               "BAD PIXEL MAP"));
00311 
00312         /* FIXME: Change to CPL_TYPE_UNSPECIFIED when supported */
00313         cpl_image_delete(image);
00314         image = cpl_image_load(filename, CPL_TYPE_INT, 0, 0);
00315         skip_if(image == NULL);
00316 
00317         /* FIXME: Change to CPL_TYPE_UNSPECIFIED when supported */
00318         skip_if(cpl_image_save(image, proname, CPL_BPP_8_UNSIGNED,
00319                                plist, CPL_IO_EXTEND));
00320     }
00321 
00322     if (errframes != NULL) {
00323         /* 3) The error frame */
00324 
00325         frame = irplib_framelist_get_const(errframes, eshared ? 0 : i);
00326 
00327         filename = cpl_frame_get_filename(frame);
00328 
00329         cpl_propertylist_delete(plist);
00330         plist = cpl_propertylist_load_regexp(filename, 0, " ESO QC | ESO PRO ",
00331                                              0);
00332         skip_if(plist == NULL);
00333 
00334         bug_if(cpl_propertylist_append_string(plist, "EXTNAME", "ERROR MAP"));
00335 
00336         /* FIXME: Change to CPL_TYPE_UNSPECIFIED when supported */
00337         cpl_image_delete(image);
00338         image = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, 0);
00339         skip_if(image == NULL);
00340 
00341         /* FIXME: Change to CPL_TYPE_UNSPECIFIED when supported */
00342         skip_if(cpl_image_save(image, proname, CPL_BPP_IEEE_FLOAT,
00343                                plist, CPL_IO_EXTEND));
00344 
00345         if (conframes == NULL && bpmframes == NULL) {
00346 
00347             cpl_propertylist_empty(plist);
00348             bug_if(cpl_propertylist_append_string(plist, "EXTNAME",
00349                                                   "BAD PIXEL MAP"));
00350 
00351             /* FIXME: Find a better way to create a image BPM */
00352             bpm = cpl_mask_threshold_image_create(image, -DBL_EPSILON, 1e8);
00353             cpl_mask_not(bpm);
00354             cpl_image_delete(image);
00355             image = cpl_image_new_from_mask(bpm);
00356 
00357             skip_if(cpl_image_save(image, proname, CPL_BPP_8_UNSIGNED,
00358                                    plist, CPL_IO_EXTEND));
00359         }
00360 
00361     }
00362 
00363 
00364     if (conframes != NULL) {
00365         /* 4) The contribution frame(s) */
00366 
00367         if (cshared || irplib_framelist_get_size(conframes) == n) {
00368 
00369             frame = irplib_framelist_get_const(conframes, cshared ? 0 : i);
00370 
00371             filename = cpl_frame_get_filename(frame);
00372 
00373             cpl_propertylist_delete(plist);
00374             plist = cpl_propertylist_load_regexp(filename, 0,
00375                                                  " ESO QC | ESO PRO ", 0);
00376             skip_if(plist == NULL);
00377 
00378             /* FIXME: Change to CPL_TYPE_UNSPECIFIED when supported */
00379             cpl_image_delete(image);
00380             image = cpl_image_load(filename, CPL_TYPE_INT, 0, 0);
00381             skip_if(image == NULL);
00382 
00383         } else {
00384 
00385             const int    ncon = irplib_framelist_get_size(conframes);
00386             const int    nz   = ncon / n;
00387             int          j;
00388 
00389             for (j = i * nz; j < i * nz + nz; j ++) {
00390                 frame = irplib_framelist_get_const(conframes, j);
00391                 bug_if(cpl_frameset_insert(usedframes,
00392                                            cpl_frame_duplicate(frame)));
00393 
00394                 filename = cpl_frame_get_filename(frame);
00395 
00396                 if (j == i * nz) {
00397 
00398                     cpl_image_delete(image);
00399                     image = cpl_image_load(filename, CPL_TYPE_INT, 0, 0);
00400                     skip_if(image == NULL);
00401 
00402                 } else {
00403                     csum = cpl_image_load(filename, CPL_TYPE_INT, 0, 0);
00404                     skip_if(csum == NULL);
00405                     skip_if(cpl_image_add(image, csum));
00406                 }
00407             }
00408 
00409             cpl_propertylist_empty(plist);
00410         }
00411 
00412         bug_if(cpl_propertylist_append_string(plist, "EXTNAME",
00413                                               "CONTRIBUTION MAP"));
00414 
00415         /* FIXME: Change to CPL_TYPE_UNSPECIFIED when supported */
00416         skip_if(cpl_image_save(image, proname, CPL_BPP_32_SIGNED,
00417                                plist, CPL_IO_EXTEND));
00418 
00419         if (bpmframes == NULL) {
00420 
00421             bug_if(cpl_propertylist_update_string(plist, "EXTNAME",
00422                                                   "BAD PIXEL MAP"));
00423 
00424             /* FIXME: Find a better way to create a image BPM */
00425             bug_if(cpl_image_threshold(image, -0.5, 1.0, 0.5, 1.0));
00426             bug_if(cpl_image_multiply_scalar(image, -1.0));
00427             bug_if(cpl_image_add_scalar(image, 1.0));
00428             skip_if(cpl_image_save(image, proname, CPL_BPP_8_UNSIGNED,
00429                                    plist, CPL_IO_EXTEND));
00430         }
00431     }
00432 
00433     if (wgtframes != NULL) {
00434         /* 5) The weight frame */
00435 
00436         frame = irplib_framelist_get_const(wgtframes, wshared ? 0 : i);
00437 
00438         filename = cpl_frame_get_filename(frame);
00439 
00440         cpl_propertylist_delete(plist);
00441         plist = cpl_propertylist_load_regexp(filename, 0, " ESO QC | ESO PRO ",
00442                                              0);
00443         skip_if(plist == NULL);
00444 
00445         bug_if(cpl_propertylist_append_string(plist, "EXTNAME", "WEIGHT MAP"));
00446 
00447         /* FIXME: Change to CPL_TYPE_UNSPECIFIED when supported */
00448         cpl_image_delete(image);
00449         image = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, 0);
00450         skip_if(image == NULL);
00451 
00452         /* FIXME: Change to CPL_TYPE_UNSPECIFIED when supported */
00453         skip_if(cpl_image_save(image, proname, CPL_BPP_IEEE_FLOAT,
00454                                plist, CPL_IO_EXTEND));
00455     }
00456 
00457     for (frame = cpl_frameset_get_first_const(products);
00458          frame != NULL;
00459          frame = cpl_frameset_get_next_const(products)) {
00460         cpl_frame * copy = cpl_frame_duplicate(frame);
00461         cpl_error_code error;
00462 
00463 #ifdef _OPENMP
00464 #pragma omp critical(visir_util_join_one)
00465 #endif
00466         error = cpl_frameset_insert(framelist, copy);
00467 
00468         if (error) break;
00469     }
00470     bug_if(frame != NULL);
00471 
00472     end_skip;
00473 
00474     cpl_image_delete(image);
00475     cpl_mask_delete(bpm);
00476     cpl_free(proname);
00477     cpl_propertylist_delete(plist);
00478     cpl_frameset_delete(usedframes);
00479     cpl_frameset_delete(products);
00480 
00481     return cpl_error_get_code();
00482 }

Generated on Mon Feb 6 15:23:49 2012 for VISIR Pipeline Reference Manual by  doxygen 1.5.8