00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031
00032
00033
00034
00035
00036 #include "visir_recipe.h"
00037
00038 #include <string.h>
00039
00040
00041
00042
00043
00044 #define RECIPE_STRING "visir_util_clip"
00045
00046 #ifndef VISIR_UTIL_CLIP_KEEPFRAC
00047 #define VISIR_UTIL_CLIP_KEEPFRAC 0.9
00048 #endif
00049 #ifndef VISIR_UTIL_CLIP_KAPPA
00050 #define VISIR_UTIL_CLIP_KAPPA 3.0
00051 #endif
00052
00053 #ifndef VISIR_UTIL_CLIP_MAXITE
00054 #define VISIR_UTIL_CLIP_MAXITE 3
00055 #endif
00056
00057 #define VISIR_UTIL_CLIP_QC_MAP_MAX "ESO QC CONTRIBUTION MAX"
00058 #define VISIR_UTIL_CLIP_QC_MAP_MEAN "ESO QC CONTRIBUTION MEAN"
00059 #define VISIR_UTIL_CLIP_QC_MAP_MEDIAN "ESO QC CONTRIBUTION MEDIAN"
00060
00061
00062
00063
00064
00065 static
00066 cpl_error_code visir_util_clip_kappa_sigma_double(cpl_imagelist *,
00067 cpl_imagelist *,
00068 double, double, int,
00069 const int *);
00070 static
00071 cpl_error_code visir_util_clip_kappa_sigma_float(cpl_imagelist *,
00072 cpl_imagelist *,
00073 double, double, int,
00074 const int *);
00075 static
00076 cpl_error_code visir_util_clip_kappa_sigma_int(cpl_imagelist *,
00077 cpl_imagelist *,
00078 double, double, int,
00079 const int *);
00080
00081 static cpl_error_code visir_util_clip_one(cpl_frameset *,
00082 irplib_framelist *,
00083 irplib_framelist *,
00084 int, cpl_boolean,
00085 const cpl_parameterlist *);
00086
00087 static cpl_error_code visir_util_clip_kappa_sigma(cpl_imagelist *,
00088 cpl_imagelist *,
00089 const cpl_parameterlist *,
00090 const int * shifts);
00091
00092
00093 cpl_recipe_define(visir_util_clip, VISIR_BINARY_VERSION,
00094 "Lars Lundin", PACKAGE_BUGREPORT, "2011",
00095 "Kappa-sigma clipping of outliers for each pixel",
00096 "The files listed in the Set Of Frames (sof-file) "
00097 "must be tagged pair-wise:\n"
00098 "VISIR-raw-file.fits " VISIR_UTIL_INPUTS_RAW "\n"
00099 "VISIR-bpm-file.fits " VISIR_CALIB_BPM "\n"
00100 "\nThe product(s) will have a FITS card\n"
00101 "'HIERARCH ESO PRO CATG' with a value of:\n"
00102 VISIR_IMG_CLIPPED_PROCATG "\n"
00103 "The outliers are marked as rejected in the matching\n"
00104 "bad pixel map.");
00105
00106
00110
00111
00112
00113
00114
00115
00116
00124
00125 static
00126 cpl_error_code visir_util_clip_fill_parameterlist(cpl_parameterlist * self)
00127 {
00128
00129 const char * context = PACKAGE "." RECIPE_STRING;
00130 cpl_error_code err;
00131
00132 cpl_ensure_code(self, CPL_ERROR_NULL_INPUT);
00133
00134
00135
00136
00137 err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
00138 "keepfrac", VISIR_UTIL_CLIP_KEEPFRAC,
00139 NULL, context, "The fraction of "
00140 "pixels to keep for the initial"
00141 "median");
00142 cpl_ensure_code(!err, err);
00143
00144
00145
00146 err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
00147 "kappa", VISIR_UTIL_CLIP_KAPPA,
00148 NULL, context, "Clip outside "
00149 "+ or - kappa * sigma "
00150 "(the standard deviation)");
00151 cpl_ensure_code(!err, err);
00152
00153
00154
00155 err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING,
00156 "maxite", VISIR_UTIL_CLIP_MAXITE, NULL,
00157 context, "Max number of kappa-sigma "
00158 "clipping iterations");
00159
00160
00161 err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
00162 "shift-beams", CPL_TRUE, NULL,
00163 context, "Account for movements of the "
00164 "object defined in CRPIX[12]");
00165 cpl_ensure_code(!err, err);
00166
00167
00168 err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
00169 "error", CPL_TRUE, NULL,
00170 context, "Output clipped standard "
00171 "deviation as error map");
00172 cpl_ensure_code(!err, err);
00173
00174
00175 err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
00176 "badimage", 0.2, NULL,
00177 context, "If percentage of clipped "
00178 "pixels above this value the whole "
00179 "image is considered bad");
00180 cpl_ensure_code(!err, err);
00181
00182 return CPL_ERROR_NONE;
00183
00184 }
00185
00186
00193
00194 static int visir_util_clip(cpl_frameset * framelist,
00195 const cpl_parameterlist * parlist)
00196 {
00197 cpl_errorstate cleanstate = cpl_errorstate_get();
00198 cpl_error_code didfail = CPL_ERROR_NONE;
00199 irplib_framelist * allframes = NULL;
00200 irplib_framelist * rawframes = NULL;
00201 irplib_framelist * bpmframes = NULL;
00202 cpl_size i, n, nbad = 0;
00203
00204
00205
00206 skip_if (visir_dfs_set_groups(framelist));
00207
00208
00209 allframes = irplib_framelist_cast(framelist);
00210 skip_if(allframes == NULL);
00211 rawframes = irplib_framelist_extract(allframes, VISIR_UTIL_INPUTS_RAW);
00212 skip_if (rawframes == NULL);
00213 bpmframes = irplib_framelist_extract(allframes, VISIR_CALIB_BPM);
00214 if (bpmframes == NULL)
00215 cpl_errorstate_set(cleanstate);
00216 else
00217 nbad = irplib_framelist_get_size(bpmframes);
00218
00219 n = irplib_framelist_get_size(rawframes);
00220 error_if(nbad != n && nbad != 1 && nbad != 0, CPL_ERROR_INCOMPATIBLE_INPUT,
00221 "%d raw-frames <=> %d bpm frames", n, nbad);
00222
00223 #ifdef _OPENMP
00224 #pragma omp parallel for private(i)
00225 #endif
00226 for (i = 0; i < n; i++) {
00227 if (!didfail) {
00228
00229
00230
00231
00232
00233
00234
00235
00236 if (visir_util_clip_one(framelist, rawframes, bpmframes, i,
00237 nbad == 1, parlist)) {
00238 const cpl_error_code errori = cpl_error_set_where(cpl_func);
00239 #ifdef _OPENMP
00240
00241
00242 cpl_errorstate_dump(cleanstate, CPL_FALSE, NULL);
00243 cpl_errorstate_set(cleanstate);
00244 #pragma omp critical(visir_util_clip)
00245 #endif
00246 didfail = errori;
00247 }
00248 }
00249 }
00250
00251 error_if(didfail, didfail, "Failed to clip %d frame(s)", n);
00252
00253 end_skip;
00254
00255 irplib_framelist_delete(allframes);
00256 irplib_framelist_delete(rawframes);
00257 irplib_framelist_delete(bpmframes);
00258
00259 return cpl_error_get_code();
00260 }
00261
00262 static inline int round_to_int(const double x)
00263 {
00264 return x >= 0 ? (int)(x + 0.5) : (int)(x - 0.5);
00265 }
00266
00267
00268
00279
00280 static cpl_error_code visir_util_clip_one(cpl_frameset * framelist,
00281 irplib_framelist * rawframes,
00282 irplib_framelist * bpmframes,
00283 int i, cpl_boolean bshared,
00284 const cpl_parameterlist * parlist)
00285 {
00286
00287 cpl_frameset * products = cpl_frameset_new();
00288 cpl_frameset * usedframes = cpl_frameset_new();
00289 const cpl_frame * frame;
00290
00291 char * bpmname = cpl_sprintf(RECIPE_STRING "_bpm_%03d" CPL_DFS_FITS, i);
00292 char * mapname = cpl_sprintf(RECIPE_STRING "_map_%03d" CPL_DFS_FITS, i);
00293 char * errname = cpl_sprintf(RECIPE_STRING "_error_%03d" CPL_DFS_FITS, i);
00294
00295 const int n = irplib_framelist_get_size(rawframes);
00296 cpl_frameset * rawone = cpl_frameset_new();
00297 cpl_frameset * bpmone = cpl_frameset_new();
00298 cpl_frame * rawframe =
00299 cpl_frame_duplicate(irplib_framelist_get_const(rawframes, i));
00300 cpl_frame * bpmframe = bpmframes ?
00301 cpl_frame_duplicate(irplib_framelist_get_const(bpmframes,
00302 bshared ? 0 : i))
00303 : NULL;
00304 const cpl_error_code errr = cpl_frameset_insert(rawone, rawframe);
00305 const cpl_error_code errb = bpmframe ?
00306 cpl_frameset_insert(bpmone, bpmframe) : CPL_ERROR_NONE;
00307
00308 const cpl_boolean berr = irplib_parameterlist_get_bool(parlist, PACKAGE,
00309 RECIPE_STRING,
00310 "error");
00311
00312 const cpl_boolean bshifts = irplib_parameterlist_get_bool(parlist, PACKAGE,
00313 RECIPE_STRING,
00314 "shift-beams");
00315
00316 const double badimage = irplib_parameterlist_get_double(parlist, PACKAGE,
00317 RECIPE_STRING,
00318 "badimage");
00319
00320 cpl_imagelist * devlist = cpl_imagelist_new();
00321 cpl_imagelist * rawlist =
00322 cpl_imagelist_load_frameset(rawone, CPL_TYPE_UNSPECIFIED, 0, -1);
00323 cpl_imagelist * bpmlist = bpmframe ?
00324 cpl_imagelist_load_frameset(bpmone, CPL_TYPE_INT, 0, -1) : NULL;
00325
00326 #ifdef CPL_BPP_IEEE_FLOAT
00327
00328 const cpl_type_bpp stdev_save_type = CPL_TYPE_UNSPECIFIED;
00329 #else
00330 const cpl_type_bpp stdev_save_type = cpl_image_get_type
00331 (cpl_imagelist_get_const(rawlist, 0)) == CPL_TYPE_DOUBLE
00332 ? CPL_BPP_IEEE_DOUBLE : CPL_BPP_IEEE_FLOAT;
00333 #endif
00334
00335 cpl_mask * bpm = NULL;
00336 cpl_image * newbpmimg = NULL;
00337 cpl_image * map = NULL;
00338 cpl_propertylist * qclist = cpl_propertylist_new();
00339 cpl_propertylist * xtlist = cpl_propertylist_new();
00340
00341 const int m = rawlist ? cpl_imagelist_get_size(rawlist) : 0;
00342 const int nbpm = bpmlist ? cpl_imagelist_get_size(bpmlist) : 0;
00343 int j;
00344 int mapmax, mapmean, mapmedian;
00345 int * shifts = cpl_calloc(m * 2, sizeof(int));
00346
00347 skip_if(0);
00348
00349 for (int e = 0; bshifts && e < m; e++) {
00350 const cpl_propertylist * plist;
00351 double crpix1, crpix2;
00352 double fx, fy;
00353 cpl_errorstate prestate = cpl_errorstate_get();
00354
00355 irplib_framelist_load_propertylist(rawframes, i, e + 1, "^("
00356 IRPLIB_PFITS_WCS_REGEXP ")$",
00357 CPL_FALSE);
00358 plist = irplib_framelist_get_propertylist_const(rawframes, i);
00359 if (!cpl_propertylist_has(plist, "CRPIX1") ||
00360 !cpl_propertylist_has(plist, "CRPIX2")) {
00361 cpl_errorstate_set(prestate);
00362 break;
00363 }
00364
00365 crpix1 = irplib_pfits_get_double(plist, "CRPIX1");
00366 crpix2 = irplib_pfits_get_double(plist, "CRPIX2");
00367
00368 if (e == 0) {
00369 shifts[0] = 0;
00370 shifts[1] = 0;
00371 fx = crpix1;
00372 fy = crpix2;
00373 } else {
00374 shifts[e * 2] = round_to_int(-(fx - crpix1));
00375 shifts[e * 2 + 1] = round_to_int(-(fy - crpix2));
00376 }
00377 cpl_msg_debug(cpl_func, "CRPIX shifts %d %d, %f %f", shifts[e * 2],
00378 shifts[e * 2 + 1], crpix1 - fx, crpix2 - fy);
00379 }
00380
00381 skip_if(rawlist == NULL);
00382
00383 bug_if(errr);
00384 bug_if(errb);
00385
00386 error_if(nbpm != 1 && nbpm != m && nbpm != 0, CPL_ERROR_INCOMPATIBLE_INPUT,
00387 "Frame-pair %d/%d: %d image(s) <=> %d bad pixel map(s)", 1+i, n,
00388 m, (int)cpl_imagelist_get_size(bpmlist));
00389
00390 bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(rawframe)));
00391 if (bpmframe)
00392 bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate(bpmframe)));
00393
00394 for (j = 0; bpmframe && j < m; j++) {
00395 const cpl_image * bpmimg =
00396 cpl_imagelist_get_const(bpmlist, nbpm > 1 ? j : 0);
00397 cpl_image * rawimg = cpl_imagelist_get(rawlist, j);
00398
00399 cpl_mask_delete(bpm);
00400 bpm = cpl_mask_threshold_image_create(bpmimg, 0.5, FLT_MAX);
00401
00402 bug_if(cpl_image_reject_from_mask(rawimg, bpm));
00403
00404 }
00405
00406
00407 skip_if(visir_util_clip_kappa_sigma(rawlist, devlist, parlist, shifts));
00408
00409
00410 {
00411 cpl_image * img = cpl_imagelist_get(rawlist, 0);
00412 const int total_size =
00413 cpl_image_get_size_x(img) * cpl_image_get_size_y(img);
00414 cpl_binary * _allbad = cpl_malloc(sizeof(cpl_binary) * total_size);
00415 cpl_mask * allbad;
00416
00417 memset(_allbad, CPL_BINARY_1, sizeof(cpl_binary) * total_size);
00418 allbad = cpl_mask_wrap(cpl_image_get_size_x(img),
00419 cpl_image_get_size_y(img), _allbad);
00420
00421 for (int i = 0; i < cpl_imagelist_get_size(rawlist); i++) {
00422 cpl_mask * mask = cpl_image_get_bpm(cpl_imagelist_get(rawlist, i));
00423 img = cpl_imagelist_get(rawlist, i);
00424 if (badimage < (double)cpl_mask_count(mask) / total_size)
00425 cpl_image_reject_from_mask(img, allbad);
00426 }
00427
00428 cpl_mask_delete(allbad);
00429 }
00430
00431
00432 bug_if(cpl_imagelist_get_size(devlist) != 2);
00433
00434 map = cpl_image_new_from_accepted(rawlist);
00435 mapmax = cpl_image_get_max(map);
00436 mapmean = cpl_image_get_mean(map);
00437 mapmedian = cpl_image_get_median(map);
00438 bug_if(cpl_propertylist_append_int(qclist, VISIR_UTIL_CLIP_QC_MAP_MAX,
00439 mapmax));
00440 bug_if(cpl_propertylist_append_int(qclist, VISIR_UTIL_CLIP_QC_MAP_MEAN,
00441 mapmean));
00442 bug_if(cpl_propertylist_append_int(qclist, VISIR_UTIL_CLIP_QC_MAP_MEDIAN,
00443 mapmedian));
00444 bug_if(cpl_propertylist_set_comment(qclist, VISIR_UTIL_CLIP_QC_MAP_MAX,
00445 "The maximum contribution on a pixel"));
00446 bug_if(cpl_propertylist_set_comment(qclist, VISIR_UTIL_CLIP_QC_MAP_MEAN,
00447 "The mean contribution on a pixel"));
00448 bug_if(cpl_propertylist_set_comment(qclist, VISIR_UTIL_CLIP_QC_MAP_MEDIAN,
00449 "The median contribution on a pixel"));
00450
00451 #ifdef VISIR_MIME_CUBE_SUPPORT
00452
00453 #error "Save data as a cube in the primary data unit"
00454 #else
00455
00456 if (bpmframe)
00457 skip_if(irplib_dfs_save_propertylist(products, parlist, usedframes,
00458 RECIPE_STRING,
00459 VISIR_IMG_CLIPPED_PROCATG, qclist,
00460 NULL, visir_pipe_id, bpmname));
00461
00462 if (berr)
00463 skip_if(irplib_dfs_save_propertylist(products, parlist, usedframes,
00464 RECIPE_STRING,
00465 "ERROR_MAP", qclist,
00466 NULL, visir_pipe_id, errname));
00467
00468 for (j = 0; j < m; j++) {
00469 cpl_image * rawimg = cpl_imagelist_get(rawlist, j);
00470
00471 if (bpmframe) {
00472 const cpl_mask * newbpm = cpl_image_get_bpm_const(rawimg);
00473
00474 cpl_image_delete(newbpmimg);
00475 newbpmimg = cpl_image_new_from_mask(newbpm);
00476
00477 skip_if(cpl_image_save(newbpmimg, bpmname, CPL_BPP_8_UNSIGNED,
00478 NULL, CPL_IO_EXTEND));
00479 }
00480
00481 if (berr)
00482 skip_if(cpl_image_save(cpl_imagelist_get_const(devlist, 1), errname,
00483 stdev_save_type, NULL, CPL_IO_EXTEND));
00484 }
00485 #endif
00486
00487 skip_if(irplib_dfs_save_image(products, parlist, usedframes,
00488 map, m < 256 ? CPL_BPP_8_UNSIGNED
00489 : (m < 65536 ? CPL_BPP_16_UNSIGNED
00490 : CPL_BPP_32_SIGNED), RECIPE_STRING,
00491 VISIR_IMG_CLIPPED_MAP_PROCATG, qclist,
00492 NULL, visir_pipe_id, mapname));
00493
00494 bug_if(cpl_propertylist_append_string(xtlist, "EXTNAME", "NO CLIP STANDARD "
00495 "DEVIATION MAP"));
00496
00497 skip_if(cpl_image_save(cpl_imagelist_get_const(devlist, 0), mapname,
00498 stdev_save_type, xtlist, CPL_IO_EXTEND));
00499
00500 bug_if(cpl_propertylist_update_string(xtlist, "EXTNAME", "CLIPPED STANDARD "
00501 "DEVIATION MAP"));
00502
00503 skip_if(cpl_image_save(cpl_imagelist_get_const(devlist, 1), mapname,
00504 stdev_save_type, xtlist, CPL_IO_EXTEND));
00505
00506 for (frame = cpl_frameset_get_first_const(products);
00507 frame != NULL;
00508 frame = cpl_frameset_get_next_const(products)) {
00509 cpl_frame * copy = cpl_frame_duplicate(frame);
00510 cpl_error_code error;
00511
00512 #ifdef _OPENMP
00513 #pragma omp critical(visir_util_clip_one)
00514 #endif
00515 error = cpl_frameset_insert(framelist, copy);
00516
00517 if (error) break;
00518 }
00519
00520 bug_if(frame != NULL);
00521
00522
00523 end_skip;
00524
00525 cpl_free(bpmname);
00526 cpl_free(mapname);
00527 cpl_free(errname);
00528 cpl_free(shifts);
00529
00530 cpl_frameset_delete(rawone);
00531 cpl_frameset_delete(bpmone);
00532 cpl_frameset_delete(usedframes);
00533 cpl_frameset_delete(products);
00534
00535 cpl_propertylist_delete(qclist);
00536 cpl_propertylist_delete(xtlist);
00537
00538 cpl_mask_delete(bpm);
00539 cpl_image_delete(newbpmimg);
00540 cpl_image_delete(map);
00541 cpl_imagelist_delete(rawlist);
00542 cpl_imagelist_delete(bpmlist);
00543 cpl_imagelist_delete(devlist);
00544
00545 return cpl_error_get_code();
00546
00547 }
00548
00549
00550
00558
00559 static
00560 cpl_error_code visir_util_clip_kappa_sigma(cpl_imagelist * self,
00561 cpl_imagelist * devlist,
00562 const cpl_parameterlist * parlist,
00563 const int * shifts)
00564 {
00565
00566 const double keepfrac = irplib_parameterlist_get_double(parlist, PACKAGE,
00567 RECIPE_STRING,
00568 "keepfrac");
00569 const double kappa = irplib_parameterlist_get_double(parlist, PACKAGE,
00570 RECIPE_STRING,
00571 "kappa");
00572 const int maxite = irplib_parameterlist_get_int(parlist, PACKAGE,
00573 RECIPE_STRING,
00574 "maxite");
00575 const cpl_image * img = cpl_imagelist_get_const(self, 0);
00576
00577 switch (cpl_image_get_type(img)) {
00578 case CPL_TYPE_DOUBLE:
00579 skip_if(visir_util_clip_kappa_sigma_double(self, devlist, keepfrac,
00580 kappa, maxite, shifts));
00581 break;
00582 case CPL_TYPE_FLOAT:
00583 skip_if(visir_util_clip_kappa_sigma_float(self, devlist, keepfrac,
00584 kappa, maxite, shifts));
00585 break;
00586 case CPL_TYPE_INT:
00587 skip_if(visir_util_clip_kappa_sigma_int(self, devlist, keepfrac,
00588 kappa, maxite, shifts));
00589 break;
00590 default:
00591 bug_if( 1 );
00592 }
00593
00594 end_skip;
00595
00596 return cpl_error_get_code();
00597
00598 }
00599
00600
00601
00602
00603 #define CONCAT(a,b) a ## _ ## b
00604 #define CONCAT2X(a,b) CONCAT(a,b)
00605
00606 #define PIXEL_TYPE double
00607 #define STDEV_TYPE CPL_TYPE_DOUBLE
00608 #define PIXEL_TYPE_CPL CPL_TYPE_DOUBLE
00609 #include "visir_util_clip_body.c"
00610 #undef PIXEL_TYPE
00611 #undef STDEV_TYPE
00612 #undef PIXEL_TYPE_CPL
00613
00614 #define PIXEL_TYPE float
00615 #define PIXEL_TYPE_CPL CPL_TYPE_FLOAT
00616 #define STDEV_TYPE CPL_TYPE_FLOAT
00617 #include "visir_util_clip_body.c"
00618 #undef PIXEL_TYPE
00619 #undef STDEV_TYPE
00620 #undef PIXEL_TYPE_CPL
00621
00622 #define PIXEL_TYPE int
00623 #define PIXEL_TYPE_CPL CPL_TYPE_INT
00624 #define STDEV_TYPE CPL_TYPE_FLOAT
00625 #include "visir_util_clip_body.c"
00626 #undef PIXEL_TYPE
00627 #undef STDEV_TYPE
00628 #undef PIXEL_TYPE_CPL