00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027 #ifdef HAVE_CONFIG_H
00028 #include <config.h>
00029 #endif
00030
00031
00037
00040
00041
00042
00043 #include <math.h>
00044
00045 #include <xsh_drl.h>
00046 #include <xsh_data_pre.h>
00047 #include <xsh_dfs.h>
00048 #include <xsh_pfits.h>
00049 #include <xsh_error.h>
00050 #include <xsh_msg.h>
00051 #include <xsh_badpixelmap.h>
00052
00053 #include <cpl.h>
00054
00055
00056
00057
00058
00059
00060
00061
00062
00063
00064
00065
00066
00067 static void add_noisy_pixel_to_ascii_file( int x, int y )
00068 {
00069 static FILE *fout ;
00070 const char * fname = "noisy_pixels.dat" ;
00071
00072 if ( fout == NULL )
00073 fout = fopen( fname, "w" ) ;
00074 fprintf( fout, "%d %d\n", x, y ) ;
00075 }
00076
00089 static int
00090 flag_noisy_pixels (cpl_imagelist * raws,
00091 cpl_image * bpmap,
00092 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00093 cpl_image * noisymap,
00094 #endif
00095 int nx, int ny,
00096 xsh_clipping_param * noise_clipping,xsh_instrument* inst)
00097 {
00098 int ix, iy, i;
00099 cpl_image **pimg = NULL;
00100 int nimg, nbad = 0, npix = 0;
00101 double medStackv = 0.,
00102 aveStackv = 0.,
00103 sumStackv = 0., errorStackv = 0.;
00104 double *Stackv = NULL,
00106 *pStackv = NULL;
00107 double *stack = NULL;
00108 double *bisStackv = NULL,
00109 *pbis = NULL;
00110 double diff = 0.0, diff2 = 0.0;
00111 double min = 999999., max = -9999999.;
00112 double maxdelta = 0.0;
00113 cpl_binary *bpmap_mask = NULL;
00114 int idpix = 0;
00115 float ** pixdata = NULL;
00116 cpl_binary **pixmask = NULL;
00117 int nhot = 0;
00118 int totpix = nx * ny;
00119 nimg = cpl_imagelist_get_size(raws);
00120
00121 XSH_MALLOC(pimg, cpl_image*, nimg);
00122
00123
00124 for( i = 0; i < nimg; i++) {
00125 *(pimg + i) = cpl_imagelist_get (raws, i);
00126 }
00127
00128 XSH_MALLOC(Stackv, double, nx * ny);
00129
00130 pStackv = Stackv;
00131
00132 xsh_msg_dbg_low( "Rejected in bpmap: %" CPL_SIZE_FORMAT "", cpl_image_count_rejected( bpmap ));
00133
00134
00135 XSH_MALLOC(stack, double, nimg);
00136
00137
00138 bpmap_mask = cpl_mask_get_data(cpl_image_get_bpm(bpmap));
00139 assure( bpmap_mask != NULL, cpl_error_get_code(),
00140 "Cant get bpmap Mask" );
00141
00142 XSH_MALLOC(bisStackv, double, totpix);
00143 pbis = bisStackv;
00144
00145
00146 {
00147 int j;
00148 XSH_MALLOC(pixdata, float *, nimg);
00149 XSH_MALLOC (pixmask,cpl_binary *,nimg);
00150
00151 for (j = 0; j < nimg; j++) {
00152 *(pixdata + j) = cpl_image_get_data_float(*(pimg + j));
00153 *(pixmask + j) = cpl_mask_get_data(cpl_image_get_bpm(*(pimg + j)));
00154 }
00155 }
00156 idpix = 0;
00157
00158 for (iy = 1; iy <= ny; iy++) {
00159 for (ix = 1; ix <= nx; ix++) {
00160 int j, count = 0;
00161 double sum = 0.;
00162
00163
00164 if ((*(bpmap_mask + idpix) & inst->decode_bp) > 0) {
00165 nbad++;
00166 xsh_msg_dbg_medium("Pixel %d,%d globally bad", ix, iy);
00167 *pStackv = 0.;
00168 } else {
00169 for (j = 0; j < nimg; j++) {
00170 int rej;
00171 double fval = *(*(pixdata + j) + idpix);
00172 rej = (*(*(pixmask + j) + idpix ) & inst->decode_bp);
00173
00174 if (rej > 0) {
00175
00176 xsh_msg_dbg_medium("Dont use pixel %d,%d [%d]", ix, iy, j);
00177 continue;
00178 }
00179 sum += fval;
00180
00181 *(stack + count) = fval;
00182 count++;
00183 }
00184 }
00185
00186 if (count > 1) {
00187
00188 double avg = sum / (double) count;
00189 double dif, dif2 = 0., sigma = 0.;
00190 int k;
00191
00192 xsh_msg_dbg_medium(
00193 " [%d,%d] Count=%d - sum = %lf, avg = %lf", ix, iy, count, sum, avg);
00194
00195 for (k = 0; k < count; k++) {
00196 xsh_msg_dbg_medium( " stack[%d] = %lf", k, *(stack+k));
00197 dif = *(stack + k) - avg;
00198 dif2 += dif * dif;
00199 }
00200 sigma = sqrt(dif2 / (double) (count - 1));
00201 *pStackv = sigma;
00202 *pbis++ = sigma;
00203 sumStackv += sigma;
00204 xsh_msg_dbg_medium(
00205 "++ dif2 = %lf, count= %d, sigma=%lf, Stackv=%lf, sum=%lf", dif2, count, sigma, *pStackv, sumStackv);
00206 if ((npix % 500) == 0)
00207 xsh_msg_dbg_medium(
00208 "+++++ (%d,%d) sumStackv = %lf", ix, iy, sumStackv);
00209 npix++;
00210 } else {
00211 xsh_msg_dbg_medium( "Not enough good pixels (%d)", count);
00212 *pStackv = 0.;
00213 }
00214 pStackv++;
00215 idpix++;
00216 }
00217 }
00218 xsh_msg_dbg_medium( "+++++ sumStackv = %lf", sumStackv);
00219
00220
00221
00222
00223 medStackv = xsh_tools_get_median_double(bisStackv, npix);
00224
00225
00226 aveStackv = sumStackv / (double) npix;
00227 xsh_msg_dbg_low(
00228 "--- Npix=%d - sumStackv=%lf - aveStackv=%lf", npix, sumStackv, aveStackv);
00229
00230
00231
00232 for (pStackv = bisStackv, i = 0; i < npix; i++, pStackv++) {
00233 diff = *pStackv - aveStackv;
00234 if (diff < min)
00235 min = diff;
00236 if (diff > max)
00237 max = diff;
00238 diff2 += diff * diff;
00239 }
00240 cpl_free(bisStackv);
00241 bisStackv = NULL;
00242
00243 xsh_msg_dbg_low("Diff Min: %lf, Max: %lf", min, max);
00244 errorStackv = sqrt(diff2 / (double) (npix - 1));
00245 xsh_msg_dbg_low( "errorStackv=%lf", errorStackv);
00246
00247
00248 maxdelta = noise_clipping->sigma * errorStackv;
00249
00250 xsh_msg_dbg_low(
00251 "Npix:%d/%d, sum: %lf, med:%lf, avg:%lf, err:%lf, delta: %lf", npix, nx * ny, sumStackv, medStackv, aveStackv, errorStackv, maxdelta);
00252
00253 pStackv = Stackv;
00254 idpix = 0;
00255
00256 for (iy = 1; iy <= ny; iy++) {
00257 for (ix = 1; ix <= nx; ix++) {
00258 if ( (*(bpmap_mask + idpix) & inst->decode_bp) == 0 ) {
00259 double delta = *pStackv - medStackv;
00260 if (fabs(delta) > maxdelta) {
00261 nhot++;
00262
00263 xsh_bpmap_set_bad_pixel(bpmap, ix, iy, QFLAG_ELECTRONIC_PICKUP);
00264 if (xsh_debug_level_get() >= XSH_DEBUG_LEVEL_LOW)
00265 add_noisy_pixel_to_ascii_file(ix, iy);
00266 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00267
00268 xsh_bpmap_set_bad_pixel(noisymap, ix, iy, QFLAG_ELECTRONIC_PICKUP);
00269 #endif
00270
00271 }
00272
00273 }
00274 pStackv++;
00275 idpix++;
00276 }
00277 }
00278 xsh_msg( "Found %d Electronic Pickup Noise Hot Pixels", nhot );
00279
00280 cleanup:
00281 XSH_FREE(stack);
00282 XSH_FREE(Stackv);
00283 XSH_FREE(bisStackv);
00284 XSH_FREE(pimg);
00285 XSH_FREE(pixdata);
00286 XSH_FREE(pixmask);
00287 return nhot;
00288 }
00289
00290 static void
00291 set_pickup_noise_pixels_qc (cpl_propertylist * header, int nbad,
00292 xsh_instrument * instrument)
00293 {
00294 xsh_pfits_set_qc( header, (void *)&nbad,
00295 XSH_QC_BP_MAP_PICKUP_NOISE_PIX, instrument ) ;
00296 }
00297
00308 cpl_frame *
00309 xsh_compute_noise_map (cpl_imagelist * dataList,
00310 cpl_frame * medFrame,
00311 xsh_clipping_param * noise_clipping,
00312 xsh_instrument* instr,
00313 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00314 cpl_frame ** noisyFrame
00315 #endif
00316 )
00317 {
00318 int nx, ny, nframes = 0;
00319 cpl_image *resBpMap = NULL;
00320 cpl_propertylist *bpmapHeader = NULL;
00321 cpl_frame *resFrame = NULL;
00322 int prevnbad;
00323 const char *bpmapFile = cpl_frame_get_filename (medFrame);
00324 xsh_pre *medPre = NULL;
00325 int iter = 0,curnbad = 0;
00326 char result_name[256] ;
00327 const char* tag = XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,instr);
00328 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00329 char noisy_name[256];
00330 cpl_image * noisyMap = NULL ;
00331 cpl_propertylist * noisymapHeader = NULL ;
00332 #endif
00333 int binx=0;
00334 int biny=0;
00335 cpl_image* ima_aux=NULL;
00336
00337
00338
00339
00340
00341
00342
00343 cpl_msg_indent_more ();
00344 xsh_msg ("*** Removing Noisy Pixels (%s)",
00345 xsh_instrument_arm_tostring(instr));
00346
00347
00348 xsh_msg ("Bpmap file = \"%s\"", bpmapFile);
00349
00350
00351 medPre = xsh_pre_load (medFrame,instr);
00352 XSH_ASSURE_NOT_NULL(medPre);
00353
00354 resBpMap = xsh_pre_get_qual (medPre);
00355 bpmapHeader = medPre->qual_header;
00356
00357 nx = medPre->nx;
00358 ny = medPre->ny;
00359
00360 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00361 check( noisyMap = cpl_image_new( nx, ny, CPL_TYPE_INT ) ) ;
00362 check( noisymapHeader = cpl_propertylist_duplicate( bpmapHeader ) ) ;
00363 #endif
00364 xsh_msg( " Image size: %d,%d", nx, ny ) ;
00365
00366 xsh_set_image_cpl_bpmap (resBpMap, resBpMap, instr->decode_bp);
00367 prevnbad = cpl_image_count_rejected (resBpMap);
00368
00369 nframes = cpl_imagelist_get_size (dataList);
00370 xsh_msg ("%d dark images in image list", nframes);
00371
00372
00373 for (iter = 0; iter < noise_clipping->niter; iter++) {
00374 int nhot ;
00375
00376 xsh_msg (">>>> Iteration Nb %d/%d", iter+1, noise_clipping->niter);
00377 cpl_msg_indent_more ();
00378
00379
00380 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00381 nhot = flag_noisy_pixels (dataList, resBpMap, noisyMap,
00382 nx, ny, noise_clipping,instr);
00383 #else
00384 nhot = flag_noisy_pixels (dataList, resBpMap, nx, ny, noise_clipping);
00385 #endif
00386 cpl_msg_indent_less ();
00387 if ( nhot == 0 ) break ;
00388 }
00389
00390
00391 curnbad = cpl_image_count_rejected (resBpMap);
00392 xsh_msg ("End of noisy pixels, total bad pixels: %d", curnbad);
00393
00394 xsh_msg (" Nb of noisy pixels: %d", curnbad - prevnbad);
00395
00396
00397
00398
00399
00400
00401 strcpy(result_name,bpmapFile) ;
00402 xsh_msg ("save frame %s\n", result_name);
00403
00404
00405
00406
00407
00408
00409 check(set_pickup_noise_pixels_qc (medPre->data_header, curnbad, instr ));
00410
00411 check(xsh_pfits_set_qc_nhpix( medPre->data_header, curnbad - prevnbad )) ;
00412 if ( xsh_instrument_get_arm(instr) == XSH_ARM_NIR ) {
00413 sprintf(noisy_name,"%s.fits",XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,instr));
00414 } else {
00415 check(binx=xsh_pfits_get_binx(medPre->data_header));
00416 check(biny=xsh_pfits_get_biny(medPre->data_header));
00417 sprintf(noisy_name,"%s_%dx%d.fits",XSH_GET_TAG_FROM_ARM(XSH_BP_MAP_PN,
00418 instr),
00419 binx,biny);
00420 }
00421 ima_aux=cpl_image_cast(medPre->data,CPL_TYPE_FLOAT);
00422 check(cpl_image_save (ima_aux,result_name, CPL_BPP_IEEE_FLOAT,
00423 medPre->data_header, CPL_IO_DEFAULT));
00424 xsh_free_image(&ima_aux);
00425
00426 xsh_pfits_set_extname (medPre->errs_header, "ERRS");
00427 ima_aux=cpl_image_cast(medPre->errs,CPL_TYPE_FLOAT);
00428 check(cpl_image_save (ima_aux,result_name, CPL_BPP_IEEE_FLOAT,
00429 medPre->errs_header, CPL_IO_EXTEND));
00430 xsh_free_image(&ima_aux);
00431
00432
00433 xsh_pfits_set_extname (bpmapHeader, "QUAL");
00434 check(cpl_image_save (resBpMap, result_name, XSH_PRE_DATA_BPP, bpmapHeader,
00435 CPL_IO_EXTEND));
00436
00437
00438 check(resFrame=xsh_frame_product(result_name,tag,
00439 CPL_FRAME_TYPE_IMAGE,
00440 CPL_FRAME_GROUP_PRODUCT,
00441 CPL_FRAME_LEVEL_FINAL));
00442
00443 #if defined(PICKUP_NOISE_HOT_PIXEL_MAP)
00444
00445
00446 set_pickup_noise_pixels_qc (noisymapHeader, curnbad, instr );
00447 xsh_pfits_set_qc_nhpix( noisymapHeader, (double)(curnbad - prevnbad) ) ;
00448 cpl_image_save (noisyMap, noisy_name, XSH_PRE_DATA_BPP,
00449 noisymapHeader,
00450 CPL_IO_DEFAULT );
00451
00452 check(*noisyFrame=xsh_frame_product(noisy_name,tag,
00453 CPL_FRAME_TYPE_IMAGE,
00454 CPL_FRAME_GROUP_PRODUCT,
00455 CPL_FRAME_LEVEL_TEMPORARY));
00456
00457 #endif
00458
00459 cleanup:
00460
00461 xsh_free_propertylist(&noisymapHeader);
00462 xsh_free_image(&noisyMap);
00463 xsh_free_image(&ima_aux);
00464 xsh_pre_free( &medPre ) ;
00465
00466 return resFrame ;
00467 }
00468
00469
00470