procDetRon.c

00001 
00002 /******************************************************************************
00003 *******************************************************************************
00004 *               European Southern Observatory
00005 *          VLTI MIDI Maintenance Templates Software
00006 *
00007 * Module name:  procDetRon.c
00008 * Description:  Contains routines for processing the templates
00009 *
00010 * History:      
00011 * 16-Jun-04     (csabet) created
00012 *******************************************************************************
00013 ******************************************************************************/
00014 
00015 /******************************************************************************
00016 *   Compiler directives
00017 ******************************************************************************/
00018 
00019 /******************************************************************************
00020 *   Include files
00021 ******************************************************************************/
00022 #include <math.h>
00023 #include <stdio.h>
00024 #include <cpl.h>
00025 #include "midiGlobal.h"
00026 #include "midiLib.h"
00027 #include "memoryHandling.h"
00028 #include "statistics.h"
00029 #include "createProdDetRon.h"
00030 #include "cpl_image_stats.h"
00031 #include "cpl_image_io.h"
00032 #include "imageProcessing.h"
00033 #include "midiFitsUtility.h"
00034 #include "fitsAnalysisTec.h"
00035 #include "procDetRon.h"
00036 #include "errorHandling.h"
00037 #include "diagnostics.h"
00038 #include "qfits.h"
00039 
00040 /**********************************************************
00041 *   Constant definitions
00042 **********************************************************/
00043 
00044 /**********************************************************
00045 *   Global Variables 
00046 **********************************************************/
00047 
00048 /*============================ C O D E    A R E A ===========================*/
00049 
00050 
00051 
00052 /******************************************************************************
00053 *               European Southern Observatory
00054 *          VLTI MIDI Maintenance Templates Software
00055 *
00056 * Module name:  procDetRon
00057 * Input/Output: See function arguments to avoid duplication
00058 * Description:  Computes the detector read-out noise. This is the main routine for this
00059 *                application.
00060 *
00061 * History:      
00062 * 03-May-05     (csabet) Created
00063 ******************************************************************************/
00064 void procDetRon (
00065     MidiFiles    *fileNames,    // In: Pointer to file names
00066     int            *error)        // Ou: Error status
00067 {
00068 
00069     //    Local Declarations
00070     //    ------------------
00071     const char      routine[] = "procDetRon";
00072     ImageFormat        *format=NULL;
00073     DetRonNoise        *noiseProfile=NULL;
00074     int                numOfFiles;
00075     FILE            *signaturePtr=NULL;
00076     
00077     //    Algorithm
00078     //    ---------
00079     if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking      routine   '%s' \n", routine);
00080     if (diagnostic > 4) fprintf (midiReportPtr, "Invoking      routine   '%s' \n", routine);
00081 
00082     //    Write a signature
00083     signaturePtr = fopen ("MIDI_sig_ron.log", "w");
00084     fclose (signaturePtr);
00085 
00086     //    Reset status
00087     *error = 0;
00088     numOfFiles = 0;
00089     
00090     //    Allocate memory
00091     format = callocImageFormat ();
00092     
00093     analyseFitsDetRon (fileNames, format, &numOfFiles, error);
00094     if (*error)
00095     {
00096         midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot analyse DETRON");
00097         freeImageFormat (format);
00098         return;
00099     }
00100     
00101     noiseProfile = callocDetRon (format);
00102     computeDetRon (numOfFiles, format, fileNames, noiseProfile, error);
00103     if (*error)
00104     {
00105         midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot process DETRON");
00106         freeDetRon (noiseProfile);
00107         freeImageFormat (format);
00108         return;
00109     }
00110     
00111     createDetRonProd (fileNames, format, noiseProfile, error);
00112     if (*error)    midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot create DETRON products");
00113 
00114     //    Release memory
00115     freeDetRon (noiseProfile);
00116     freeImageFormat (format);
00117 
00118     return; 
00119 }
00120 /*****************************************************************************/
00121 
00122 
00123 
00124 /******************************************************************************
00125 *               European Southern Observatory
00126 *          VLTI MIDI Maintenance Templates Software
00127 *
00128 * Module name:  computeDetRon
00129 * Input/Output: See function arguments to avoid duplication
00130 * Description:  Computes the detector read-out noise.
00131 *                Compute the standard deviation pixel by pixel along the time sequence:
00132 *                This method allows one to obtain a noise map and distribution of noise 
00133 *                values. All    noise components contribute to the final value, including 
00134 *                a frame to frame variation in the offset that is easily removed in the 
00135 *                observations. To remove it, one can subtract from each frame the mean 
00136 *                value, channel by channel:
00137 *
00138 *                IMPORTANT:
00139 *                To obtain the noise in electrons, multiply the result in ADUs by the 
00140 *                conversion factor (145 e-/ADU).
00141 *
00142 *                Summary: Find standard deviation of each pixel along the time (frame axis). 
00143 *                Essentially you will end up with 320 X 240 number of standard deviations
00144 *    
00145 *
00146 * History:      
00147 * 23-Jun-04     (csabet) Created
00148 ******************************************************************************/
00149 void computeDetRon (
00150     int            numOfFiles,        // In: Number of data files
00151     ImageFormat    *maxFormat,        // In: Pointer to the maximum image format
00152     MidiFiles    *fileNames,        // In: Pointer to the MIDI file structure
00153     DetRonNoise *noiseProfile,    // Ou: Pointer to the detector noise data structure
00154     int            *error)            // Ou: Error status
00155 {
00156   
00157     // Local Declarations
00158     // ------------------
00159     const char  routine[] = "computeDetRon";
00160     char        *fileTemp, *classification, *firstFile;
00161     FILE           *inFitsBatchPtr;
00162     ImageFormat    *format;
00163     int            fileNumber, extNumOfImagingDataFile, subWindowSize;
00164     float        *arrayPixel;
00165     
00166     //    Algorithm
00167     //    ---------
00168     if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking      routine   '%s' \n", routine);
00169     if (diagnostic > 4) fprintf (midiReportPtr, "Invoking      routine   '%s' \n", routine);
00170 
00171     //    Reset status
00172     *error = 0;
00173     fileNumber = 0;
00174     noiseProfile->exists = 0;
00175     subWindowSize = maxFormat->iXWidth * maxFormat->iYWidth;
00176 
00177     //    Allocate memory
00178     classification = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
00179     firstFile = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
00180     fileTemp = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
00181     format = callocImageFormat ();
00182     arrayPixel = (float *) calloc (subWindowSize * maxFormat->numOfFrames, sizeof (float));
00183 
00184     //    Initialise
00185     format->hasData = 0;
00186         
00187     //    Open the list of files
00188     if ((inFitsBatchPtr = fopen (fileNames->inFitsBatch, "r")) == NULL)
00189     {
00190         midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, 
00191             "Cannot open input FITS file list\n                 No preprocessing has been carried out for this batch");
00192         freeImageFormat (format);
00193         free (fileTemp);
00194         free (classification);
00195         free (firstFile);
00196         free (arrayPixel);
00197         *error = 1;
00198         return;
00199     }
00200 
00201     //    Loop through the files and analyse
00202     while (fgets (fileTemp, MAX_STRING_LENGTH, inFitsBatchPtr) != NULL)
00203     {
00204         sprintf (classification, "%s", "");
00205         sscanf (fileTemp, "%s%s", fileNames->inFitsName, classification);
00206         if (diagnostic)cpl_msg_info(cpl_func,"\n   Processing file   %s \n", fileNames->inFitsName);
00207         fprintf(midiReportPtr, "\n   Processing file   %s \n", fileNames->inFitsName);
00208 
00209         //    Get 'extNumOfImagingDataFile' extension number of IMAGING_DATA in input file
00210         extNumOfImagingDataFile  = findImagingDataExtension (fileNames->inFitsName, TAB_IMAGING_DATA, error);
00211         if (*error)    break;
00212 
00213         //    Get Image Format parameters
00214         if (extNumOfImagingDataFile > 0)
00215         {
00216             getImageFormat (fileNames->inFitsName, extNumOfImagingDataFile, format, error);
00217             if (*error)    break;
00218         }
00219         else format->hasData = 0;
00220 
00221         //    Check if the file has data
00222         if (format->hasData)
00223         {
00224             //    Check Categ, Tech and Type and then compute the image size
00225             if ((strcmp (format->obsCatg, "CALIB") == 0) &&
00226                 (strcmp (format->obsTech, "IMAGE") == 0) &&
00227                 (strcmp (format->obsType, "BIAS") == 0))
00228             {
00229                 //    Increment file number but make sure it is not greater than the allocated
00230                 fileNumber++;
00231                 if (fileNumber > numOfFiles)
00232                 {
00233                     *error = 1;
00234                     midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Inconsistent file quantity");
00235                     break;
00236                 }                
00237 
00238                 //    Save the name of the first file
00239                 if (fileNumber == 1) sprintf (firstFile, "%s", fileNames->inFitsName);
00240                 
00241                 //    Compress data
00242                 compressDetRon (fileNumber, numOfFiles, maxFormat->numOfFrames, fileNames->inFitsName, 
00243                     extNumOfImagingDataFile, format, arrayPixel, noiseProfile, error);
00244                 if (*error)
00245                 {
00246                     sprintf (midiMessage, "Cannot compress data in %s", fileNames->inFitsName);
00247                     midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
00248                     break;
00249                 }
00250             }
00251             else
00252                 midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, "The above file is not suitable for this task");
00253         }
00254         else
00255         {
00256             if (diagnostic)
00257             {
00258                 sprintf (midiMessage, "No data tables in %s. Not processed", fileNames->inFitsName);
00259                 midiReportInfo (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
00260             }
00261         }
00262     }
00263 
00264     
00265     //    Check if reduction has been carried out
00266     if (*error || (fileNumber != numOfFiles))
00267         midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot create a noise pattern for this batch");
00268     else
00269     {
00270        cpl_msg_info(cpl_func,"\nDetector Readout Noise Inventry: \n");
00271        cpl_msg_info(cpl_func,"=============================== \n");
00272        cpl_msg_info(cpl_func,"   Expected number of data files  = %d\n", numOfFiles);
00273        cpl_msg_info(cpl_func,"   Number of data files processed = %d\n", fileNumber);
00274        cpl_msg_info(cpl_func,"\n");
00275 
00276         fprintf (midiReportPtr, "\nDetector Readout Noise Inventry: \n");
00277         fprintf (midiReportPtr, "=============================== \n");
00278         fprintf (midiReportPtr, "   Expected number of data files  = %d\n", numOfFiles);
00279         fprintf (midiReportPtr, "   Number of data files processed = %d\n", fileNumber);
00280         fprintf (midiReportPtr, "\n");
00281     }
00282 
00283     //    Close the file list
00284     fclose (inFitsBatchPtr);
00285     free (arrayPixel);
00286 
00287     //    Create Noise Pattern
00288     createNoisePattern (fileNumber, firstFile, maxFormat, noiseProfile, error);
00289 
00290     //    If no error set the exists flag
00291     if (!(*error)) noiseProfile->exists = 1;
00292     
00293     //    Release memory
00294     freeImageFormat (format);
00295     free (fileTemp);
00296     free (classification);    
00297     free (firstFile);
00298     
00299     return; 
00300 }
00301 /*****************************************************************************/
00302 
00303 
00304 
00305 /******************************************************************************
00306 *               European Southern Observatory
00307 *          VLTI MIDI Maintenance Templates Software
00308 *
00309 * Module name:  compressDetRon
00310 * Input/Output: See function arguments to avoid duplication
00311 * Description:  Compresses the DETRON data
00312 *
00313 * History:      
00314 * 25-Apr-06     (csabet) Created
00315 ******************************************************************************/
00316 void compressDetRon (
00317     int            fileNumber,            // In: File number
00318     int            numOfFiles,            // In: Total number of files expected
00319     int            maxNumOfFrames,        // In: Maximum number of frames expected
00320     char        *fileName,            // In: Name of file to process
00321     int            extensionNumber,    // In: Extension number of the IMAGE_DATA
00322     ImageFormat    *format,            // In: Pointer to the image format
00323     float        *arrayPixel,        // In: Pointer to the accumulated data
00324     DetRonNoise    *noiseProfile,        // Ou: Pointer to the detector noise data structure
00325     int            *error)                // Ou: Error status
00326 
00327 {
00328   
00329     //    Local Declarations
00330     //    ------------------
00331     const char  routine[] = "compressDetRon";
00332     qfits_table *pTable=NULL;
00333     short int   *inData;
00334     char        *tempStr, *dataName;
00335     int         i, k, frame, foundData = 0, indexData, pixel, subWindowSize, scalingOffset;
00336     static int    indexFrame;
00337     float        *arrayFrame, standDev, variance;
00338 
00339     
00340     //    Algorithm
00341     //    ---------
00342     if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking      routine   '%s' \n", routine);
00343     if (diagnostic > 4) fprintf (midiReportPtr, "Invoking      routine   '%s' \n", routine);
00344     
00345     //    Reset status
00346     *error = 0;
00347     subWindowSize = format->iXWidth * format->iYWidth;
00348     if (fileNumber == 1) indexFrame = 0;
00349     
00350     //    Open IMAGING_DATA = INDEX 2
00351     pTable = qfits_table_open (fileName, extensionNumber);
00352     if (!pTable)
00353     {
00354         midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot load IMAGING_DATA");
00355         *error = 1;
00356         return;
00357     }
00358         
00359     //    Get data table information
00360     for (i = 0; i < pTable->nc; i++)
00361     {
00362         if (strcmp (pTable->col[i].tlabel, "DATA1") == 0)
00363         {
00364             foundData = 1;
00365             indexData = i;
00366         }
00367     }
00368     if (foundData == 0)
00369     {
00370         midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot find requested columns in data FITS file");
00371         qfits_table_close (pTable);
00372         *error = 1;
00373         return;
00374     }
00375 
00376     //    Read column DATA
00377     inData = (short int*) qfits_query_column (pTable, indexData, NULL); 
00378 
00379     //    Get the scaling offset
00380     dataName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
00381     for (i = 14; i < 25; i++)
00382     {
00383         sprintf (dataName, "TZERO%d", i);
00384         tempStr = qfits_query_ext (fileName, dataName, extensionNumber);
00385         if (tempStr != NULL)
00386         {
00387             if (diagnostic)cpl_msg_info(cpl_func,"Scaling Offset = %s\n", tempStr);
00388             if (diagnostic) fprintf (midiReportPtr, "Scaling Offset = %s\n", tempStr);
00389             sscanf (tempStr, "%d", &scalingOffset);
00390             break;
00391         }
00392     }
00393     if (tempStr == NULL)
00394     {
00395         scalingOffset = 0;
00396         midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, "Cannot read Scaling Offset. It is set to 0");
00397     }
00398     free (dataName);
00399 
00400     //    Now load or/and compress
00401     for (pixel = 0; pixel < subWindowSize; pixel++)
00402     {
00403         //    Along the time axis (frame)
00404         for (frame = 0; frame < format->numOfFrames; frame++)
00405         {
00406             //    Load each pixel data along the time axis
00407             i = frame * subWindowSize + pixel;
00408             k = (frame+indexFrame) * subWindowSize + pixel;
00409             if (isnan (inData[i]))
00410             {
00411                 sprintf (midiMessage, "inData has an INVALID value at frame %d", (indexFrame+frame));
00412                 midiReportWarning (midiReportPtr, routine, __FILE__, __LINE__, midiMessage);
00413             }
00414             noiseProfile->aveImage[pixel] += (float) (inData[i] + scalingOffset);
00415             arrayPixel[k] = (float) (inData[i] + scalingOffset);
00416         }
00417     }
00418 
00419     //    When all files have been read, compute an average 
00420     //    image and create an image from pixels standard deviation
00421     if (fileNumber == numOfFiles)
00422     {
00423         arrayFrame = (float *) calloc (maxNumOfFrames, sizeof (float));
00424         for (pixel = 0; pixel < subWindowSize; pixel++)
00425         {
00426             noiseProfile->aveImage[pixel] /= maxNumOfFrames;
00427 
00428             //    Along the time axis (frame)
00429             for (frame = 0; frame < maxNumOfFrames; frame++)
00430             {
00431                 //    Load each pixel data along the time axis
00432                 i = frame * subWindowSize + pixel;
00433                 arrayFrame[frame] = arrayPixel[i];
00434             }
00435 
00436             //    Compute the standard deviation for the pixel array
00437             variance =  signalVariance (arrayFrame, 0, maxNumOfFrames, &standDev);
00438             noiseProfile->pattern[pixel] = standDev;
00439         }
00440         free (arrayFrame);
00441     }
00442 
00443     //    Set it correctly for the next call to this function
00444     indexFrame += format->numOfFrames;
00445 
00446     //    Release memory
00447     qfits_table_close (pTable);
00448     free (inData);
00449 
00450     return; 
00451 }
00452 /*****************************************************************************/
00453 
00454 
00455 
00456 /******************************************************************************
00457 *               European Southern Observatory
00458 *          VLTI MIDI Maintenance Templates Software
00459 *
00460 * Module name:  createNoisePattern
00461 * Input/Output: See function arguments to avoid duplication
00462 * Description:  Creates a noise pattern made up of standard deviations of image pixels
00463 *
00464 * History:      
00465 * 15-Jun-04     (csabet) Created
00466 ******************************************************************************/
00467 void createNoisePattern (
00468     int            fileNumber,        // In: File number
00469     char        *inFile,        // In: Name of the input FITS file
00470     ImageFormat    *format,        // In: Pointer to the image format
00471     DetRonNoise    *noiseProfile,    // Ou: Pointer to the detector noise data structure
00472     int            *error)            // Ou: Error status
00473 
00474 {
00475   
00476     //    Local Declarations
00477     //    ------------------
00478     const char  routine[] = "createNoisePattern";
00479     char        *title, *fileName;
00480     int         subWindowSize;
00481     cpl_image    *cplImage;
00482     
00483     
00484     //    Algorithm
00485     //    ---------
00486     if (diagnostic > 4)cpl_msg_info(cpl_func,"Invoking      routine   '%s' \n", routine);
00487     if (diagnostic > 4) fprintf (midiReportPtr, "Invoking      routine   '%s' \n", routine);
00488     
00489     //    Reset status
00490     *error = 0;
00491     subWindowSize = format->iXWidth * format->iYWidth;
00492     
00493     //    Create an image FITS file
00494     title = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
00495     fileName = (char *) calloc (MAX_STRING_LENGTH, sizeof (char));
00496     sprintf (title, "AveImg");
00497     sprintf (fileName, "file %d", fileNumber);
00498     createFitsImage (fileName, title, inFile, format->iXWidth, format->iYWidth, noiseProfile->aveImage);
00499     free (fileName);
00500     free (title);
00501 
00502     //    Compute the median of the noise pattern
00503     cplImage = cpl_image_wrap_float (format->iXWidth, format->iYWidth, noiseProfile->pattern);
00504     noiseProfile->median = cpl_image_get_median (cplImage);
00505     cpl_image_unwrap (cplImage);
00506 
00507     if (diagnostic)cpl_msg_info(cpl_func,"   Median of the noise pattern  = %8.4f\n", noiseProfile->median);
00508     fprintf (midiReportPtr, "   Median of the noise pattern  = %8.4f\n", noiseProfile->median);
00509 
00510     //    Create plot file
00511     if (plotFile) midiCreatePlotFile3D ("3dSigmaMap", "Sigma Map", "X", "Y", "Sigma", 
00512         0, noiseProfile->pattern, format->iXWidth, format->iYWidth, "lines", "0");
00513 
00514     return; 
00515 }
00516 /*****************************************************************************/
00517 

Generated on 5 Mar 2013 for MIDI Pipeline Reference Manual by  doxygen 1.6.1