pro IRIS_PREP, iindex, idata, oindex, odata, $ verbose = verbose, loud = loud, run_time = run_time, tdelt = tdelt, $ version = version, only_version = only_version, strict = strict, $ pstop = pstop, check = check, missing = missing, cubic = cubic, $ spikefile = spikefile, despike_here = despike_here, update_roll = update_roll, $ measure_fid = measure_fid, measure_wave = measure_wave, measure_aia = measure_aia, $ write = write, logdir = logdir, dose = dose, ccdstat = ccdstat, $ shift_fid = shift_fid, shift_wave = shift_wave, shift_aia = shift_aia, $ filter_fid = filter_fid, filter_wave = filter_wave, filter_aia = filter_aia, $ minbox = minbox, maxbox = maxbox, samebox = samebox, poly2d = poly2d, $ nowarp = nowarp, nodark = nodark, noflat = noflat, nospat = nospat, $ nobad = nobad, noback = noback, $ sat_thresh = sat_thresh, nosat = nosat, desat = desat, nsatpix = nsatpix, $ header_temps = header_temps, scaled = scaled, post_warp = post_warp, $ nrt_pipeline = nrt_pipeline, _nrt_pipeline = _nrt_pipeline, pipeline = pipeline, $ dont_filter_faint_fids = dont_filter_faint_fids, do_filter_faint_fids = do_filter_faint_fids, $ ptest = ptest, kludge_fuv = kludge_fuv, noversion = noversion, $ keyword_only = keyword_only, masks = masks, obsfile = obsfile, $ fitchisq_max = fitchisq_max, _extra=_extra ;+ ; ; NAME: ; IRIS_PREP ; ; PURPOSE: ; Convert IRIS level 1 images to level 1.5, including: ; - Dark/pedestal removal and flat-fielding ; - Replacement of spikes/bad pixels ; - Geometric correction (rotation, translation, distortion, platescale) ; - Wavelength correction (dispersion/shift) ; - Update header keywords to reflect the above actions ; ; CATEGORY: ; Image alignment ; ; SAMPLE CALLS: ; Inputing file list: ; IDL> IRIS_PREP, infiles, [0,1,2], oindex, odata ; Inputing iindex and idata: ; IDL> IRIS_PREP, iindex, idata, oindex, odata ; ; INPUTS: ; There are 2 basic usages for inputing the image and header data into IRIS_PREP: ; = Case 1: Already read in the FITS files = ; iindex - Array of structures containing FITS headers ; idata - Array containing FITS image data ; = Case 2: Just have the file names = ; iindex - Array of strings giving path/filename for FITS files to read ; idata - integer array of indices to specify which filenames in iindex should ; be read (defaults to all) ; ; OUTPUTS (OPTIONAL): ; oindex - The updated index structure of the input image(s) ; odata - Prepped output image(s). ; ; KEYWORDS: ; /pstop - This program brakes (breaks?) for debugging ; /check - Display a window showing the warped data with the transformed ROIs ; superimposed (passed through to IRIS_PREP_GEOWAVE_CORRECT) ; /strict - If set, then bail out if a non-lev1 input is given or other unexpected ; funniness is detected. Also, when bailing out, set the output ; index to the input index, and the output data to NaNs ; /run_time - Print out the time taken to complete ; tdelt - Blank variable that returns the elapsed run time (in seconds) ; nsatpix - Blank variable that returns the number of saturated pixels ; sat_thresh- Number of DN to consider a pixel saturated (default is 1.6e4) ; /verbose- Print some extra stuff ; /loud - Super verbose (passed through to IRIS_PREP_ALIGN_IMAGE) ; /maxbox - If set, then transformed ROIs are re-sized to include all valid data ; This is the default behavior; if something else is desired, set either ; /minbox or /samebox ; /minbox - If set, then transformed ROIs are re-sized to exclude any missing data. ; /samebox - If set, then transformed ROIs are mapped to the same size as the ; input ROIs (not generally a good idea since they may have moved) ; /nowarp - If set, then do not apply any warping to the image ; /nobad - If set, then do not set permanently bad pixels (dust) to 0 ; /nosat - If set, then do not set saturated pixels to Inf ; /desat - If set, then replace saturated pixels with a large finite value ; /noflat - If set, then do not apply flatfielding ; /nospat - If set, then do not apply the spatially-varying flatfield correction ; to spectra ; /nodark - If set, then do not subtract dark/pedestal ; /noback - If set, then do not subtract visible scattered light background ; from FUV spectra ; _extra - Passed through to READ_IRIS ; spikefile - Can be set as a switch (/spikefile) to look for the spike file ; in the local (LMSAL) tree, or can be set to a string giving ; the path and filename to the spikes file. A spikes file is a ; FITS file describing the spikes found during pipeline processing ; of each image. If set, then the spikes are corrected. ; despike_here - If set as a switch, then iris_prep_despike is called at ; run time to correct spikes ; missing - Can be set to a value that is used to indicate missing data. Defaults ; to NaN (so set this to 0 or something like that if you don't like NaNs) ; /scaled - if set, then the output is converted to scaled integers using ; iris_prep_float2int ; /measure_fid - Set if you want to run IRIS_PREP_FID_FINDER to find the ; location of the fiducials and write it to a file ; /measure_wave - Set if you want to run IRIS_PREP_WAVECORR to find the ; location of the Ni I NUV line and write it to a file ; /measure_aia - Set if you want to run IRIS_AIA_CORR to find the offset ; between the IRIS slitjaw image and the nearest corresponding ; AIA/HMI image (results are written to ; /write - set to allow measure_fid, measure_wave and measure_aia to write ; to external files (specified by logdir) ; logdir= - The path to an area where output logs and trending information ; should be written. ; /shift_fid - Set if you want to apply the time- and temperature-dependent ; shift to put the fiducials in their nominal positions. ; /shift_wave - Set if you want to apply the time- and temperature-dependent ; shift to put the wavelengths in their nominal positions. ; /shift_aia - Set if you want to apply the updates to the WCS information ; in the header determined by cross-correlation with SDO. IF ; /measure_aia is set, then the correction is from the real-time ; measurements of this image; otherwise, it is loaded from the ; side database ; /header_temps- Set if you want to use only the temperatures in the header (not ; currently a complete set); by default, temperatures are looked ; up at runtime using iris_time2hk ; /update_roll- Set if you want to update the header SAT_ROT, OPHASE and ; temperature values using iris_time2hk; by default, the header ; values are used (except for those temperatures not set in the ; level 1 header, which are always looked up) ; /post_warp - Set if you want to measure the fiducial and wavelength shift ; after warping; by default, they are measured before warping ; /keyword_only- Set if you want to produce only an output header, not an output ; image (much faster...) ; obsfile= - Set to the name of the genx file with all the per-image fits of fiducial ; position and wavelength are stored; if not set, then it is ; looked up from the date directories in /irisa/data/prep/ ; ; /pipeline - If set, then assuming it is running on the definitive data ; in the pipeline, and set keywords appropriately ; /nrt_pipeline - If set, then assume it is running on the NRT data in the ; pipeline, and set keywords appropriately ; /_nrt_pipeline- Synonym for nrt_pipeline ; /ptest - If set, then do some things differently (changes depending ; on what is being debugged) ; /kludge_fuv - Put in a very crude fix to correct for pedestal quadrant offsets ; by subtracting out the median of the lower half of the data ; values in each quadrant ; /noversion - If set (and keyword_only is set), then do not look up ; versions of the flat, dark, background subtraction and bad ; pixel map ; fitchisq_max - Passed through to iris_prep_align_image ; /dont_filter_faint_fids - If set, then /filter_fid is overriden for the FUV SJI. ; Note that this is set by the /pipeline keyword ; /do_filter_faint_fids - If set (along with /pipeline), then /dont_filter_faint_fids ; is overriden and the fiducial trend data is applied to all ; channels. (Don't use unless setting /pipeline) ; ; WRITTEN: ; P.Boerner 2013/06 boerner ~ at ~ lmsal ~ dot ~ com ; ; HISTORY: ; See https://www.lmsal.com/iris_science/test?cmd=view-logbook&test_id=TNCIRISCI2 ; ;- ;++++++++++++++++++++++++++++++++++++ ; 0. Set up: ; ; Define some defaults, ; ; Check inputs for consistency ; ;------------------------------------ ; Set up defaults for keywords and constants t0 = SYSTIME(/sec) run_time = KEYWORD_SET(run_time) verbose = KEYWORD_SET(verbose) loud = KEYWORD_SET(loud) strict = KEYWORD_SET(strict) nodark = KEYWORD_SET(nodark) nowarp = KEYWORD_SET(nowarp) nospat = KEYWORD_SET(nospat) header_temps = KEYWORD_SET(header_temps) poly2d = KEYWORD_SET(poly2d) if not KEYWORD_SET(pstop) then pstop = 0 ; Define version variables ;progver = 'V1.0' ; 2013-02-10 (PB) ;progver = 'V1.1' ; 2013-07-18 (PB) ;progver = 'V1.2' ; 2013-08-26 (PB) ;progver = 'V1.3' ; 2013-08-31 (PB) ;progver = 'V1.4' ; 2013-09-19 (PB) ;progver = 'V1.5' ; 2013-09-26 (PB) ;progver = 'V1.6' ; 2013-09-30 (PB) ;progver = 'V1.7' ; 2013-10-02 (PB) ;progver = 'V1.8' ; 2013-10-03 (PB) ;progver = 'V1.9' ; 2013-10-11 (PB) ;progver = 'V1.10' ; 2013-10-14 (PB) ;progver = 'V1.11' ; 2013-10-15 (PB) ;version = '1.12' ; 2013-10-16 (PB) ;version = '1.121'; 2013-10-16 (SLF) ;version = '1.13'; 2013-10-17 (PB) ;version = '1.14'; 2013-10-23 (PB) ;version = '1.15'; 2013-10-24 (PB) ;version = '1.16'; 2013-10-25 (PB) ;version = '1.17'; 2013-11-01 (PB) ;version = '1.18'; 2013-12-04 (PB) ;version = '1.19'; 2013-12-13 (PB) ;version = '1.20'; 2014-01-17 (PB) ;version = '1.21'; 2014-02-18 (PB) ;version = '1.22'; 2014-03-06 (PB) ;version = '1.23'; 2014-03-07 (PB) ;version = '1.24'; 2014-03-27 (PB) ;version = '1.25'; 2014-04-03 (PB) ;version = '1.26'; 2014-04-17 (PB) ;version = '1.27'; 2014-07-07 (PB) ;version = '1.28'; 2014-08-28 (PB) ;version = '1.29'; 2014-09-24 (PB) ;version = '1.30'; 2014-09-24 (PB) ;version = '1.31'; 2015-01-12 (PB) ;version = '1.32'; 2015-02-10 (PB) ;version = '1.33'; 2015-03-17 (PB) ;version = '1.34'; 2015-03-23 (PB) ;version = '1.35'; 2015-04-10 (PB) ;version = '1.36'; 2015-05-27 (PB) ;version = '1.37'; 2015-06-01 (PB) ;version = '1.38'; 2015-06-12 (PB) ;version = '1.39'; 2015-06-26 (PB) ;version = '1.40'; 2015-07-10 (PB) ;version = '1.41'; 2015-07-13 (PB) ;version = '1.42'; 2015-07-15 (PB) ;version = '1.43'; 2015-10-12 (SF) ; per PB, default option/kw settings for /NRT_PIPELINE ;version = '1.44'; 2015-11-04 (RPT) - marking dark and calibration updates; no code changes relative to 1.43 ;version = '1.45'; 2015-12-18 (RPT) - marking flat updates, again no code change vs 1.43/44 ;version = '1.46'; 2015-12-21 (RPT) - retroactive NUV flat updates w/better darks for Sarah's whole set. ;version = '1.47'; 2016-01-05 (RPT) - new flats 12/30/15 ;version = '1.48'; 2016-02-03 (RPT) - marking installing dark updates (iris_make_dark from 11/25/15, iris_dark_trend_fix from 1/10/16) ;version = '1.49'; 2016-02-08 (RPT, with Wei) - marking flat updates from 2016/01/27 ;version = '1.50'; 2016-03-13 (Wei Liu) - updated flats from 2016/02/24, fuv background from 2016/02/22, done by Sean Brannon (handed over by Sarah Jaeggli, 2016/02/29-03/04; Sean finished 03/11, Friday) ; version = '1.51'; 2016-04-20 (Wei Liu) - updated 20160312_220808_FUV_background, flat_NUV_SPATIAL_20160327_120040, flat_SJI_*_20160323, from Sean Brannon (done on 2016/04/18) ;version = '1.52'; 2016-05-05 (RPT) - cloned flat fields & placed with fake times adjacent to bakeouts in Oct. 2014 and Oct 2015, so that pipeline will use the appropriate pre/post bakeout flat ;version = '1.53'; 2016-05-18 (RPT cleaning up after Wei) - new iris_dark_trend_fix from S. Saar, now with proper name - progver = 'v2016.May.13' ;--- (SSaar) V6 update of double sine model + quad ; trend, P2=P1/2, data thru 05/16 ;version = '1.54'; 2016-06-10 (Wei Liu) - marking SJI flat updates from 2016/04/20 and 2016/05/06, as well as FUV background from 2016/04/10 and 2016/05/08; no sg flats yet, wait for Sean B. updates ;version = '1.55'; 2016-07-01 (Wei Liu) - marking updates: FUV background - 20160605; NUV SG Spatial flats - 20160327, 20160420, 20160507, 20160616; SJI flat - 20160615, after Sean B.'s recent updates. ;version = '1.56';2016-07-06 (RPT) - Marking flats (both SJI and SG) 20160518-0519. Also covers pre/post bakeout cloning of 20160427 (copying the before/after flats to 0427_000000 and 0428_000000) ;version = '1.57'; 2016-08-19 (Wei Liu) - marking updates from Sean: FUV background - 20160703; NUV SG Spatial flats - 20160715; SJI flat - 20160715-20160716 ;version = '1.58'; 2016-08-24 (Wei Liu) - marking updates from Sean: FUV background - 20160801; NUV SG Spatial flats - 20160810; SJI flat - 20160810 ;version = '1.59'; 2016-09-08 (Wei Liu) - upaded FUV background - 201608027 from Sean ;version = '1.60'; 2016-09-13 (Wei Liu) - upaded NUV SG Spatial and SJI flats - 20160907 from Sean ;version = '1.61'; 2016-9-14 (RPT) - converted the bad pixel maps to .geny (savegenx/restgenx) methods because they were large enough to crash ultra/solserv ; redid the badpixel maps from ; yesterday's flat up date in ; new format. Should not be any real ; difference in data but marking regardless ;version = '1.62'; 2016-10-07 (Wei Liu) - new iris_dark_trend_fix from Steve Saar, progver = 'v2016.Oct.07' ;--- (SSaar) V7 update of 2 sine model + shifted quad trend, P2=P1/2, data thru 09/16 ;version = '1.63'; 2016-10-13 (Wei Liu) - marking updates from Sean: FUV background - 20160926; NUV SG Spatial flats - 20161005; SJI flat - 20161005 ;version = '1.64'; 2016-10-28 (JP Wuesler) - new iris_prep_align_image has the real change, correct bug in fiducial shifting, now decides good fit/applies shift separately in X and Y axes ;version = '1.65'; 2016-11-16 (Wei Liu) - marking updates from Sean Brannon: FUV background taken on 2016-10-22; NUV SG Spatial flat and SJI flats taken on 2016-11-02 ;version = '1.66'; 2016-11-18 (Wei Liu) - new iris_dark_trend_fix from Steve Saar: progver = 'v2016.Nov.14' ;--- (SSaar) V8 same as V7, fixed indexing bug ;version = '1.67'; 2016-12-09 (Wei Liu) - marking updates from Sean Brannon: (no FUV background due to eclipse season); NUV SG Spatial flat from 2016-12-01 and SJI flats from 2016-12-02 ;version = '1.68'; 2016-12-28 (Wei Liu) - new iris_dark_trend_fix from Steve Saar: progver = 'v2016.Dec.28' ;--- (SSaar) V9 update (NUV only) of double sine model, +linear trend, P2=P1/2, data thru 12/16 ;version = '1.69'; 2017-01-18 (Wei Liu) - marking updates from Sean Brannon: (no FUV background due to eclipse season); NUV SG Spatial flat from 2016-12-28 and SJI flats from 2016-12-29 ;version = '1.70'; 2017-01-30 (Ryan Timmons) - change to iris_time2hk for temperatures, using the /irisa/data/prep/aux/temp_interpolated folder. ;version = '1.71'; 2017-02-10 (Wei Liu) - marking updates from Sean Brannon: (no FUV background due to eclipse season); NUV SG Spatial flat from 2017-01-27 and SJI flats from 2016-01-26 ;--- Notes by Wei Liu 2017-03-13: restored the original version 1.71 comments above, disregard the following two lines of comments about the test version 1.71 which was never released to SSW -------- ;version = '1.71' ; 2017-01-31 (Wei Liu) - special version for testing Martin W.'s double precision fits header code. ;EDIT 1.71 became a real version for Jan 26-27 2017 flatfields sooo ;version = '1.72' ; ; 2017-02-21 (Ryan Timmons) official version of Martin's double precision kewyords/header + updated FUV1/2 wavelength offset, paths chagned: ;/archive/ssw/iris/idl/uio/iris_simulator/irissim_constants__define.pro ;/archive/ssw/site/idl/iris/irisl12_getkeywords.pro ;/archive/ssw/iris/idl/lmsal/calibration/iris_mk_pointdb.pro ;/archive/ssw/iris/idl/uio/level1to2/iris_level1to2.pro (also just a ;version/date stamping) ;version = '1.73'; 2017-02-24 (Ryan Timmons) - doh, had onlining issues with the two uio paths in 1.72. This just marks correcting those. ;version = '1.74'; 2017-03-06 (Ryan Timmons) - still more issues, finally solved with Viggo/Sam/Martin..so this is really the official online of double precision header/keyword splus the FUV1/2 offset ;version = '1.75'; 2017-03-13 (Wei Liu) - marking updates from Sean Brannon: FUV background taken on 2017-02-26; NUV SG Spatial flat and SJI flats taken on 2017-02-24/25 ;version = '1.76'; 2017-03-28 (Wei Liu) - marking updates from Sean Brannon: NUV SG Spatial flat and SJI flats taken on 2017-03-22 ;version = '1.77'; 2017-04-10 (Wei Liu) - new iris_dark_trend_fix from Steve Saar: progver = 'v2017.Apr.07' ;--- (SSaar) V10 update of double sine model +quad trend, P2=P1/2, data thru 03/17 (also typo fix from "+linear" to "+quad" in progver = 'v2016.Dec.28' comment line) ;version = '1.78'; 2017-04-18 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2017-04-09, NUV SG Spatial flat and SJI flats from 2017-04-05; also automated the process for ingesting these updates (revised my IDL routines). ;version = '1.79'; 2016-08-29 (J-P Wuelser) - fixed bug causing filter_* to be ignored when called with filename,indices instead of index,data, if indices had single element ; 2017-02-28 (J-P Wuelser) - changing lvl_num of input in 2nd call to iris_prep_align image ; 2017-04-20 (J-P Wuelser) - also with v.1.79: updated iris_mk_pointdb.pro to v.10 & upgrade of iris_isp2wcs.pro ; 2017-04-24 (J-P Wuelser) - also with v.1.79: tweaked iris_prep_wavecorr.pro & iris_prep_filter_wave.pro ;version = '1.80' ; 2017-05-10 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2017-05-07, NUV SG Spatial flat and SJI flats from 2017-05-03 ;version = '1.81'; 2017-05-19 (Ryan Timmons) - marking updates from Sean Brannon: FUV background from 2017-05-07 (redone due to data recovery from a bad pass after the first one). Also contains JP's fix for the dose logging although that should not directly affect pipeline results. ;version = '1.82' ; 2017-06-09 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2017-06-04, NUV SG Spatial flat and SJI flats from 2017-05-31. ;version = '1.83' ; 2017-06-12 (Wei Liu) - new iris_dark_trend_fix from Steve Saar: progver = 'v2017.Jun.06' ;--- (SSaar,JPrchlik) V11 update of double sine model +quad trend, P2=P1/2, data thru 05/17 ;version = '1.84' ; 2017-07-07 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2017-07-02, NUV SG Spatial flat and SJI flats from 2017-06-28/29. ;version = '1.85' ; 2017-08-07 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2017-07-30, NUV SG Spatial flat and SJI flats from 2017-07-26/27. ;version = '1.86' ; 2017-09-01 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2017-08-27, NUV SG Spatial flat and SJI flats from 2017-08-24. ;version = '1.87' ; 2017-10-02 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2017-09-25, NUV SG Spatial flat and SJI flats from 2017-09-20. ;version = '1.88' ; 2017-10-17 (Wei Liu) - new iris_dark_trend_fix from Steve Saar: progver = 'v2017.Oct.16' ;--- (SSaar,JPrchlik) V12 update of double sine model +quad trend, P2=P1/2, data thru 10/17 ;version = '1.89' ; 2017-10-25 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2017-10-20, NUV SG Spatial flat and SJI flats from 2017-10-18. ;version = '1.90' ; 2017-11-21 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c of eclipse season, NUV SG Spatial flat and SJI flats from 2017-11-15/16. ;version = '1.91' ; 2017-12-14 (Wei Liu) - new iris_dark_trend_fix from Steve Saar: progver = 'v2017.Dec.14' ;--- (SSaar,JPrchlik) V13 update of double sine model +quad trend (now with stop time for FUV), P2=P1/2, data thru 11/17 ; Jakub/Steve recommended reprocessing data after 2017-Aug-9 ;version = '1.92' ; 2017-12-18 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c of eclipse season, NUV SG Spatial flat and SJI flats from 2017-12-13/14. ;version = '1.93' ; 2018-01-19 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c of eclipse season, NUV SG Spatial flat and SJI flats from 2018-01-10. ;version = '1.94' ; 2018-02-06 (Wei Liu) - new iris_dark_trend_fix from Steve Saar: progver = 'v2018.Feb.02' ;--- (SSaar,JPrchlik) V14 update of double sine model +quad trend (now with stop time), P2=P1/2, data thru 01/18; recommended data reprocessing starting Nov. 1, 2017 ;version = '1.95' ; 2018-02-15 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c of eclipse season, NUV SG Spatial flat and SJI flats from 2018-02-07/08. ;version = '1.96' ; 2018-03-14 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2018/03/10, NUV SG Spatial flat and SJI flats from 2018/03/07. ;version = '1.97' ; 2018-04-12 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2018/04/08, NUV SG Spatial flat and SJI flats from 2018/04/04-05. ;version = '1.98' ; 2018-05-10 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2018/05/06, NUV SG Spatial flat and SJI flats from 2018/05/02. ;version = '1.99' ; 2018-06-07 (Wei Liu) - (1) updates from Sean Brannon: FUV background from 2018/06/03, NUV SG Spatial flat and SJI flats from 2018/05/30-31. ; (2) new iris_dark_trend_fix from Steve Saar: progver = 'v2018.May.29': ;--- (SSaar,JPrchlik) V15 update of double sine model +quad trend, P2=P1/2, data thru 05/18; linear+quad trend now reduced after 8/17 ;version = '2.00' ; 2018-07-05 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2018/07/01, NUV SG Spatial flat and SJI flats from 2018/06/27. ; also made duplicates of the 2018-06-07 update to have time-stamps of 20180613_170000, and of today's update to have timestamps of 20180615_000000, right before and after the bakeout ; to make sure the pipleline grab the calibration files from the same side of the bakeout nearest in time for the data to be processed. ;version = '2.01' ; 2018-08-03 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2018/07/29, NUV SG Spatial flat and SJI flats from 2018/07/25-26. ;version = '2.02' ; 2018-08-31 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2018/08/24, NUV SG Spatial flat and SJI flats from 2018/08/22. ;version = '2.03' ; 2018-09-26 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2018/09/21, NUV SG Spatial flat and SJI flats from 2018/09/19. ;version = '2.04' ; 2018-10-18 (Wei Liu) - new iris_dark_trend_fix from Jakub Prchlik/Steve Saar: progver = 'v2018.Oct.17' ;--- (SSaar,JPrchlik) V16 update of double sine model+quad trend, P2=P1/2, data thru 05/18, linear+quad trend now reduced after 8/17, fractional drop in offset and increase in quadratic term following the 6/18 bakeout; ; recommended data reprocessing since 2018-Jun-15, after the 2018-Jun-13 to 14 CCD bakeout. ;version = '2.05' ; 2018-10-29 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2018/10/24, NUV SG Spatial flat and SJI flats from 2018/10/16. ;version = '2.06' ; 2018-12-21 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c of eclipse season, NUV SG Spatial flat and SJI flats from 2018/12/15-16. ;version = '2.07' ; 2019-01-10 (Wei Liu) - new iris_dark_trend_fix from Jakub Prchlik/Steve Saar, progver = 'v2019.Jan.10' ;--- (SSaar,JPrchlik) V17 update of double sine model.... ;version = '2.08' ; 2019-01-16 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c of eclipse season, NUV SG Spatial flat and SJI flats from 2019/01/09. ;version = '2.09' ; 2019-01-23 (Wei Liu) - new iris_dark_trend_fix from Jakub Prchlik/Steve Saar, improved temporary fix: progver = 'v2019.Jan.22' ;--- (SSaar,JPrchlik) V18 update of double sine model... ;version = '2.10' ; 2019-02-20 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c of eclipse season, NUV SG Spatial flat and SJI flats from 2019/02/06. ;version = '2.11' ; 2019-03-15 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2019/02/27, NUV SG Spatial flat and SJI flats from 2019/03/06-07. ;version = '2.12' ; 2019-04-09 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2019/03/31, NUV SG Spatial flat and SJI flats from 2019/04/03. ;version = '2.13' ; 2019-04-30 (Wei Liu) - new iris_dark_trend_fix from SAO: progver = 'v2019.Apr.26' ;--- (VPolito,JPrchlik,SSaar) V19 ;version = '2.14' ; 2019-05-10 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2019/04/28, NUV SG Spatial flat and SJI flats from 2019/05/01-02. ;version = '2.15' ; 2019-06-06 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2019/05/26, NUV SG Spatial flat and SJI flats from 2019/05/29. ;version = '2.16' ; 2019-07-03 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2019/06/23, NUV SG Spatial flat from 2019/06/29 and SJI flats from 2019/06/26. ;version = '2.17' ; 2019-07-29 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2019/07/20, NUV SG Spatial flat and SJI flats from 2019/07/24. ;version = '2.18' ; 2019-08-30 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2019/08/17, NUV SG Spatial flat and SJI flats from 2019/08/21. ;version = '2.19' ; 2019-09-24 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2019/09/15, NUV SG Spatial flat and SJI flats from 2019/09/17. ;version = '2.20' ; 2019-10-27 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2019/10/14, NUV SG Spatial flat and SJI flats from 2019/10/16-17. ;version = '2.21' ; 2019-11-12 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: progver = 'v2019.Oct.30' ;--- (Guliano) V20 (FUV1,2,3, 4 and NUV 2, 3, 4): refit all model parameters. ;version = '2.22' ; 2019-11-19 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c eclipse season, NUV SG Spatial flat and SJI flats from 2019/11/13. ;version = '2.23' ; 2019-12-17 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c eclipse season, NUV SG Spatial flat and SJI flats from 2019/12/11. ;version = '2.24' ; 2020-01-19 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c eclipse season, NUV SG Spatial flat and SJI flats from 2020/01/08-09. ;version = '2.25' ; 2020-02-04 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: progver = 'v2020.Jan.22' ;--- (Guliano) V21 (FUV 2 and 4, NUV3) ;version = '2.26' ; 2020-02-17 (Wei Liu) - marking updates from Sean Brannon: no FUV background b/c eclipse season, NUV SG Spatial flat and SJI flats from 2020/02/05. ;version = '2.27' ; 2020-03-10 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2020/03/01 (1st one after the recent eclipse season), NUV SG Spatial flat and SJI flats from 2020/03/04-05. ;version = '2.28' ; 2020-04-11 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2020/03/29, NUV SG Spatial flat and SJI flats from 2020/04/01. ;version = '2.29' ; 2020-04-27 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: progver = 'v2020.April.21' ;--- (Guliano) V22 (FUV1,2,3,and 4, NUV1,3 and 4): refit all parameters for all channgels except NUV2 ;version = '2.30' ; 2020-04-28 (Ryan Timmons) - merging in edits to this file for JP Wuelser/Paul Boerner work on applying the IRIS-AIA correlation updates to the pointing keywords. Includes updates to iris_aia_corr, iris_align_aia, iris_prep_filter_aia, iris_prep_filter_obs, and irisl12_shiftaia ;version = '2.31' ; 2020-05-08 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2020/04/26, NUV SG Spatial flat and SJI flats from 2020/04/29. ;version = '2.32' ; 2020-05-13 (Paul Boerner) - Adding dont_filter_faint_fids keyword ;version = '2.33' ; 2020-06-09 (Ryan Timmons) - online iris_mk_pointdb v11 (from JP Wuelser) - significant improvements to fallback solutions for fiducial, wavelength calibrations. ;iris_prep now defaults to *not* using the fiducial fits for the FUV SJI, as they have rarely worked correctly since 2015. The fallback solution should give equal ;or superior performance for the whole mission. ;version = '2.34' ; 2020-06-10 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2020/05/25, NUV SG Spatial flat and SJI flats from 2020/05/27-28. ;version = '2.35' ; 2020-07-02 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2020/06/20, NUV SG Spatial flat and SJI flats from 2020/06/24. ;version = '2.36' ; 2020-08-02 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2020/07/19, NUV SG Spatial flat and SJI flats from 2020/07/22-23. ;version = '2.37' ; 2020-08-28 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: progver = 'v2020.August.26' ;--- (Guliano) V23 (FUV1,2,3,and 4, NUV1,3 and 4): refit Scale, OffDrop, and OffNSop parameters for all channgels except NUV2 to account for the continued over estimation from the model in these 7 channels. ;version = '2.38' ; 2020-09-07 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2020/08/15 (onlined 2020/09/01), NUV SG Spatial flat and SJI flats from 2020/08/19 (onlined today, 2020/09/07). ;version = '2.39' ; 2020-10-20 (Wei Liu) - marking combo updates from Sean Brannon: FUV background from 2020/09/13 and 10/13, NUV SG Spatial flat and SJI flats from 2020/09/16 and 10/14 ;version = '2.40' ; 2020-11-18 (Wei Liu) - marking updates from Sean Brannon: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2020/11/12 ;version = '2.41' ; 2020-12-15 (Wei Liu) - marking updates from Sean Brannon: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2020/12/09-10 ;version = '2.42' ; 2021-01-11 (Wei Liu) - marking updates from Sean Brannon: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2021/01/06-07 ;version = '2.43' ; 2021-01-12 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: progver = 'v2021.January.12' ;--- (Guliano) V24 (FUV1,2,4, and NUV3): refit Amp1, Amp2, and Scale for 4 channels to account for the drop due to eclipse season ;version = '2.44' ; 2021-02-16 (Wei Liu) - marking updates from Sean Brannon: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2021/02/03 ;version = '2.45' ; 2021-03-09 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2021/02/28, NUV SG Spatial flat and SJI flats from 2021/03/03. ;version = '2.46' ; 2021-04-26 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2021/03/27, NUV SG Spatial flat and SJI flats from 2021/03/31. ;version = '2.47' ; 2021-04-27 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: 'v2021.Apr.27' ;--- (Guliano) V25 (FUV1,2,3,and 4, NUV1,3 and 4) ;version = '2.48' ; 2021-05-12 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2021/04/25, NUV SG Spatial flat and SJI flats from 2021/04/28. ;version = '2.49' ; 2021-06-04 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2021/05/23, NUV SG Spatial flat and SJI flats from 2021/05/26. ;version = '2.50' ; 2021-07-15 (Wei Liu) - 1) marking updates from Sean Brannon: FUV background from 2021/06/20, NUV SG Spatial flat and SJI flats from 2021/06/23-24. ; 2) new iris_dark_trend_fix.pro from SAO: 'v2021.Jul.14' ;--- (Guliano) V26 (All Ports) ;version = '2.51' ; 2021-07-30 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2021/07/18, NUV SG Spatial flat and SJI flats from 2021/07/21. ;version = '2.52' ; 2021-08-24 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2021/08/15, NUV SG Spatial flat and SJI flats from 2021/08/18-19. ;version = '2.53' ; 2021-09-21 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2021/09/12, NUV SG Spatial flat and SJI flats from 2021/09/15. ;version = '2.54' ; 2021-10-21 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2021/10/10, NUV SG Spatial flat and SJI flats from 2021/10/13-14. ;version = '2.55' ; 2021-11-12 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: 'v2021.Nov.10' ;--- (Guliano) V27: All ports were refit following continued overestimates and consistent darks. ;version = '2.56' ; 2021-11-20 (Wei Liu) - marking updates from Sean Brannon: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2021/11/10. ;version = '2.57' ; 2021-11-22 (Ryan Timmons) - marking a fix from Martin affecting some of the new high-cadence flare observations and fixing the FOV bug. ;version = '2.58' ; 2021-12-24 (Wei Liu) - marking updates from Sean Brannon: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2021/12/08-09. ;version = '2.59' ; 2022-01-14 (Wei Liu) - marking updates from Sean Brannon: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2022/01/05 ;version = '2.60' ; 2022-01-21 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: 'v2022.January.14' ;--- (Guliano) V28 (All Ports): All ports were refit following continued overestimates. ;version = '2.61' ; 2022-02-11 (Wei Liu) - marking updates from Sean Brannon: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2022/02/02. ;version = '2.62' ; 2022-03-09 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2022/02/27 (1st one after the eclipse season), NUV SG Spatial flat and SJI flats from 2022/03/03. ;version = '2.63' ; 2022-04-11 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2022/03/26, NUV SG Spatial flat and SJI flats from 2022/03/29. ;version = '2.64' ; 2022-05-10 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2022/04/24, NUV SG Spatial flat and SJI flats from 2022/04/27. ;version = '2.65' ; 2022-06-10 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2022/05/22, NUV SG Spatial flat from 2022/05/25, and SJI flats from 2022/05/28. ;version = '2.66' ; 2022-07-27 (Wei Liu) - marking updates from Sean Brannon: FUV background from 2022/06/19, NUV SG Spatial flat from 2022/06/22, and SJI flats from 2022/06/22. ;version = '2.67' ; 2022-08-01 (Wei Liu) - 1) marking updates from Sean Brannon: FUV background from 2022/07/17, NUV SG Spatial flat from 2022/07/20, and SJI flats from 2022/07/20. ; 2) new iris_dark_trend_fix.pro from SAO: progver = 'v2022.Jul.22' ;--- (Guliano) V29 (All Ports): MAJOR CHANGE TO DARK MODEL. ;version = '2.68' ; 2022-09-14 (Wei Liu) - 1) marking updates from Sean Brannon: FUV background from 2022/08/13, NUV SG Spatial flat and SJI flats from 2022/08/17. ; 2) Note: this is the first flat update since the relocation of all calibration files from $SSW/iris/data to $SSWDB/iris/data (made on 2022/08/10 by Ryan Timmons). ;version = '2.69' ; 2022-11-12 (Wei Liu) - 1) marking updates from Sean Brannon and Chunming Zhu, FUV background from 2022/09/11 and 10/08, NUV SG Spatial flat and SJI flats from 2022/09/14 and 10/13. Delay mainly due to changes of hands (from Sean Brannon to Chunming Zhu). ;version = '2.70' ; 2023-01-25 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: progver = 'v2023.Jan.20' ;--- (Guliano) V30 (All Ports): Model C was created to cover data from 2022. ;version = '2.71' ; 2023-06-02 (Wei Liu) - 1) marking updates from Chunming Zhu @MSU: FUV background from 2023/02-2023/04, NUV SG Spatial flat from 2022/11-2023/04, and SJI flats from 2022/11-2023/04. ; 2) new iris_dark_trend_fix.pro from SAO: 'v2023.June.01' ;--- (Guliano) V31 (All Ports); iris data reprocessing from 2022/10/25 to date was recommended for both flat/dark updates made today. ;version = '2.72' ; 2023-06-05 (Wei Liu) - marking updates from Chunming Zhu @MSU: FUV background from 2023/05/20, NUV SG Spatial and SJI flats from 2023/05/24. ;version = '2.73' ; 2023-07-20 (Wei Liu) - marking updates from Chunming Zhu @MSU: FUV background from 2023/06/18, NUV SG Spatial and SJI flats from 2023/06/21. ;version = '2.74' ; 2023-08-14 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: progver = 'v2023.Aug.11' ;--- (Guliano) V32 (All Ports): Minor adjustments to all ports due to larger than expected decrease. ;version = '2.75' ; 2023-08-21 (Wei Liu) - marking updates from Chunming Zhu @MSU: FUV background from 2023/07/17, NUV SG Spatial and SJI flats from 2023/07/19 to 07/21. ;version = '2.76' ; 2023-08-23 (Wei Liu) - marking updates from Chunming Zhu @MSU: FUV background from 2023/08/13, NUV SG Spatial and SJI flats from 2023/08/16 to 08/18. ;version = '2.77' ; 2023-10-22 (Wei Liu) - marking updates from Chunming Zhu @MSU: FUV background from 2023/09/12 and 2023/10/09, NUV SG Spatial and SJI flats from 2023/09/13-14 and 2023/10/11. ;version = '2.78' ; 2023-12-20 (Wei Liu) - marking updates from Chunming Zhu @MSU: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2023/11/08 and 2023/12/06. ;version = '2.79' ; 2024-01-25 (Wei Liu) - marking updates from Chunming Zhu @MSU: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2024/01/04. ;version = '2.80' ; 2024-01-26 (Wei Liu) - new iris_dark_trend_fix.pro from SAO: progver = 'v2024.Jan.10' ;--- (Guliano) V33 (All Ports): Improved all fit parameters of all ports to match the downward trend of the fit. ;version = '2.81' ; 2024-02-10 (Wei Liu) - marking updates from Chunming Zhu @MSU: no FUV background (eclipse season), NUV SG Spatial flat and SJI flats from 2024/01/31. version = '2.82' ; 2024-03-13 (Wei Liu) - marking updates from Chunming Zhu @MSU: FUV background from 2024/02/25 calib run, NUV SG Spatial flat and SJI flats from 2024/02/28 calib run. if KEYWORD_SET(only_version) then RETURN if verbose then PRINT, 'Running IRIS_PREP.PRO ', version ; Set up to handle case where only header is to be processed if KEYWORD_SET(keyword_only) then begin nosat = 1 nobad = 1 nodark = 1 noflat = 1 noback = 1 pipeline = 0 nrt_pipeline = 0 kludge_fuv = 0 endif else keyword_only = 0 ; Set batches of keywords if it is being run in the near-real-time pipeline if KEYWORD_SET(nrt_pipeline) or KEYWORD_SET(_nrt_pipeline) then begin poly2d = 1 update_roll = 1 measure_aia = 0 post_warp = 0 verbose = 1 run_time = 1 shift_wave = 1 shift_fid = 1 write = 0 pipeline = 0 endif ; Set batches of keywords if it is being run in the pipeline if KEYWORD_SET(pipeline) then begin poly2d = 1 update_roll = 1 verbose = 1 run_time = 1 shift_wave = 1 shift_fid = 1 filter_fid = 1 filter_wave = 1 filter_aia = 1 dose = 1 ccdstat = 1 if KEYWORD_SET(do_filter_faint_fids) then dont_filter_faint_fids = 0 $ else dont_filter_faint_fids = 1 endif ; Ensure that /shift_X is set if /filter_X is set if KEYWORD_SET(filter_fid) then shift_fid = 1 if KEYWORD_SET(filter_wave) then shift_wave = 1 if KEYWORD_SET(filter_aia) then shift_aia = 1 ; Define paths to ancillary data if not KEYWORD_SET(logdir) then logdir = '/irisa/data/prep/' iintype = SIZE(iindex, /tname) case iintype of 'STRUCT' : begin ; Case of index/data inputs input_mode = 'index_data' iindex0 = iindex[0] n_img = N_ELEMENTS(iindex) end 'STRING' : begin ; Case of file list/index list inputs input_mode = 'file_list' file_list = iindex if N_ELEMENTS(idata) eq 0 then $ ss_infile = INDGEN(N_ELEMENTS(file_list)) $ else ss_infile = idata ss_not_exist = WHERE(FILE_EXIST(file_list[ss_infile]) ne 1, n_not_exist) if n_not_exist gt 0 then begin BOX_MESSAGE, ['IRIS_PREP: Not all files found.', 'Returning...'] RETURN endif n_img = N_ELEMENTS(ss_infile) infile_arr = iindex[ss_infile] READ_IRIS, infile_arr, iindex, idata, _extra=_extra end else : begin ; Funny input specified BOX_MESSAGE, ['IRIS_PREP: Specify either header array', $ 'or list of filenames to read. Returning...'] RETURN end endcase ; Do some basic checks on the internal consistency of the data quality = IRIS_PREP_CHECKSIZE(iindex, idata, imgpath = imgpath, $ sumspat = sumspat, sumsptrl = sumsptrl, datasize = datasize, $ keyword_only = keyword_only, masks= masks) if STRMID(imgpath, 0, 3) eq 'SJI' then isspec = 0 else isspec = 1 ; Check for mixed image types on input, or a weird image size if quality mod 64 ne 0 then begin errmsg = 'IRIS_PREP encountered a fatal error and is returning:' if (quality and 1) ne 0 then errmsg = [errmsg, 'Mixed image paths...'] if (quality and 2) ne 0 then errmsg = [errmsg, 'Mixed sumspat or NAXIS2...'] if (quality and 4) ne 0 then errmsg = [errmsg, 'Mixed sumsptrl or NAXIS1...'] if (quality and 8) ne 0 then errmsg = [errmsg, 'Unexpected image size...'] if (quality and 16) ne 0 then errmsg = [errmsg, 'Bad T_OBS (corrupt ISP)...'] if (quality and 32) ne 0 then errmsg = [errmsg, 'Unexpected win_flip...'] BOX_MESSAGE, errmsg if strict then begin oindex = iindex if (keyword_only eq 0) then odata = !values.f_nan * idata endif RETURN endif ; Check for lvl_num keyword itags = TAG_NAMES(iindex) lvlnum_tag = WHERE(itags eq 'LVL_NUM', haslvl) if haslvl then if iindex[0].lvl_num ne 1 then begin BOX_MESSAGE, ['IRIS_PREP: Non-lev1 data input', 'I hope you know what youre doing...'] if strict then begin oindex = iindex if (keyword_only eq 0) then odata = !values.f_nan * idata RETURN endif endif ; Turn off application of fiducial fitter on FUV SJIs if /dont_filter_faint_fids is set if KEYWORD_SET(filter_fid) and $ KEYWORD_SET(dont_filter_faint_fids) and $ (STRMID(imgpath, 0, 5) eq 'SJI_1' ) then begin BOX_MESSAGE, ['IRIS_PREP: /dont_filter_faint_fids is set', 'So we wont...'] filter_fid = 0 endif if KEYWORD_SET(pstop) then STOP ;++++++++++++++++++++++++++++++++++++++++ ; 1. Generate output variables; ; ; flag missing and saturated pixels ; ;---------------------------------------- oindex = IRIS_PREP_ADD_HEADER(iindex) oindex.iprpver = FLOAT(version) if (keyword_only eq 0) then begin odata = FLOAT(idata) missvals = WHERE(idata eq -32768, num_miss) if num_miss gt 0 then odata[missvals] = !values.f_nan endif else begin odata = FLTARR(1,1,N_ELEMENTS(iindex)) if not KEYWORD_SET(noversion) then oindex = IRIS_PREP_GET_VERSIONS(oindex) endelse dsize = SIZE(odata) ; Weird win_flip not necessarily fatal for SJIs... if ( (quality and 64) ne 0) then begin ; Check for special case of full-frame FUV SJIs full_sji_crs = [26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 405, 408, 413, 419, 422] wherefull = WHERE(oindex[0].iicrsid eq full_sji_crs, isfull) if (STRMID(imgpath, 0, 5) eq 'SJI_1') and (isfull gt 0) then begin if (keyword_only eq 0) then for i = 0, n_img - 1 do $ odata[*,*,i] = ROTATE(TEMPORARY(odata[*,*,i]), 5) oindex.win_flip = 2 quality = quality - 64 masks.flip_mask = masks.flip_mask * 0 masks.anybit = (masks.imgpath_mask + masks.sumspat_mask + $ masks.sumsptrl_mask + masks.size_mask + masks.badisp_mask) gt 0 UPDATE_HISTORY, oindex, 'Re-flipped full-frame FUV SJI' BOX_MESSAGE, ['IRIS_PREP: Full-frame FUV SJI', 'Re-flipping and carrying on...'] endif else begin BOX_MESSAGE, ['IRIS_PREP: Unexpected win_flip', 'Proceeding with processing...'] endelse endif if not KEYWORD_SET(nosat) then begin if N_ELEMENTS(sat_thresh) eq 0 then sat_thresh = 1.6e4 satpix = WHERE(odata ge sat_thresh, nsatpix) if nsatpix gt 0 then begin odata[satpix] = !values.f_infinity UPDATE_HISTORY, oindex, 'Set ' + STRTRIM(nsatpix,2) + ' saturated pixels to Inf' endif endif ; Look up temperatures for the interval spanning all observations ; and 1 hour previous, and populate the temperature keywords in the header tsecs = ANYTIM(oindex.t_obs) startt = ANYTIM(MIN(tsecs)-4000, /ccsds) endt = ANYTIM(MAX(tsecs)+1000, /ccsds) if (not header_temps) and ( (not nodark) or (not nowarp) ) then begin ; Load and cache temperature defaults common IRIS_PREP_DEFAULT_TEMPS, default_temps if N_ELEMENTS(default_temps) eq 0 then begin sswdata = CONCAT_DIR('$SSW_IRIS','data') CD, sswdata, current = old_dir RESTGEN, file = 'iris_default_temp.genx', str = default_temps CD, old_dir endif ; Load and cache relevant temperatures from housekeeping common IRIS_PREP_HK_TEMPS, alltemps temps_cached = 0 if N_ELEMENTS(alltemps) gt 0 then begin tcache_tlim = LIMITS(ANYTIM(alltemps.date_obs)) if (tcache_tlim[0] le (startt + 400)) and (tcache_tlim[1] ge (endt - 900)) then temps_cached = 1 endif if temps_cached eq 0 then alltemps = IRIS_TIME2HK(startt, endt, /temps) temptags = N_TAGS(alltemps) if temptags eq 0 then begin ; If there was no temperature file, just make a flat one if KEYWORD_SET(verbose) then PRINT, 'IRIS_PREP: No temperatures found!' time_tab = [startt, endt] alltemps = REPLICATE(default_temps, 2) endif else begin time_tab = alltemps.date_obs endelse temptagnames = TAG_NAMES(alltemps) deftagnames = TAG_NAMES(default_temps) ttag_list = ['IT15SPPY', 'IT17SPNY', 'IT18SPPZ', 'IT19SPNZ', 'IT01PMRF', 'IT06TELM'] ; If update_roll is set, then also update the temperature keywords in the header if KEYWORD_SET(update_roll) then ttag_list = [ttag_list, 'ITF1CCD1', 'ITF2CCD2', $ 'ITNUCCD3', 'ITSJCCD4', 'BT06CBPX', 'BT07CBNX', 'BT15IEB', 'IT08GTWM', 'IT14SPPX', 'IT16SPNX'] for i = 0, n_img -1 do begin for j = 0, N_ELEMENTS(ttag_list) - 1 do begin ttag_index = WHERE(temptagnames eq ttag_list[j]) hdr_t_index = WHERE(TAG_NAMES(oindex) eq ttag_list[j]) temp_tab = alltemps.(ttag_index) missing_temps = WHERE(temp_tab lt -9998., nmt) case 1 of nmt eq N_ELEMENTS(temp_tab) : fixval = default_temps.(WHERE(deftagnames eq ttag_list[j])) nmt eq 0 : fixval = -9999 else : fixval = MEAN(temp_tab[WHERE(temp_tab ge -9998.)]) endcase if nmt gt 0 then temp_tab[missing_temps] = fixval oindex[i].(hdr_t_index) = INTERPOL(temp_tab, ANYTIM(time_tab), ANYTIM(oindex[i].t_obs)) endfor endfor numtime = N_ELEMENTS(time_tab) endif ; Look up roll and orbital phase for the interval spanning all observations and ; update the headers if KEYWORD_SET(update_roll) then begin common IRIS_PREP_HK_POINT, newpoint point_cached = 0 if N_ELEMENTS(newpoint) gt 0 then begin pcache_tlim = LIMITS(ANYTIM(newpoint.date_obs)) if (pcache_tlim[0] le (startt + 400)) and (pcache_tlim[1] ge (endt - 900)) then point_cached = 1 endif if point_cached eq 0 then newpoint = IRIS_TIME2HK(startt, endt, /magnetometer) pointtags = N_TAGS(newpoint) if pointtags eq 0 then begin ; If you couldn't find the roll/phase online, just leave them alone if KEYWORD_SET(verbose) then PRINT, 'IRIS_PREP: Unable to update roll/phase' endif else begin point_time_tab = newpoint.date_obs for i = 0, n_img -1 do begin oindex[i].ophase = INTERPOL(newpoint.ophase, ANYTIM(point_time_tab), ANYTIM(oindex[i].t_obs)) oindex[i].sat_rot = INTERPOL(newpoint.a_eulerbr_z, ANYTIM(point_time_tab), ANYTIM(oindex[i].t_obs)) endfor endelse endif ;+++++++++++++++++++++++++++++++++++ ; 2. Apply dark/flatfield correction ; ;----------------------------------- ; Update a running library of text files giving statistics on the CCD overscan/BLS/etc. if KEYWORD_SET(ccdstat) then IRIS_PREP_CCDSTAT_LOGGER, oindex, odata, logdir = logdir ; Apply dark and note it in the header if not nodark then begin dmsgs = STRARR(n_img) if header_temps then for i = 0, n_img - 1 do begin ; Just rely on whatever iris_make_dark does with what's in the headers odata[*,*,i] = IRIS_PREP_DARK(TEMPORARY(odata[*,*,i]), oindex[i], dmsg = dmsg) dmsgs[i] = dmsg endfor else begin ; Use the previously-looked-up temperature values temp_tab = FLTARR(6, numtime) temp_tab[0,*] = alltemps.ITF1CCD1 temp_tab[1,*] = alltemps.ITF2CCD2 temp_tab[2,*] = alltemps.ITNUCCD3 temp_tab[3,*] = alltemps.ITSJCCD4 temp_tab[4,*] = alltemps.BT06CBPX temp_tab[5,*] = alltemps.BT07CBNX if KEYWORD_SET(pstop) then STOP for i = 0, n_img - 1 do begin odata[*,*,i] = IRIS_PREP_DARK(TEMPORARY(odata[*,*,i]), oindex[i], $ time_tab = time_tab, temp_tab = temp_tab, dmsg = dmsg, $ tempstring = tempstring, dversion = dversion) oindex[i].iprpdtmp = tempstring oindex[i].iprpdver = dversion dmsgs[i] = dmsg endfor endelse UPDATE_HISTORY, oindex, dmsgs, /mode endif ; Subtract out the FUV background if not KEYWORD_SET(noback) and imgpath eq 'FUV' then begin for i = 0, n_img - 1 do begin thisback = IRIS_PREP_GET_BACK(oindex[i], fbver) if N_ELEMENTS(thisback) gt 1 then odata[*,*,i] = odata[*,*,i] - thisback endfor oindex.iprpbver = fbver[0] UPDATE_HISTORY, oindex, 'FUV background subtracted' endif ; Apply a kludgy fix for the FUV pedestal variation if KEYWORD_SET(kludge_fuv) and imgpath eq 'FUV' then begin submeds = FLTARR(2,2) for i = 0, 1 do for j = 0, 1 do begin xoff = i*dsize[1]/2 xrange = dsize[1]/2-1 yoff = j*dsize[2]/2 yrange = dsize[2]/2-1 subimg = odata[xoff:xoff+xrange,yoff:yoff+yrange,*] submeds[i,j] = MEDIAN(subimg[WHERE(subimg lt MEDIAN(subimg))]) - 0.5 odata[xoff:xoff+xrange,yoff:yoff+yrange,*] = subimg - submeds[i,j] endfor UPDATE_HISTORY, oindex, 'Kludged dark: ' + STRING(REFORM(submeds,4), form = '(4f8.2)') endif ; Update a running library of FITS files indicating how many photons have hit each pixel if KEYWORD_SET(dose) then IRIS_PREP_DOSE_LOGGER, oindex, odata, logdir = logdir ; Apply flatfield and note it in the header if not KEYWORD_SET(noflat) then begin fmsgs = STRARR(n_img) for i = 0, n_img - 1 do begin odata[*,*,i] = IRIS_PREP_FLAT(TEMPORARY(odata[*,*,i]), oindex[i], $ frecnum = frecnum, fmsg = fmsg, nospat = nospat) oindex[i].iprpfver = frecnum fmsgs[i] = fmsg endfor UPDATE_HISTORY, oindex, fmsgs, /mode endif if KEYWORD_SET(pstop) then STOP if run_time and verbose then $ PRINT, 'IRIS_PREP: Flat/dark...', SYSTIME(/sec) - t0, form = '(a30,f10.2)' ;+++++++++++++++++++++++++++++++++++ ; 3. Set permanently bad pixels to ; zero ;----------------------------------- if not KEYWORD_SET(nobad) then begin if imgpath eq 'FUV' then fsize = 4144l else fsize = 2072l badpix = IRIS_PREP_GET_BADPIX(oindex[0], badpix_recnum = badpix_recnum) bpx = badpix mod fsize bpy = badpix / fsize bpmask = FLTARR(dsize[1], dsize[2]) + 1. bpmask[bpx/sumsptrl, bpy/sumspat] = 0. for i = 0, n_img - 1 do begin odata[*,*,i] = odata[*,*,i] * bpmask endfor oindex.iprppver = badpix_recnum UPDATE_HISTORY, oindex, 'Set permanently bad pixels to 0 prior to warping' endif ;+++++++++++++++++++++++++++++++++++ ; 4. Despike (optional, and not ; generally recommended) ;----------------------------------- case 1 of ; Run iris_prep_despike in real time if despike_here is set... KEYWORD_SET(despike_here) : begin if N_ELEMENTS(spikefile) gt 0 then begin MESSAGE, 'Keyword /despike_here means that spikefile will be ignored...' endif for i = 0, n_img - 1 do begin odata[*,*,i] = IRIS_PREP_DESPIKE(TEMPORARY(odata[*,*,i])) ; dummy = IRIS_PREP_DESPIKE(TEMPORARY(odata[*,*,i]), goodmap = goodmap) ; tt = oindex[i].t_obs ; this_spikefile = '~/Desktop/scratch/ispike/iris_' + TIME2FILE(tt, /sec) + $ ; STRMID(tt,20,2) + '_' + STRLOWCASE(STRMID(oindex[i].img_path, 0, 3)) + $ ; '_spikes.fits' ; WRITEFITS, this_spikefile, goodmap endfor UPDATE_HISTORY, oindex, 'ran IRIS_PREP_DESPIKE' if run_time and verbose then $ PRINT, 'IRIS_PREP: Despiked...', SYSTIME(/sec) - t0, form = '(a30,f10.2)' end ; Use a previously-run set of spikes if spikefile is set N_ELEMENTS(spikefile) gt 0 : begin ; If set as a switch, find spikefiles for all images if spikefile[0] eq '1' then spikefile = STRARR(n_img) ; Repair the spikes in the spikefile for i = 0, n_img - 1 do begin if spikefile[i] eq '' then begin tt = oindex[i].t_obs this_spikefile = '/irisa/data/spike_del/aia/lev1/' + $ STRMID(tt, 0, 4) + '/' + STRMID(tt, 5, 2) + '/' + STRMID(tt, 8,2) + $ '/H' + STRMID(tt,11,2) + '00/iris' + TIME2FILE(tt, /sec) + $ STRMID(tt,20,2) + '_' + STRLOWCASE(STRMID(oindex[i].img_path, 0, 3)) + $ 'spikes.fits' if FILE_EXIST(this_spikefile) then spikefile[i] = this_spikefile endif if spikefile[i] ne '' then begin READ_IRIS, spikefile[i], spikehead, spikedat tmpdata = odata[*,*,i] tmpdata[spikedat[*,0]] = spikedat[*,2] odata[*,*,i] = tmpdata endif endfor UPDATE_HISTORY, oindex, 'used spikefile to replace spikes' if run_time and verbose then $ PRINT, 'IRIS_PREP: Despiked...', SYSTIME(/sec) - t0, form = '(a30,f10.2)' end ; Generally, don't worry about the spikes else : endcase ;+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ; 5. Warp the image ; - Look up interpolation matrix used to straighten spectral lines (identity matrix for SJI) ; - Apply scaling, rotation and shift specified for each image type by pointing database ; - Optionally, apply additional time-varying shifts based on measurements of fiducials/neutral lines ; - Update the headers to reflect the shifts/warping ; - Optionally, update the headers to reflect correlation with SDO images ;------------------------------------------------------------------------------------------------------- if not nowarp then begin ; Select the algorithm and parameters that will be used for warping if not KEYWORD_SET(cubic) then cubic = -0.5 SETENV, 'IRIS_PREP_POLY2D='+STRTRIM(poly2d,2) ; Determine how the CRS boxes should be affected by the warping based on ; the minbox/maxbox keywords case 1 of KEYWORD_SET(maxbox) : boxwarp = 1 KEYWORD_SET(minbox) : boxwarp = 0 KEYWORD_SET(samebox): boxwarp = 2 else : boxwarp = 1 endcase ; Look up geometric/wavelength calibration parameters geowave_params = IRIS_PREP_GET_GEOWAVE(oindex[0]) if imgpath eq 'FUV' then geover = geowave_params.fuvs.rec_num $ else geover = geowave_params.rec_num pdb = IRIS_MK_POINTDB(version = pdb_ver) ; Decide whether to write the alignment results to txt files if KEYWORD_SET(write) then begin al_logdir = logdir if KEYWORD_SET(post_warp) then nowarp_logdir = 0 else nowarp_logdir = logdir endif else begin al_logdir = 0 nowarp_logdir = 0 endelse ; Shift the image to put the fiducials where they're supposed to be based on ; long-term temperature trends if KEYWORD_SET(shift_fid) then fidmsgs = STRARR(n_img) else shift_fid = 0 ; Shift the image to put the reference wavelength exactly where it's supposed to be if KEYWORD_SET(shift_wave) and (isspec eq 1) then begin do_shift_wave = 1 wavemsgs = STRARR(n_img) endif else begin do_shift_wave = 0 endelse ; Update the headers to identify the shifts needed to coalign with SDO if KEYWORD_SET(shift_aia) then aiamsgs = STRARR(n_img) else shift_aia = 0 ; If you're running on dummy data, then ignore the /filter_* and /measure_* keywords if N_ELEMENTS(idata) lt 2 then begin do_measure_wave = 0 do_measure_fid = 0 do_measure_aia = 0 do_filter_wave = 0 do_filter_fid = 0 do_filter_aia = 0 obsfit = '' do_post = 0 xfitflag = 0 endif else begin ; Look up the per-OBS fit results on the fiducials, wavelength correction, ; and AIA cross-correlation so that they can be applied if KEYWORD_SET(filter_fid) then do_filter_fid = 1 else do_filter_fid = 0 if KEYWORD_SET(filter_wave) then do_filter_wave = 1 else do_filter_wave = 0 if KEYWORD_SET(filter_aia) then do_filter_aia = 1 else do_filter_aia = 0 if KEYWORD_SET(filter_fid) or KEYWORD_SET(filter_wave) or KEYWORD_SET(filter_aia) then begin if KEYWORD_SET(obsfile) gt 0 then begin RESTGEN, file = obsfile, str = obsfit endif else begin obsfit = IRIS_PREP_OBSFIT_READER(oindex) endelse if N_TAGS(obsfit) eq 0 then begin BOX_MESSAGE, ['IRIS_PREP: Missing obsfit.genx', oindex[0].t_obs] do_filter_fid = 0 do_filter_wave = 0 do_filter_aia = 0 endif endif else begin obsfit = '' endelse if KEYWORD_SET(measure_fid) then do_measure_fid = 1 else do_measure_fid = 0 if KEYWORD_SET(measure_wave) then do_measure_wave = 1 else do_measure_wave = 0 if KEYWORD_SET(post_warp) then begin do_post = 1 if do_measure_fid + do_measure_wave gt 0 then $ BOX_MESSAGE, ['IRIS_PREP: /measure_fid and /measure_wave are not needed', $ 'if /post_warp is set, and their results are discarded.', $ 'But we will leave them set and assume you know what you are doing...'] endif else begin do_post = 0 endelse if do_shift_wave or (shift_fid and (isspec eq 0) ) then xshiftflag = 1 else xshiftflag = 0 if do_filter_wave or (do_filter_fid and (isspec eq 0) ) then xfitflag = 1 else xfitflag = 0 oindex[*].iprpofff = xshiftflag + (2 * xfitflag) + (16 * shift_fid) + (32 * do_filter_fid) ; The AIA correlation requires an environmental rather than a local variable if KEYWORD_SET(measure_aia) then begin aia_logdir = CONCAT_DIR(al_logdir, 'aiacorr') do_measure_aia = 1 endif else begin do_measure_aia = 0 endelse endelse ; For FUV, need separate S/L messages for the wavelength and fiducial finder if imgpath eq 'FUV' then begin if N_ELEMENTS(wavemsgs) ne 0 then lwavemsgs = wavemsgs if N_ELEMENTS(fidmsgs) ne 0 then lfidmsgs = fidmsgs endif ; Apply geometric/wavelength calibration, along with thermal drift fixes for i = 0, n_img - 1 do begin ; Modify the warping parameters to do shifting, rotation, scaling this_geowave_params = IRIS_PREP_ALIGN_IMAGE(oindex[i], odata[*,*,i], $ geowave_params, pdb, fidmsg = fidmsg, wmsg = wmsg, logdir = nowarp_logdir, $ measure_fid = do_measure_fid, shift_fid = shift_fid, obsfit = obsfit, loud = loud, $ filter_fid = do_filter_fid, filter_wave = do_filter_wave, $ measure_wave = do_measure_wave, shift_wave = do_shift_wave, pstop = (pstop gt 1), $ xfiltfail = xfiltfail, yfiltfail = yfiltfail, fitchisq_max = fitchisq_max) ; Apply the warping/shifting/rotation/scaling to the image and header odata[*,*,i] = IRIS_PREP_GEOWAVE_CORRECT(/update, TEMPORARY(odata[*,*,i]), $ this_geowave_params, inhead=oindex[i], outhead = nindex, $ /boxwarp, cubic = cubic, keyword_only = keyword_only, pstop = (pstop gt 1), check = check) ; If requested, do the fiducial and wavelength measurements after warping ; These measurements are not reported in the header or folded back into the image if do_post then begin nindexi = nindex if haslvl then nindexi.lvl_num = 1.5 else nindexi = ADD_TAG(nindexi, 1.5, 'LVL_NUM') dummy = IRIS_PREP_ALIGN_IMAGE(nindexi, odata[*,*,i], $ this_geowave_params, pdb, logdir = al_logdir, /measure_fid, /measure_wave, $ pstop = (pstop gt 1), /level1p5, loud = loud) endif ; Measure and/or apply shift to header WCS to align with SDO if do_measure_aia and (imgpath eq 'SJI_1330' or imgpath eq 'SJI_1400' or imgpath eq 'SJI_2832') $ then begin iris_align_str = IRIS_AIA_CORR( nindex, odata[*,*,i], do_log=KEYWORD_SET(write), $ /quiet, /short, use_lev1p5=0, pstop=pstop, logdir = aia_logdir) if shift_aia then nindex = IRIS_ALIGN_AIA(nindex, iris_align_str, msg = aiamsg) endif else if do_filter_aia then nindex = IRIS_ALIGN_AIA(nindex, obsfit, msg = aiamsg) $ else aiamsg='' ; Document shifts applied if do_shift_wave then begin wavemsgs[i] = wmsg[0] nindex.iprpoffx = FLOAT(STRMID(wmsg[0], 40, 9)) endif if shift_fid then begin fidmsgs[i] = fidmsg[0] if (isspec eq 1) then begin nindex.iprpoffy = FLOAT(STRMID(fidmsg[0], 40, 9)) endif else begin nindex.iprpoffx = FLOAT(STRMID(fidmsg[0], 40, 9)) nindex.iprpoffy = FLOAT(STRMID(fidmsg[0], 49, 9)) endelse endif if (xfitflag eq 1) and (xfiltfail eq 1) then nindex.iprpofff = nindex.iprpofff + 4 if (do_filter_fid eq 1) and (yfiltfail eq 1) then nindex.iprpofff = nindex.iprpofff + 64 if shift_aia then aiamsgs[i] = aiamsg[0] if imgpath eq 'FUV' then begin if do_shift_wave then lwavemsgs[i] = wmsg[1] if shift_fid then lfidmsgs[i] = fidmsg[1] endif ; Set up the array of the warped headers to replace the old headers if i eq 0 then begin newindex = nindex indtemplate = nindex endif else begin STRUCT_ASSIGN, nindex, indtemplate newindex = [newindex, indtemplate] endelse endfor oindex = newindex oindex.iprppdbv = pdb_ver oindex.iprpgver = geover UPDATE_HISTORY, oindex, 'Used iris_mk_pointdb ver ' + STRTRIM(pdb_ver, 2) if do_shift_wave then UPDATE_HISTORY, oindex, wavemsgs, /mode if shift_fid then UPDATE_HISTORY, oindex, fidmsgs, /mode if shift_aia then UPDATE_HISTORY, oindex, aiamsgs, /mode if imgpath eq 'FUV' then begin if do_shift_wave then UPDATE_HISTORY, oindex, lwavemsgs, /mode if shift_fid then UPDATE_HISTORY, oindex, lfidmsgs, /mode endif if poly2d eq 1 then begin UPDATE_HISTORY, oindex, 'Used INF_POLY_2D for warping' endif else begin UPDATE_HISTORY, oindex, 'Used INFTERPOLATE for warping' endelse if KEYWORD_SET(pstop) then STOP if run_time and verbose then $ PRINT, 'IRIS_PREP: Warping...', SYSTIME(/sec) - t0, form = '(a30,f10.2)' endif ;+++++++++++++++++++++++++++++++++++ ; 6. Convert to physical units ; (and replace missing pixels with ; dummy values if requested) ;----------------------------------- if KEYWORD_SET(photons) then begin odata = IRIS_PREP_CONVERT2PHOTONS(TEMPORARY(odata), inhead = oindex, outhead = nindex) oindex = nindex endif ; If the missing keyword is set to something, then poke that value into all the ; pixels marked as missing (with NaNs) if N_ELEMENTS(missing) ne 0 then begin missind = WHERE(odata ne odata, nummiss) if nummiss ne 0 then odata[missind] = missing endif ; If the desat keyword is set to something, then poke that value into all the ; pixels marked as saturated (with Infs) if N_ELEMENTS(desat) ne 0 then begin if desat eq 1 then desat = 16384. desat = desat[0] satind = WHERE(odata eq !values.f_infinity, nsatpix_1p5) if nsatpix_1p5 ne 0 then odata[satind] = desat endif if KEYWORD_SET(scaled) then IRIS_PREP_FLOAT2INT, ohdr, TEMPORARY(odata), ohdr, odata ;+++++++++++++++++++++++++++++++++++ ; 7. Update FITS header ;----------------------------------- if haslvl then oindex[*].lvl_num = 1.5 else oindex = ADD_TAG(oindex, 1.5, 'LVL_NUM') if (keyword_only eq 0) then oindex = IRIS_PREP_IMAGE_STATS(oindex, odata) UPDATE_HISTORY, oindex, caller = 'iris_prep', version = version UPDATE_HISTORY, oindex, ' ran on ' + TIME2FILE(/sec, RELTIME(/now)) oindex.keywddoc = 'http://www.lmsal.com/iris_science/irisfitskeywords.pdf' oindex.iprpver = version ;+++++++++++++++++++++++++++++++++++ ; 8. Clean up and exit ;----------------------------------- if input_mode eq 'file_list' then begin ; put the input variables back iindex = file_list idata = ss_infile endif if KEYWORD_SET(run_time) then begin tdelt = SYSTIME(/sec)-t0 BOX_MESSAGE, $ [STRING('IRIS_PREP: ', n_img, 'images prepped', format = '(a12, i5, a15)'), $ STRING('Run time: ' , tdelt, 'seconds', form = '(a12, f7.1, a8)')] endif end