; Combine particles from good classes
;
; SOURCE: spider/docs/techs/recon1/Procs/verify-combine-classes.spi
;
; PURPOSE: Combine particles from good classes
;
; USAGE: clean ; ./spider spi/dat @verify-combine-classes
;
; REQUIRES: spider/docs/techs/recon1/Procs/verify-settings.spi
;
; INPUTS: (Where *** denotes group, ### denotes view, ??? denotes class)
; [ref_view_list] [rec_dir]/sel_proj List of projection views (one)
; [prj_dir] views/prj### View sub-directories (one/view)
; [good_classes_list] views/prj###/sel_class_good Selected classes (one/view)
; [class_doc] views/prj###/sel_part_byclass_??? Particle selection by class (one/class/view)
; [view_list] views/prj###/sel_part_byv Total particle selection files (one/view)
; [byhand_list] views/prj###/sel_part_byhand OPTIONAL hand-picked particle list (one/class)
; [first_good_part_doc] views/prj###/first_good_part OPTIONAL for reference views (one/view)
;
; If writing average images:
; [params] ../params' Parameter doc file (one)
; [aligned_images] [rec_dir]/dala_01_***@ Aligned-images (one/group)
;
; OUTPUTS:
; [good_particles] views/prj###/sel_part_byv_good Good-particle list (one/view)
; [bad_particles] views/prj###/sel_part_byv_bad Sorted bad-particle list (one/view)
; [parts_vsview_good] views/parts_vsview_good Total # of good particles (one/view)
; [good_particle_copy] ../Particles/good/sel_part_*** OPTIONAL Archive copy of good particle list (one/view)
;
; ---------------------- Parameters ----------------------
[saveAvgYN] = 1 ; Flag to save unfiltered view-average (1 == save)
[archiveYN] = 0 ; Copy files to Particles/good/... for archiving? (0 == no)
; ------------------- END BATCH HEADER -------------------
MD
TR OFF ; Decrease results file output
MD
VB OFF ; Decrease results file output
; Set common filenames & parameters
@verify-settings
; Set temporary filenames
[temp_goodbyview_wdupes_unsort_noccrot] = 'tmp_goodbyview_1_combined'
[temp_goodbyview_nodupes_unsort_noccrot] = 'tmp_goodbyview_2_wodupes'
[temp_goodbyview_nodupes_unsorted_ccrot] = 'tmp_goodbyview_3_ccrot'
[temp_badbyview_unsort] = 'tmp_badbyview_1_unsort'
[temp_good_class] = 'tmp_good_class'
[temp_fullsize_stack] = '_1'
[temp_variance_incore] = '_3'
SYS
echo -n " Combining good particles "; date
IF ( [archiveYN] == 0 ) THEN
SYS
echo " Not archiving copy of selection files" ; echo
ELSE
SYS
echo " Archiving copy of output selection files" ; echo
ENDIF
; Make circular mask
IF ( [saveAvgYN] > 0 ) THEN
SYS
echo " Will write averages" ; echo
; Get window size
UD 17, [windowSize]
[params]
UD E
DE
[good_view_avg]@
ENDIF
; Prepare output-stats doc
DE
[parts_vsview_good]
SD / #KEPT GOODCLASSES PREVERIFY PERCENT
[parts_vsview_good]
; Initialize total kept-particle counter
[kept-parts] = 0
; Get number of reference-views
UD N [num-refs]
[ref_view_list]
; Loop through reference-views
DO [view-key] = 1,[num-refs]
; Get view #
UD IC [view-key],[view]
[ref_view_list]
IF ( [archiveYN] .NE. 0 ) THEN
SYS
cp -f [good_particles][view].$DATEXT [good_particle_copy][view].$DATEXT
ENDIF
; CHECK IF THERE ARE ANY PARTICLES
; Look for selection file
IQ FI [sel-exists]
[view_list][view]
; If no selection file
IF ( [sel-exists] == 0 ) THEN
SYS
echo " View: {%I4%[view]} No particles found "
[totViewParts] = 0
[goodClassParts] = 0
[goodViewParts] = 0
[fractionKept] = 0
SYS
touch [good_particles][view].$DATEXT
; Downstream procedures will look for this file
GOTO LB7
ENDIF
; Get total number of particles in current reference view
UD N [totViewParts]
[view_list][view]
; Check if good-class list exists
IQ FI [goodclasses-exists]
[good_classes_list][view] ; File (input)
IF ( [goodclasses-exists] == 1 ) THEN
; Get number of good classes
UD N [num-good-classes]
[good_classes_list][view]
ENDIF
; If good-class list doesn't exist or is empty
IF ( [num-good-classes]*[goodclasses-exists] == 0 ) THEN
SYS
echo " View: {%I0%[view]} No good classes found"
[goodClassParts] = 0
[goodViewParts] = 0
[fractionKept] = 0
SYS
touch [good_particles][view].$DATEXT
; Downstream procedures will look for this file
; Copy total-particle list to bad-particle list (to be sorted)
DOC REN
[view_list][view] ; File (input)
[temp_badbyview_unsort] ; File (output)
GOTO LB4
ENDIF
; JUGGLE PARTICLE-PICKING MODES
; Clean up pre-existing files
DE
[good_particles][view]
DE
[temp_goodbyview_wdupes_unsort_noccrot]
; Get first good-class number
UD 1, [class-num]
[good_classes_list][view] ; File (input)
UD E
; Check if by-hand class doc exists
IQ FI [byhand-exists]
[byhand_list][view]{***[class-num]} ; File (input)
; PARTICLES PICKED BY HAND
; If by-hand class docs exist then
IF ( [byhand-exists] == 1 ) THEN
SYS
echo " View: {%I0%[view]} Using hand-picked particle list"
; Make sure by-hand lists exist (will crash DOC COMBINE otherwise)
; Loop through good classes
DO [good-class-key5] = 1,[num-good-classes] ; Loop through good classes -----
; Get class number
UD IC [good-class-key5], [class-num]
[good_classes_list][view]
; Check if by-hand list exists
IQ FI [b-exists]
[byhand_list][view]{***[class-num]}
; If by-hand list doesn't exist, create empty one
IF ( [b-exists] == 0 ) THEN
SYS
echo " View: {%I4%[view]}, Class: {%I3%[class-num]} No by-hand particle-list found -- Continuing"
SYS
touch [byhand_list][view]{***[class-num]}.$DATEXT
ENDIF
ENDDO ; End class-loop ----------------------
; Combine good classes for hand-picked particles
DOC COMBINE
[byhand_list][view]*** ; File (input)
[good_classes_list][view] ; Selection file (input)
[temp_goodbyview_wdupes_unsort_noccrot] ; File (output)
; Remove duplicates & renumber
AT IT
[temp_goodbyview_wdupes_unsort_noccrot] ; File (input)
[temp_goodbyview_nodupes_unsort_noccrot] ; File (output)
; Get # particles kept
UD N [goodViewParts]
[temp_goodbyview_nodupes_unsort_noccrot] ; File (input)
; Calculate fraction kept
[fractionKept] = [goodViewParts]/[totViewParts]
; Combine good classes for total particles
DOC COMBINE
[class_doc][view] ; File template (input)
[good_classes_list][view] ; Selection file (input)
[good_particles][view]_notbyhand ; Temporary file (output)
; Renumber
DOC RENUM
[good_particles][view]_notbyhand ; Temporary file (input)
[good_particles][view]_nohandrenum ; Temporary file (output)
; Get number of particles in good classes
UD N [goodClassParts]
[good_particles][view]_nohandrenum ; Temporary (input)
; Clean up
UD ICE
[good_classes_list][view] ; File (closed)
DE
[good_particles][view]_notbyhand ; File (removed)
DE
[good_particles][view]_nohandrenum ; File (removed)
GOTO LB6
ENDIF
; End by-hand conditional
; FIRST GOOD PARTICLE PICKED FOR EACH CLASS
; Check if first good particle doc exists
IQ FI [exists]
[first_good_part_doc][view]
; If first good particle doc exists then
IF ( [exists] == 1 ) THEN
SYS
echo " View: {***[view]} Starting from first good particle"
[goodViewParts] = 0
[goodClassParts] = 0
; Loop through good classes
DO [good-class-key2] = 1,[num-good-classes]
; Get class number
UD IC [good-class-key2],[class-num]
[good_classes_list][view]
; Get number of particles in class
UD N [class-parts]
[class_doc][view][class-num] ; File (input)
; Get first good particle# from current class
UD IC [class-num],[first-good-part]
[first_good_part_doc][view] ; File (input)
; Increment good-class particle-counter
[goodClassParts] = [goodClassParts] + [class-parts]
; Initialize flag that good particles haven't started
[good-part-flag] = 0
; Loop through particles
DO [part-key] = 1,[class-parts] ; Loop through particles ----------
; Get particle#
UD IC [part-key],[partNum]
[class_doc][view][class-num] ; File (input)
; PART-NUM is probably VIEW-SLICE, but it could be unstacked particle #
; Check if first good particle hasn't been reached
IF ( [good-part-flag] == 0 ) THEN
; Check if current particle is first good one
IF ( [partNum] == [first-good-part] ) [good-part-flag] = 1
ENDIF
; Check if first good particle has been reached
IF ( [good-part-flag] == 1 ) THEN
; Increment good-particle counter
[goodViewParts] = [goodViewParts] + 1
; Write particle # to truncated-class, good-particle list
SD [goodViewParts],[partNum]
[temp_goodbyview_wdupes_unsort_noccrot] ; File (output)
ENDIF
ENDDO ; End particle-loop ---------------------
UD ICE
[class_doc][view][class-num] ; File (closed)
ENDDO ; End class-loop-------------------------
SD E ; Close doc
[temp_goodbyview_wdupes_unsort_noccrot] ; File (closed)
; Remove duplicates
AT IT
[temp_goodbyview_wdupes_unsort_noccrot] ; File (input)
[temp_goodbyview_nodupes_unsort_noccrot] ; File (output)
; End first good conditional
; WHOLE CLASSES PICKED
ELSE
; Check to see if all selected classes actually exist
DE
[temp_good_class] ; Temp selection file (removed)
[n] = 0
DO
UD NEXT [key],[classnum] ; Get class number
[good_classes_list][view] ; Selection file (input)
IF ( [key] == 0 ) EXIT ; End of list
IQ FI [c-exists] ; See if class file exists
[class_doc][view][classnum] ; File template (input)
IF ( [c-exists] > 0) THEN
[n] = [n] + 1 ; This class exists
SD [n], [classnum] ; Save class number in temp file
[temp_good_class] ; Temp selection file (output)
ENDIF
ENDDO
SD E ; Close temp file
[temp_good_class] ; Temp selection file (closed)
UD N [numlists] ; Get number of existing classes
[temp_good_class] ; Temp selection file (output)
SYS
echo " View: {%I4%[view]} Combining:{%I3%[numlists]} class-membership lists"
; Combine good classes
DOC COMBINE
[class_doc][view] ; File template (input)
[temp_good_class] ;[good_classes_list][view] ; Selection file (input)
[temp_goodbyview_wdupes_unsort_noccrot] ; File (output)
; Remove duplicates
AT IT
[temp_goodbyview_wdupes_unsort_noccrot] ; File (input)
[temp_goodbyview_nodupes_unsort_noccrot] ; File (output)
; Get number of particles kept
UD N [goodViewParts]
[temp_goodbyview_nodupes_unsort_noccrot] ; File (input)
[goodClassParts] = [goodViewParts]
ENDIF
; End whole-classes conditional
LB6 ; Jump here if picking by hand
; Clean up
UD ICE
[good_classes_list][view]
UD ICE
[first_good_part_doc][view]
DE
[temp_goodbyview_wdupes_unsort_noccrot]
; Add CCC info to selection-doc
DOC AND
[view_list][view] ; Re-keyed total particle selection file (input)
[temp_goodbyview_nodupes_unsort_noccrot] ; File (input)
[temp_goodbyview_nodupes_unsorted_ccrot] ; File (output)
1 ; Column # to find intersection
DE
[good_particles][view] ; File (removed)
SD / VIEW_WIN GLOBAL_NUM GRP_WIN CCROT MIRROR GRP_NUM VIEW
[good_particles][view] ; File (output)
SD E
[good_particles][view] ; File (closed)
; Sort good particles by CCC, Append to labeled header
DOC SORT A
[temp_goodbyview_nodupes_unsorted_ccrot]
[good_particles][view] ; File (output)
-4 ; Column # to sort: CCROT, Reverse sort order
Y ; Renumber keys
; Clean up
DE
[temp_goodbyview_nodupes_unsort_noccrot] ; File (removed)
DE
[temp_goodbyview_nodupes_unsorted_ccrot] ; File (removed)
; Copy to single directory (for archival purposes)
IF ( [archiveYN] .NE. 0 ) THEN
SYS
cp -f [good_particles][view].$DATEXT [good_particle_copy][view].$DATEXT
ENDIF
; CREATED SORTED BAD-PARTICLE LIST
; Create bad-particle list
DOC SUBTRACT
[view_list][view] ; Re-keyed total-particle selection doc (w/CCC) (input)
[good_particles][view] ; Good-particle selection doc, no CCC (input)
[temp_badbyview_unsort] ; Bad-particle selection doc, unrenumbered (output)
1 ; Column # to subtract: view-slice#
LB4 ; Jump here if no good-class list found
DE
[bad_particles][view]
; Get # of bad particles
UD N [bad-parts]
[temp_badbyview_unsort]
IF ( [bad-parts] > 0 ) THEN
; Sort bad particles by CCC, from highest to lowest
SD / VIEW_WIN GLOBAL_NUM GRP_WIN CCROT MIRROR GRP_NUM VIEW
[bad_particles][view] ; File (output)
SD E
[bad_particles][view] ; File (closed)
DOC SORT A
[temp_badbyview_unsort]
[bad_particles][view] ; File (output)
-4 ; Column# to sort: CCROT, Reverse sort
Y ; Renumber keys
ENDIF
; Clean up
DE
[temp_badbyview_unsort]
; Calculate percentage kept
IF ( [totViewParts] > 0 ) [fractionKept] = 100*[goodViewParts]/[totViewParts]
LB7 ; Jump here if no selection file
IF ( [saveAvgYN] > 0 ) THEN
IF ( [goodViewParts] > 0 ) THEN
DE
[temp_fullsize_stack]@ ; Stack (removed)
MS
[temp_fullsize_stack]@ ; Stack (output)
[windowSize],[windowSize] ; Dimensions
[goodViewParts] ; Max image allowed
; Loop through particles
DO LB62 [pkey] = 1,[goodViewParts]
; Get global#, group#, group-slice#, and mirroring
UD IC [pkey], [viewWin],[gloNum],[grpWin],[ccrot],[mir],[grp]
[good_particles][view]
; ; Read alignment parameters from combined alignment doc
; UD FIND [key]
; [combined_align_doc]
; Mirror, if necessary
IF ( [mir] < 0 ) THEN
MR
[aligned_images][grp][grpWin] ; Image (input)
[temp_fullsize_stack]@{******[pkey]} ; Mirrored image (output)
Y ; Mirror around y-axis
ELSE
CP
[aligned_images][grp][grpWin]
[temp_fullsize_stack]@{******[pkey]}
ENDIF
LB62
; End particle-loop
; Close docs
UD ICE
[good_particles][view]
;; UD FIND E
;; [combined_align_doc]
; Average images
AS R
[temp_fullsize_stack]@****** ; File (input)
1-[goodViewParts] ; Image range
A ; All particles
[good_view_avg]@{***[view]} ; Average file (output)
[temp_variance_incore] ; variance (output, not saved)
ENDIF
ENDIF
SYS
echo " View: {%I4%[view]} Found {%i0%[goodViewParts]} good particles"
; Write to stats doc
SD [view], [goodViewParts],[goodClassParts],[totViewParts],[fractionKept]
[parts_vsview_good]
; Increment total kept-particle counter
[kept-parts] = [kept-parts] + [goodViewParts]
ENDDO ; End view-loop -----------------
; Close docs
UD ICE
[ref_view_list]
DE
[temp_good_class] ; Temp selection file (removed)
[dummy] = -[num-refs] ; Dummy variable
SD / GOODPARTICLES
[parts_vsview_good] ; File (output)
SD [dummy], [kept-parts] ; File (output)
[parts_vsview_good]
SD E
[parts_vsview_good] ; File (closed)
SYS
echo ; echo -n " Done, kept: {%I0%[kept-parts]} particles "; date ; echo
EN D
; Modified 2016-10-17
; 2016-05-12 (trs) -- added option to save averages
; 2013-10-23 (agl) -- New file names, modernized syntax & cosmetic
; 2013-10-23 (agl) -- replaced: reversedoc-7col with reverse DOC SORT
; 2012-03-15 (trs) -- summary doc now has the format of HOW_MANY
; 2012-03-01 (trs) -- switched to named registers
; 2011-01-11 (trs) -- error-check for zero bad particles
; 2009-06-30 (trs) -- creates bad-particle list even if no good particles
; 2009-06-05 (trs) -- removes duplicate particles
; 2009-06-03 (trs) -- changed format of select/sel files
; 2009-05-13 (trs) -- sorted good and bad particle lists
; 2009-05-08 (trs) -- changed fraction kept per view to percentage kept
; 2009-04-03 (trs) -- added copy of GOODSEL for archival purposes (all in one directory)
; 2009-02-24 (trs) -- selection files now written to select/prj###
; 2009-02-23 (trs) -- added view# to GOODSEL output
; 2008-02-05 (trs) -- added divide-by-0 trap for empty views
; 2007-10-05 (trs) -- modified for stacked particles
; -- TO DO: check whether this still works on unstacked particles
; 2006-02-06 (trs,pp) -- bug fix: if picking by hand, CCC wasn't added to GOOD_PARTICLE_DOC
; 2005-08-30 (trs,hg) -- bug fix: error when good-class list not found
; 2004-10-19 (trs) -- bug fix: I had labelled loops with LB2 twice
; 2004-07-23 (trs,ga) -- added option to pick particles from classes by hand
; 2004-07-23 (trs) -- gets number of total particles from SELECT/SEL rather than HOW_MANY
; 2004-05-31 (trs) -- adds CCC to output selection doc
; 2004-05-05 (trs) -- handles reference-views with no good particles
; 2004-05-04 (trs) -- gets number of classes from class-stats doc
;