; Regroup existing groups into new groups
;
; SOURCE: spider/docs/techs/recon1/Procs/recon-regroup
;
; PURPOSE: Regroup existing groups into new groups for reconstruction
; Creates new group selection file, new particle selection file,
; new group stacks, and new alignment parameter doc files
;
; USAGE: ./spider spi/dat @recon-regroup
; ------------------------ Parameters ------------------------
[newnumgrps] = 8 ; Number of new groups e.g. number of nodes (0 == keep same grouping)
[wantStacksYN] = 0 ; Want to actually create new groups? (1 == Yes, 0 == diagnostic)
[wantGlobalYN] = 1 ; Want global lookup table? (0 == no)
[wantAlignYN] = 1 ; Want alignment files also? (0 == no)
[wantDalaYN] = 1 ; Want aligned image files also? (0 == no)
; Get reconstruction parameters & file names
@recon-settings([pixsiz],[ang-step],[r2],[alignsh],[prj-radius],[winsiz],[incore-yn],[bp-type],[qsub],[masterCPUs],[groupCPUs])
; -------------------------- Inputs --------------------------
; INPUTS: (Where ### denotes view *** denotes group)
[avg_dir] = '../Averages' ; Input selection doc dir (one)
[old_sel_grp] = '[avg_dir]/sel_group_best' ; Old group selection list (one)
[old_sel_part] = '[avg_dir]/sel_part_best_***' ; Old particle selection lists (one/group)
; WIN MIC_NUM MIC_WIN GLOBAL_NUM GRP
[init_dir] = 'iter_0' ; Input alignment dir (will override recon-settings)
; Defined in recon-settings.spi: ([init_dir] denotes input directory)
; [out_align_doc] [init_dir]/align_01_*** Old alignment doc files (one/group)
; [init_unaligned] [init_dir]/data_*** Old particle stacks (one/group)
; [aligned_images] [init_dir]/dala_***@ Rotationally aligned particle images (one/group)
; [out_aligned] [init_dir]/dala_01_*** Old aligned images stacks (one-stack/group)
; ------------------------- Outputs -------------------------
;
; Set in: recon-settings
; OUTPUTS: (Where [rec_dir] denotes output directory, ### denotes view *** denotes group)
; [rec_unaligned] [rec_dir]/data_*** New particle stacks (one/group)
; [sel_group] [rec_dir]/sel_group New group selection file (one)
; [rec_sel_part] [rec_dir]/sel_part_*** New particle selection files (one/group)
; [rec_part2global] [rec_dir]/part2glonum_**** Global particle lookup tables (one/new-group) (OPTIONAL)
; [rec_aligned] [rec_dir]/dala_01_*** New aligned images (one-stack/new-group) (OPTIONAL)
; [rec_align_doc] [rec_dir]/align_01_*** New alignment parameter files (one/group) (OPTIONAL)
;
; --------------------- END BATCH HEADER ---------------------
MD ; Skip unnecessary output
VB OFF
MD ; Skip unnecessary output
TR OFF
; ; Get reconstruction parameters & file names
; @recon-settings([pixsiz],[ang-step],[r2],[alignsh],[prj-radius],[winsiz],[incore-yn],[bp-type],[qsub],[masterCPUs],[groupCPUs])
[iter] = 0
[next-iter] = [iter] + 1
@backup-dir ; Back up pre-existing directory
[init_dir] ; Directory to check
9 ; Maximum number of versions to back up
SYS ; Create output directory
mkdir -p [rec_dir]
UD N [old-num-grps] ; Find number of old groups (usually micrographs)
[old_sel_grp] ; Old group (micrograph) selection file (input)
; If regrouping not required, simply link xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
IF ( [newnumgrps] == 0 ) THEN
SYS
echo " No regrouping, linking pre-existing group files " ; date '+ TIME: %x %X'
; DE
; [sel_grp]
; Link group selection doc file
SYS
ln -sf ../[old_sel_grp].$DATEXT [sel_group] .$DATEXT
; Loop through old groups
DO [old-grp] = 1,[old-num-grps] ; Loop through old groups ----------------------
[grp] = [old-grp] ; part of variables {[rec_sel_part], [rec_align_doc], [rec_unaligned], [rec_aligned]}
; Link particle selection doc files
SYS
ln -sf ../[old_sel_part][old-grp].$DATEXT [rec_sel_part][grp].$DATEXT
; Link alignment doc files
SYS
ln -sf ../[out_align_doc][old-grp].$DATEXT [rec_align_doc][grp].$DATEXT
IF ( [wantStacksYN] > 0 ) THEN
; Stacks requested, link stack files
SYS
ln -sf ../[init_unaligned][old-grp].$DATEXT [rec_unaligned][grp].$DATEXT
ENDIF
IF ( [wantDalaYN] > 0 ) THEN
; Want aligned images (dala files), link aligned image files
SYS
ln -sf ../[out_aligned][old-grp].$DATEXT [rec_aligned][grp].$DATEXT
ENDIF
ENDDO ; End group loop -----------------------------
EN ; Finished (return)
ENDIF ; If simply linking to old groups
; Regrouping requested xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
SYS
echo -n " Regrouping files " ; date '+ TIME: %x %X'
[ntot] = 0 ; Initialize overall cumulative total
DO ; Loop over all old groups ----------------------------------
UD NEXT [key], [ingrp] ; Get old group (micrograph) number
[old_sel_grp] ; Old group (micrograph) selection file (input)
IF ([key] <= 0) EXIT
; Accumulate total number of particles
UD N [numparts]
[old_sel_part][ingrp] ; Old particle selection file (input)
SYS
echo " Old group: {%I3%[ingrp]} had:{%I6%[numparts]} particles."
; Count number of selected particles
[ntot] = [ntot] + [numparts]
ENDDO
UD NEXT E ; Finished with doc file
[old_sel_grp] ; Finished with doc file (input)
; Compute number of particles for each new group
[newnpg] = [ntot]/[newnumgrps] ; Number of particles in new groups
![newnpg] = INT([newnpg] + 0.5) ; Number of particles in new groups
[newnpg] = INT([newnpg]) ; Number of particles in new groups
[numt] = [newnpg] + 1
SYS
echo ; echo -n " Restacking: {%I5%[ntot]} particles"
SYS
echo " From: {%I3%[old-num-grps]} groups into: {%I3%[newnumgrps]} groups, with {%I5%[newnpg]} particles each."
; Estimate memory requirements
[memMegs] = int(1 + 4*[winsiz]*[winsiz]*[newnpg]/1e+6)
SYS
echo " Each group will require {%i0%[memMegs]} MB of memory"
SYS
echo ' '
IF ( [newnumgrps] == 0 ) THEN
EN
ENDIF
DE ; Remove any new group selection file
[sel_group] ; New group selection file (removed)
SD / GROUP NUM_PARTICLES
[sel_group] ; Group selection doc file (output)
DE ; Remove any new group selection file
[sel_group] ; New group selection file (removed)
SD / GROUP NUM_PARTS
[sel_group] ; Group selection doc file (output)
[grp] = 1
DE A
[rec_sel_part][grp]
DE A
[rec_part2global][grp]
DE A
[rec_align_doc][grp]
IF ( [wantStacksYN] > 0 ) THEN
DE A
[rec_aligned][grp]
ENDIF
IF ( [wantDalaYN] > 0 ) THEN
DE A
[rec_aligned][grp]
ENDIF
[outgrp] = 0 ; Initialize output group number
[outpart] = [newnpg] + 1 ; To force new output group
[needold] = 1 ; To force new input group
[part_counter] = 0 ; Initialize overall counter
DO ; Loop over all old particles ----------------
IF ( [needold] >= 1 ) THEN
; Start another input group (micrograph)
IF ( [wantAlignYN] > 0) THEN
UD ICE ; Close doc file
[out_align_doc][ingrp] ; Old alignment doc file (closed)
ENDIF
UD NEXT E ; Close doc file
[old_sel_part][ingrp] ; Finished with this particle selection file (closed)
UD NEXT [key], [ingrp] ; Get next group (micrograph) number
[old_sel_grp] ; Group (micrograph) selection file (input)
IF ( [key] <= 0 ) EXIT ; Finished with all old groups
[needold] = 0 ; Flag to use old group
ENDIF
[calcgrpnum] = int([part_counter]*[newnumgrps]/[ntot])+1 ; The group number we *should* be on
; IF ( [outpart] > [newnpg] ) THEN ; (OUTPART is incremented after this step, so OUTPART+1 would get included)
IF ( [calcgrpnum] > [outgrp] ) THEN
[part_counter] ; Diagnostic
[calcgrpnum]
[outgrp]
; Must start another output group
IF ( [outgrp] > 0 ) THEN
; Not first output group, give details on last output group
SYS
echo " New group: {%I3%[outgrp]} has:{%I6%[outpart]} particles."
SD E ; Close doc file
[rec_sel_part][outgrp] ; Finished with this particle selection file (closed)
SD E ; Close doc file
[rec_align_doc][outgrp] ; Finished with alignment file (closed)
SD E ; Close doc file
[rec_part2global][outgrp] ; Global number doc file (closed)
SD [outgrp], [outgrp],[outpart] ; Place group # in group selection file
[sel_group] ; Group selection file (output)
!sys
!echo 'grp: {***[outgrp]} outpart: {*****[outpart]}'
ENDIF
[outgrp] = [outgrp] + 1 ; New output group #
[grp] = [outgrp]
;;; [rec_align_doc] = '[next_group_align]'
IF ( [outgrp] <= [newnumgrps] ) THEN
; Delete pre-existing files and label new doc files
DE ; Remove existing particle selection file
[rec_sel_part][outgrp] ; Particle selection file (removed)
SD / WIN MIC_NUM MIC_WIN GLOBAL_NUM GRP
[rec_sel_part][outgrp] ; particle selection file (output)
DE ; Remove existing output stack
[rec_unaligned][outgrp] ; Particle stack file (removed)
IF ( [wantGlobalYN] > 0) THEN
DE ; Remove existing global lookup file
[rec_part2global][outgrp] ; Existing global number lookup file (removed)
SD / KEY: WIN-NUM REG: GLOBAL-NUM MICROGRAPH
[rec_part2global][outgrp] ; Label for global lookup doc file (output)
ENDIF
IF ( [wantAlignYN] > 0) THEN
;; DE ; Remove existing alignment file
;; [new_align][outgrp] ; Existing global alignment file (removed)
IQ FI [alignExistsYN]
[new_align][outgrp] ; Existing global alignment file
IF ( [alignExistsYN] .EQ. 1) THEN
SYS
\mv -v [new_align][outgrp].$DATEXT [new_align][outgrp].$DATEXT.bak
ENDIF
SD / KEY PSI, THE, PHI, REF#, EXP#, CUM.{ROT, SX, SY}, NPROJ, DIFF, CCROT, ROT, SX, SY, MIR-CC
[rec_align_doc][outgrp] ; New global number doc file (output)
ENDIF
ENDIF
[outpart] = 0 ; Reset output particle # to zero
ENDIF
UD NEXT [key], [inpart],[mic],[micwin],[glonum],[tgrp] ; Get next input particle number
[old_sel_part][ingrp] ; Old particle selection file (input)
IF ( [key] <= 0 ) THEN
[needold] = 1 ; Flag for need to start next input group
CYCLE ; Finished with this input group
ENDIF
; Have old and new particle numbers now xxxxxxxxxxxxxxxxx
[outpart] = [outpart] + 1 ; New output particle #
[part_counter] = [part_counter] + 1
; Restacking
[grp] = [outgrp] ; HACK
CP
[init_unaligned][ingrp]@******[inpart] ; Existing particle stack (input)
[rec_unaligned][outgrp]@******[outpart] ; New particle stack (output)
SD [outpart], [outpart],[mic],[micwin],[glonum],[outgrp] ; Save new particle data
[rec_sel_part][outgrp] ; New particle selection file (output)
IF ( [wantGlobalYN] > 0) THEN
; Want global look-up table
SD [outpart],[glonum],[mic] ; Save lookup table
[rec_part2global][outgrp] ; New global llk-up doc file (output)
ENDIF
IF ( [wantDalaYN] > 0) THEN
; Want old aligned images (dala files)
CP
[out_aligned][ingrp]@******[inpart] ; Existing aligned particle stack (input)
[rec_aligned][outgrp]@******[outpart] ; New aligned particle stack (output)
ENDIF
IF ( [wantAlignYN] > 0 ) THEN
; Want new alignment doc file
UD IC [inpart], [r1],[r2],[r3],[r4],[r5],[r6],[r7],[r8],[r9],[r10],[r11],[r12],[r13],[r14],[r15]
[out_align_doc][ingrp] ; Old alignment doc file (input)
SD [outpart],[r1],[r2],[r3],[r4],[outpart],[r6],[r7],[r8],[r9],[r10],[r11],[r12],[r13],[r14],[r15]
[rec_align_doc][outgrp] ; New alignment doc file (output)
ENDIF
ENDDO
IF ( [outpart] > 0 ) THEN
SD [outgrp], [outgrp],[outpart] ; Place group # in group selection file
[sel_group] ; Group selection file (output)
SYS
echo " New group: {%I3%[outgrp]} has:{%I6%[outpart]} particles."
SD E ; Close doc file
[rec_sel_part][outgrp] ; Output particle selection file (closed)
ENDIF
UD ICE ; Close doc file
[out_align_doc][ingrp] ; Old alignment doc file (closed)
[dummy] = -[newnumgrps]
SD / PARTSBEFORE PARTS_AFTER
[sel_group]
SD [dummy], [ntot],[part_counter]
[sel_group]
;;UD NEXT E ; Close doc file
SD E ; Close doc file
[sel_group] ; Group selection doc file (closed)
SYS
echo ' '
EN
; Modified 2017-08-04
; 2017-07-28 (trs) -- backs up output directory
; 2016-09-15 (trs) -- Estimates memory requirements
; 2016-07-07 (agl) -- file locations altered
; 2016-03-25 (agl) -- File locations
; 2013-12-02 (trs) -- Made numbering for each group consistent with RESTACK-N-CTF
; 2013-10-16 (agl) -- Modernized syntax, rewritten
; 2012-04-16 (trs) -- If grouping not required, creates softlinks to old files
; 2012-07-19 (trs) -- Creates output directory if needed
; 2012-05-01 (trs) -- Optionally writes global particle number to header position 42
; 2012-04-05 (trs) -- Doesn't use output from pnums.spi anymore
; 2012-04-05 (trs) -- Adapted from listallparticles.spi
; 2010-09-02 (trs & ay) -- Trap for >999 micrographs for 'SD [dummy]' line
; 2009-10-16 (trs) -- Removed option for unstacked images
; 2008-10-03 (trs) -- Added output file part2global
; 2008-10-03 (trs) -- Assigns global particle number
; 2007-12-17 (trs) -- Optional parameter for maximum number of particles per micrograph
; 2007-10-10 (trs) -- Gets number of micrographs from SEL_MICROGRAPH
;