; Restacks particles
;
; PURPOSE: Restacks particles from a set of particle selection files
; into specified number of new stacks.
;
; SOURCE: spider/docs/techs/recon1/Procs/align-regroup.spi
;
; USAGE: clean ; spider spi/dat @align-regroup
;
; I/O Registers & files are set in: recon-settings.spi
;
; ------------------- Parameters -------------------
[newnumgrps] = 8 ; Number of new groups (e.g., number of nodes)
[wantStacksYN] = 1 ; Want stacks? (0 == statistics only)
[wantGlobalYN] = 1 ; Want global lookup table? (0 == no)
[wantAlignYN] = 0 ; Want alignment files also? (0 == no)
; INPUT FILES:
; [params] ../params Parameter doc file (one)
; [mic_sel_grp] ../sel_micrograph Group (or micrograph) selection file (one)
; [mic_sel_part] ../Particles/good/sel_part_**** Particle selection files (one/micrograph)
; [mic_parts] ../Particles/win/data_bymic_**** Existing particle stacks (one/micrograph)
; [mic_align] input/align_01_*** Alignment parameter files (one/group) (OPTIONAL)
;
; OUTPUT FILES:
; [init_dir] ../iter_0 New directory (one)
; [init_sel_grp] [init_dir]/sel_group New group selection file (one)
; [init_sel_part] [init_dir]/sel_part_*** New particle selection files (one/group)
; [init_unaligned] [init_dir]/data*** New particle stacks (one/group)
; [init_part2global] [init_dir]/part2glonum_*** Global particle lookup table (one/group) (OPTIONAL)
; [init_combined_parts] [init_dir]/global2group Combined global lookup table (one) (OPTIONAL)
; [init_align_doc] [init_dir]/align_01_*** Alignment parameter files (one/group) (OPTIONAL)
; -------------- END BATCH HEADER -------------------------------
; Get reconstruction parameters & file names
@recon-settings([pixsiz],[ang-step],[r2],[alignsh],[prj-radius],[winsiz],[incore-yn],[bp-type],[qsub],[masterCPUs],[groupCPUs])
MD ; Skip unnecessary output
VB OFF
MD ; Skip unnecessary output
TR OFF
; Get reconstruction file names
@recon-settings([pixsiz],[ang-step],[r2],[alignsh],[prj-radius],[winsiz],[incore-yn],[bp-type],[qsub],[masterCPUs],[groupCPUs])
[iter] = 1
@backup-dir ; Back up pre-existing directory
[init_dir] ; Directory to check
9 ; Maximum number of versions to back up
SYS ; Create output directory
mkdir -p [init_dir]
; Get image dimension
UD 17, [sp_winsiz]
[params]
UD E
UD N [old_num_grps] ; Find number of old groups (usually micrographs)
[mic_sel_grp] ; Old group (micrograph) selection file (input)
[ntot] = 0 ; Initialize overall cumulative total
DO ; Loop over all old groups (micrographs) -----------------
UD NEXT [key], [ingrp] ; Get old group (micrograph) number
[mic_sel_grp] ; Old group (micrograph) selection file (input)
IF ( [key] <= 0 ) EXIT
; Accumulate total number of particles
UD N [numparts]
[mic_sel_part][ingrp] ; Old particle selection file (input)
SYS
echo " Micrograph: {%I4%[ingrp]} has: {%I6%[numparts]} particles"
; Count number of selected particles
[ntot] = [ntot] + [numparts]
ENDDO ; End loop over micrographs -----------------
UD NEXT E ; Finished with doc file
[mic_sel_grp] ; Finished with doc file (input)
[newnpg] = [ntot]/[newnumgrps] ; Number of particles in new groups
[newnpg] = INT([newnpg] + 0.5) ; Number of particles in new groups
; [newnpg] = INT([ntot]/[newnumgrps] + 0.5) ; Expanded
SYS
echo ' ' ; echo " Restacking: {%I6%[ntot]} particles"
SYS
echo " From: {%I4%[old_num_grps]} groups into: {%I3%[newnumgrps]} groups, with {%I5%[newnpg]} particles each."
; Estimate memory requirements
[memMegs] = int(1 + 4*[sp_winsiz]*[sp_winsiz]*[newnpg]/1e+6)
SYS
echo " Each group will require {%i0%[memMegs]} MB of memory"
SYS
echo " (Reference projections and overhead will require a bit more)"
SYS
echo ' '
IF ( [wantStacksYN] == 0 ) THEN
EN D
ENDIF
DE ; Remove any new group selection file
[init_sel_grp] ; New group selection file (removed)
SD / GROUP NUM_PARTS
[init_sel_grp] ; Group selection doc file (output)
[one] = 1
DE A
[init_sel_part][one]
DE A
[init_unaligned][one]
DE A
[init_part2global][one]
IF ( [wantGlobalYN] == 1 ) THEN
DE
[init_combined_parts]
SD / APSHSLICE MIC_SLICE APSHGROUP MIC_NUM
[init_combined_parts]
ENDIF
; Compute number of particles for each new group
[newnpg] = [ntot]/[newnumgrps] ; Number of particles in new groups
[newnpg] = INT([newnpg] + 0.5) ; Number of particles in new groups
[outgrp] = 0 ; Initialize output group number
[outpart] = [newnpg] + 1 ; To force new output group
[needold] = 1 ; To force new input group
[part_counter] = 0 ; Initialize overall counter
DO ; Loop over all old particles ----------------
IF ([needold] >= 1 ) THEN
; Start another input group (micrograph)
IF ( [wantAlignYN] > 0) THEN
UD ICE ; Close doc file
[mic_align][ingrp] ; Old alignment doc file (closed)
ENDIF
UD NEXT [key], [ingrp] ; Get next group (micrograph) number
[mic_sel_grp] ; Group (micrograph) selection file (input)
IF ([key] <= 0) EXIT ; Finished with all old groups
[needold] = 0 ; Flag to use old group
ENDIF
[calcgrpnum] = int([part_counter]*[newnumgrps]/[ntot])+1 ; The group number we *should* be on
; IF ( [outpart] > [newnpg] ) THEN ; (OUTPART is incremented after this step, so OUTPART+1 would get included)
IF ( [calcgrpnum] > [outgrp] ) THEN
[part_counter] ; DIAGNOSTIC outpart newgroup
; Need to start another output group
IF ( [outgrp] > 0) THEN
; Not first output group, give details on last output group
SYS
echo ' '"Filled group: {%I3%[outgrp]} with: {%I6%[outpart]} particles."
SD E ; Close doc file
[init_sel_part][outgrp] ; Finished with this particle selection file (closed)
SD E ; Close doc file
[init_align_doc][outgrp] ; Finished with this alignment file (closed)
SD E ; Close doc file
[init_part2global][outgrp] ; Global number doc file (closed)
SD [outgrp],[outgrp],[outpart] ; Place group # in group selection file
[init_sel_grp] ; Group selection file (output)
ENDIF
[outgrp] = [outgrp] + 1 ; New output group #
IF ( [outgrp] <= [newnumgrps] ) THEN
; Delete pre-existing files and label new doc files
DE ; Remove existing particle selection file
[init_sel_part][outgrp] ; Particle selection file (removed)
SD / # ORIG_MICRO MIC_WIN_# GLOBAL_# GRP
[init_sel_part][outgrp] ; Particle selection file (output)
DE ; Remove existing output stack
[init_unaligned][outgrp] ; Particle stack file (removed)
IF ( [wantGlobalYN] > 0) THEN
DE ; Remove existing global lookup file
[init_part2global][outgrp] ; Existing global number lookup file (removed)
SD / KEY: WIN-NUM REG: GLOBAL-NUM MICROGRAPH; Label for new global number doc file
[init_part2global][outgrp] ; New global number doc file (output)
ENDIF
IF ( [wantAlignYN] > 0) THEN
;; DE ; Remove existing alignment file
;; [init_align_doc][outgrp] ; Existing global alignment file (removed)
IQ FI [alignExistsYN]
[init_align_doc][outgrp] ; Existing global alignment file
IF ( [alignExistsYN] .EQ. 1) THEN
SYS
\mv -v [init_align_doc][outgrp].$DATEXT [init_align_doc][outgrp].$DATEXT.bak
ENDIF
SD / KEY PSI, THE, PHI, REF#, EXP#, CUM.{ROT, SX, SY}, NPROJ, DIFF, CCROT, ROT, SX, SY, MIR-CC
[init_align_doc][outgrp] ; New global number doc file (output)
ENDIF
ENDIF
[outpart] = 0 ; Reset output particle # to zero
ENDIF ; For starting new group
UD NEXT [key], [inpart],[d2],[mic],[glonum] ; Get next input particle number
[mic_sel_part][ingrp] ; Old particle selection file (input)
IF ( [key] <= 0) THEN
[needold] = 1 ; Flag for need to start next input group
; SD [outgrp],[outgrp],[outpart] ; Place group # in group selection file
; [init_sel_grp] ; Group selection file (output)
UD NEXT E ; (Goes into infinite loop without this)
[mic_sel_part][ingrp]
CYCLE ; Finished with this input group
ENDIF
; Have old and new particle numbers now
[outpart] = [outpart] + 1 ; New output particle #
[part_counter] = [part_counter] + 1
; No CTF correction, just restacking
CP
[mic_parts][ingrp]@******[inpart] ; Existing particle stack (input)
[init_unaligned][outgrp]@******[outpart] ; New particle stack (output)
; FI H [mic],[glonum] ; Query image header values
; [mic_parts][ingrp]@******[inpart] ; Existing particle stack (input)
; MIC,NUM
;
; FI H [mic] ; Query image header values
; [mic_parts][ingrp]@******[inpart] ; Existing particle stack (input)
; MIC
SD [outpart], [outpart],[mic],[inpart],[glonum],[outgrp] ; Save new particle data
[init_sel_part][outgrp] ; Particle selection file (output)
IF ( [wantGlobalYN] > 0) THEN
SD [outpart],[glonum],[mic] ; Save lookup table
[init_part2global][outgrp] ; New global number doc file (output)
SD [glonum], [outpart],[inpart],[outgrp],[mic]
[init_combined_parts]
ENDIF
IF ( [wantAlignYN] > 0) THEN
; Get image dimension
UD IC [inpart], [r1],[r2],[r3],[r4],[r5],[r6],[r7],[r8],[r9],[r10],[r11],[r12],[r13],[r14],[r15]
[mic_align][ingrp] ; Old alignment doc file (input)
SD [outpart],[r1],[r2],[r3],[r4],[outpart],[r6],[r7],[r8],[r9],[r10],[r11],[r12],[r13],[r14],[r15]
[init_align_doc][outgrp] ; New alignment doc file (output)
ENDIF
ENDDO ; End loop over all old particles ----------------
IF ( [outpart] > 0) THEN
SYS
echo ' '"Filled group: {%I3%[outgrp]} with: {%I6%[outpart]} particles."
SD E ; Close doc file
[init_sel_part][outgrp] ; Output particle selection file (closed)
ENDIF
UD ICE ; Close doc file
[mic_align][ingrp] ; Old alignment doc file (closed)
; UD NEXT E ; Close doc file
; [init_sel_grp] ; Group selection doc file (closed)
UD NEXT E ; Close doc file
[mic_sel_grp] ; Group selection doc file (closed)
[dummy] = -[newnumgrps]
SD / PARTSBEFORE PARTS_AFTER
[init_sel_grp]
SD [dummy], [ntot],[part_counter]
[init_sel_grp]
SD E
[init_sel_grp]
IF ( [wantGlobalYN] == 1 ) THEN
SD E
[init_combined_parts]
ENDIF
SYS
echo ' '
EN
; Modified 2017-08-01
; 2017-07-28 (trs) -- backs up output directory
; 2016-09-21 (trs) -- renamed from restack.spi, which was previously restack-n-ctf.spi
; 2016-02-05 (trs) -- removed CTF correction
; 2016-02-05 (trs) -- estimates memory requirements
; 2014-08-13 (trs) -- added combined global lookup
; 2013-11-29 (trs) -- bug fix: went into infinite (?) loop if in-core files weren't closed
; 2013-11-28 (trs) -- bug fix: now gets micrograph# and global particle number from OLD_SEL_PART instead of stack
; 2013-10-16 (agl) -- modernized
; 2012-07-19 (trs) -- creates output directory if needed
; 2012-05-01 (trs) -- optionally writes global particle number to header position 42
; 2012-04-05 (trs) -- doesn't use output from pnums.spi anymore
; 2012-04-05 (trs) -- adapted from listallparticles.spi
; 2010-09-02 (trs & ay) -- trap for >999 micrographs for 'SD [dummy]' line
; 2009-10-16 (trs) -- removed option for unstacked images
; 2008-10-03 (trs) -- added output file part2global
; 2008-10-03 (trs) -- assigns global particle number
; 2007-12-17 (trs) -- optional parameter for maximum number of particles per micrograph
; 2007-10-10 (trs) -- gets number of micrographs from SEL_MICROGRAPH
;