VCSBeam version
Required: >= v2.18.18
# On Garrawarla, test vcsbeam version
$ module use /pawsey/mwa/software/python3/modulefiles
$ module load vcsbeam
$ make_mwa_tied_array_beam -V
MWA Beamformer v2.18.8_99df2a4
Offline FinePFB
$ cd /astro/mwavcs/asvo/252057
$ wget -O 1320499816.fits http://ws.mwatelescope.org/metadata/fits?obs_id=1320499816
$ cd /astro/mwavcs/vcs/1320499816/combined
$ for freq in {109..132}
do
sed "s/FREQ/${freq}/" < make_legacy.sbatch > make_legacy_ch${freq}.sbatch
sbatch make_legacy_ch${freq}.sbatch
done
#!/bin/bash -l
#SBATCH --nodes=1
#SBATCH --cpus-per-task=1
#SBATCH --ntasks-per-node=1
#SBATCH --mem=370gb
#SBATCH --partition=gpuq
#SBATCH --gres=gpu:1
#SBATCH --time=01:00:00
#SBATCH --account=mwavcs
#SBATCH --export=NONE
module use /pawsey/mwa/software/python3/modulefiles
module load vcsbeam
module load openmpi-ucx-gpu
DATAPATH=/astro/mwavcs/asvo/252057
srun -N 1 -n 1 fine_pfb_offline -m $DATAPATH/1320499816.metafits -b 1320499816 -T 600 -f FREQ -d $DATAPATH
Incoherent Beam
$ cd /astro/mwavcs/vcs/1320499816/vcsbeam/incoh
$ sbatch vcsbeam_incoh.sbatch
#!/bin/bash -l
#SBATCH --nodes=24
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=1
#SBATCH --mem=370gb
#SBATCH --partition=gpuq
#SBATCH --gres=gpu:1
#SBATCH --time=00:30:00
#SBATCH --account=mwavcs
#SBATCH --nice=0
#SBATCH --export=NONE
module use /pawsey/mwa/software/python3/modulefiles
module load vcsbeam
srun -N 24 -n 24 make_mwa_incoh_beam \
-m /astro/mwavcs/vcs/1320499816/vcsbeam/1320499816_legacy.metafits \
-b 1320499816 \
-T 600 \
-f 109 \
-d /astro/mwavcs/vcs/1320499816/combined \
-o incoh
After beamforming finishes, fold on B0031-07:
$ module load singularity
$ /pawsey/mwa/singularity/presto/presto.sif prepfold -psr J0034-0721 -nosearch -nodmsearch -start 0.01 incoh*.fits
Calibration
Offline Correlator
$ cd /astro/mwavcs/vcs/1320499816/cal/1320499816/vis
$ for freq in {109..132}
do
sed "s/FREQ/${freq}/" < offline_correlate.sbatch > offline_correlate_${freq}.sbatch
sbatch offline_correlate_${freq}.sbatch
done
#!/bin/bash -l
#SBATCH --nodes=1
#SBATCH --mem=370gb
#SBATCH --partition=gpuq
#SBATCH --gres=gpu:1
#SBATCH --time=01:00:00
#SBATCH --account=mwavcs
#SBATCH --export=NONE
module use /pawsey/mwa/software/python3/modulefiles
module load offline_correlator
for t in {8..599}
do
gpssecond=$(echo "$t + 1320499816" | bc)
gpubox=$(echo "FREQ - 108" | bc)
INPUT_DATA_FILE=/astro/mwavcs/vcs/1320499816/combined/1320499816_${gpssecond}_chFREQ.dat
START_SECOND=${gpssecond}
DUMPS_PER_SECOND=1 # This sets the output time resolution
# (e.g. 20 --> 1/20 = 0.05s = 50 ms)
# Minimum allowed resolution is 2 ms
CHANS_TO_AVERAGE=1 # This sets the output frequency resolution
# (e.g. 1 --> 1x10 kHz = 10 kHz)
GPUBOX_CHANNEL_NUMBER=${gpubox} # This should be chosen to "match" the input channel
# This is not easy! (mwalib handles this, but at the
# moment, offline_correlator is not using mwalib)
OUTPUT_PREFIX=1320499816 # Output files begin with this
srun -N 1 -n 1 offline_correlator \
-d ${INPUT_DATA_FILE} \
-s ${START_SECOND} \
-r ${DUMPS_PER_SECOND} \
-n ${CHANS_TO_AVERAGE} \
-c ${GPUBOX_CHANNEL_NUMBER} \
-o ${OUTPUT_PREFIX}
done
Hyperdrive
Attempt to run Hyperdrive for in-beam calibration (failed!)
$ cd /astro/mwavcs/vcs/1320499816/cal/1320499816/hyperdrive
$ sbatch hyperdrive.sbatch
#!/bin/bash -l
#SBATCH --job-name=hyp-$1
#SBATCH --output=hyperdrive.out
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=40
#SBATCH --time=01:00:00
#SBATCH --clusters=garrawarla
#SBATCH --partition=gpuq
#SBATCH --account=mwavcs
#SBATCH --export=NONE
#SBATCH --gres=tmp:50g
#SBATCH --gres=gpu:1
module use /pawsey/mwa/software/python3/modulefiles
module load hyperdrive
set -eux
which hyperdrive
#cd /astro/mwaeor/MWA/data/1090008640
METAFITS=1320499816_cal.metafits
if [[ ! -r srclist_1000.yaml ]]; then
hyperdrive srclist-by-beam -n 1000 -m ${METAFITS} /pawsey/mwa/software/python3/srclists/master/srclist_pumav3_EoR0aegean_fixedEoR1pietro+ForA_phase1+2.txt srclist_1000.yaml
fi
hyperdrive di-calibrate -s srclist_1000.yaml -d ../vis/*gpubox*.fits ${METAFITS}
Unfortunately, this doesn't work on the gpubox files as written out by the legacy offline_correlator, which doesn't write out most of the metadata info that mwalib needs to correctly parse those files. I'll work on this route later.
On MWAX_CORRELATOR observation 1320412440 (SUCCESS!)
At the moment, Harry is still helping me obtain the files. As it turned out, this obs was already downloaded for Cath and the EoR team, so Harry was able to just copy them to a place I can see them on Garrawarla: /astro/mwavcs/asvo/252007. Then, running them through Hyperdrive is straightfoward:
$ cd /astro/mwavcs/vcs/1320499816/cal/1320412440/hyperdrive
$ sbatch hyperdrive.sbatch
#!/bin/bash -l
#SBATCH --job-name=hyp-$1
#SBATCH --output=hyperdrive.out
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=40
#SBATCH --time=01:00:00
#SBATCH --clusters=garrawarla
#SBATCH --partition=gpuq
#SBATCH --account=mwavcs
#SBATCH --export=NONE
#SBATCH --gres=tmp:50g
#SBATCH --gres=gpu:1
module use /pawsey/mwa/software/python3/modulefiles
module load hyperdrive
set -eux
which hyperdrive
#cd /astro/mwaeor/MWA/data/1090008640
METAFITS=/astro/mwavcs/asvo/252007/1320412440.metafits
if [[ ! -r srclist_1000.yaml ]]; then
hyperdrive srclist-by-beam -n 1000 -m ${METAFITS} /pawsey/mwa/software/python3/srclists/master/srclist_pumav3_EoR0aegean_fixedEoR1pietro+ForA_phase1+2.txt srclist_1000.yaml
fi
hyperdrive di-calibrate -s srclist_1000.yaml -d /astro/mwavcs/asvo/252007/*.fits ${METAFITS}
This outputs hyperdrive_solutions.bin.
Using these solutions in vcsbeam requires >= v2.18.16.
$ mwa_plot_calibration -m 1320499816_legacy.metafits -c 1320499816_metafits_ppds.fits -C hyperdrive_solutions.bin -R NONE -U 0,0 -O > solution.txt
$ plot_calibration.py solution.txt
Fingers crossed this solution is still suitable for my observation!
RTS (failed!)
Create source list
$ cd /astro/mwavcs/vcs/1320499816/cal/1320499816
$ module load srclists/v1.0.0
$ wget -O 1320499816_metafits_ppds.fits http://ws.mwatelescope.org/metadata/fits?obs_id=1320499816&include_ppds=1
$ srclist_by_beam.py -m 1320499816_metafits_ppds.fits -n 1000 -s ${SRCLISTS_DIR}/srclist_pumav3_EoR0aegean_EoR1pietro+ForA.txt
Set up initial RTS job
First, we need a calibration metafits file (mode = 'HW_LFILES'), which for the moment I edited by hand. It doesn't matter whether the file is one with PPDs or without, but apparently some script somewhere is expected the file to be called \[obsid\]_metafits_ppds.fits. So I manually edited the (MWAX VCS) metafits file and just changed the mode, and then made a symbolic link to is called /astro/mwavcs/vcs/1320499816/cal/1320499816/rts/1320499816_metafits_ppds.fits. Then I ran the script from VCSTools:
cd /astro/mwavcs/vcs/1320499816/cal/1320499816/rts
$ calibrate_vcs.py -O 1320499816 -o 1320499816 -m 1320499816_metafits_ppds.fits -s ../srclist_pumav3_EoR0aegean_EoR1pietro+ForA_1320499816_patch1000.txt --offline --nosubmit
There's no good reason I couldn't have left off the --nosubmit option and had it run, but I just wanted to check it worked before trying it.
Run the RTS
$ cd /astro/mwavcs/vcs/1320499816/cal/1320499816/rts
$ sbatch /astro/mwavcs/vcs/1320499816/batch/RTS_1320499816.batch
#!/bin/bash -l
#SBATCH --export=NONE
#SBATCH --output=/astro/mwavcs/vcs/1320499816/batch/RTS_1320499816.out
#SBATCH --account=mwavcs
#SBATCH --clusters=garrawarla
#SBATCH --partition=gpuq
#
#SBATCH --cpus-per-task=1
#SBATCH --mem-per-cpu=10240MB
#SBATCH --nice=0
#SBATCH --chdir=/astro/mwavcs/vcs/1320499816/cal/1320499816/rts
#SBATCH --time=0:45:00
#SBATCH --nodes=13
#SBATCH --cpus-per-gpu=1
#SBATCH --gres=gpu:1
ncpus=1
export OMP_NUM_THREADS=1
module use /pawsey/mwa/software/python3/modulefiles/
module load vcstools/master
module load RTS/master
cd /astro/mwavcs/vcs/1320499816/cal/1320499816/rts
export UCX_MEMTYPE_CACHE=n
srun --export=all -N 25 -n 25 rts_gpu /astro/mwavcs/vcs/1320499816/cal/1320499816/rts/rts_1320499816.in
ReadAllFromSingleFile=
BaseFilename=/astro/mwavcs/vcs/1318345216/cal/1318345216/vis/*_GPUBOX
ReadGpuboxDirect=0
UseCorrelatorInput=1
ReadMetafitsFile=1
MetafitsFilename=/astro/mwavcs/vcs/1318345216/1318345216
DoCalibration=
doMWArxCorrections=1
doRawDataCorrections=1
doRFIflagging=0
useFastPrimaryBeamModels=0
generateDIjones=1
applyDIcalibration=1
UsePacketInput=0
UseThreadedVI=1
MaxFrequency=170.24
ObservationFrequencyBase=138.895
ObservationTimeBase=2455846.1252314816
ObservationPointCentreHA=-0.35952189154909175
ObservationPointCentreDec=-5.98757073636307
ChannelBandwidth=0.04
NumberOfChannels=32
CorrDumpsPerCadence=128
CorrDumpTime=1.0
NumberOfIntegrationBins=6
NumberOfIterations=1
StartProcessingAt=0
ArrayPositionLat=-26.7033194
ArrayPositionLong=116.6708152
ArrayNumberOfStations=128
ArrayFile=
SourceCatalogueFile=/astro/mwavcs/vcs/1318345216/cal/1318345216/srclist_pumav3_EoR0aegean_EoR1pietro+ForA_1318345216_patch1000.txt
NumberOfCalibrators=1
NumberOfSourcesToPeel=0
calBaselineMin=20.0
calShortBaselineTaper=40.0
FieldOfViewDegrees=1
For an as yet unknown reason, this RTS job timed out, even after allowing 4 hours of wall time.
This also requires vcsbeam >= v2.18.16
Using the Hyperdrive solution (1320412440)
$ cd /astro/mwavcs/vcs/1320499816/vcsbeam/coh
$ wget -O 1320412440_metafits_ppds.fits http://ws.mwatelescope.org/metadata/fits?obs_id=1320412440&include_ppds=1
$ sbatch vcsbeam.sbatch
00:34:08.8703 -07:21:53.409
#!/bin/bash -l
#SBATCH --nodes=24
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=1
#SBATCH --mem=370gb
#SBATCH --partition=gpuq
#SBATCH --gres=gpu:1
#SBATCH --time=01:00:00
#SBATCH --account=mwavcs
#SBATCH --nice=0
#SBATCH --export=NONE
module use /pawsey/mwa/software/python3/modulefiles
module load vcsbeam
srun -N 24 -n 24 make_mwa_tied_array_beam \
-m /astro/mwavcs/vcs/1320499816/vcsbeam/1320499816_legacy.metafits \
-b 1320499824 \
-T 592 \
-f 109 \
-d /astro/mwavcs/vcs/1320499816/combined \
-P pointings.txt \
-c 1320412440_metafits_ppds.fits \
-C /astro/mwavcs/vcs/1320499816/cal/1320412440/hyperdrive/hyperdrive_solutions.bin \
-p -R NONE -U 0,0 -O