WODEN simulations

#!/bin/bash --login
#SBATCH --nodes=1         ##Ask for one node per job in the array
#SBATCH --partition=gpuq  ##Use the GPU queue
#SBATCH --gres=gpu:1      ##Only ask for one GPU per process
#SBATCH --time=00:10:00   ##This simulation takes less than 10 mins per GPU
#SBATCH --account=mwaeor  ##If you aren't MWA EoR, you'll need a different account
#SBATCH --mem=10gb        ##Probably more memory that needed, but some calcs happen on CPU
#SBATCH --ntasks=1        ##Only one process per job in array
#SBATCH --array=1-24      ##Launch 24 jobs, with env variable $SLURM_ARRAY_TASK_ID=1 through 24 inclusive

module use /pawsey/mwa/software/python3/modulefiles       ##Use Chris' fancy module load things
module load woden                                         ##Load up everything you need for WODEN

cd /astro/mwaeor/jline/test_WODEN/MWA_analytic-vs-FEE     ##Where you want to run WODEN

mkdir -p data                                             ##If you want to store outputs somewhere else, make a dir

time run_woden.py \
    --ra0=60.0 --dec0=-27.0 \ ##Set the phase centre of the simulation to ra, dec = 60, -27.0 degrees
    --num_freq_channels=16 --freq_res=80e+3 \ ##Simulate 16 frequency channels per coarse band, of resolution 80kHz
    --num_time_steps=14 --time_res=8.0 \ ##Simulate 14 time steps, of resolution 8 seconds
    --cat_filename=/astro/mwaeor/jline/software/WODEN/woden-srclist_pumav3.txt \ ##Path to a WODEN-style sky model
    --metafits_filename=/astro/mwaeor/jline/software/WODEN/examples/metafits/1136380296_metafits_ppds.fits \ ##Path to a metafits file containing observational settings
    --band_nums=$SLURM_ARRAY_TASK_ID \ ##Use the array job number to simulate different coarse bands - 1 is lowest freq, 24 highest. Uses values in metafits file to set freq.
    --output_uvfits_prepend=./data/MWA_EoR1_FEE \ ##What to name the outputs. For --band_nums=1, this will make ./data/MWA_EoR1_FEE_band01.uvfits
    --primary_beam=MWA_FEE \ ##Which primary beam to use - this one uses the MWA Fully Embedded Element Beam (with no interpolation)
    --sky_crop_components ##WODEN defaults to horizon cropping sky model by SOURCE, but this catalogue happens to only have one giant SOURCE

$ sbatch run_woden_EoR1.sh

$ FREQUENCY OPTIONS:
  --band_nums BAND_NUMS
                        Defaults to running 24 coarse bands. Alternatively, enter
                        required numbers delineated by commas, e.g. --band_nums=1,7,9
  --lowest_channel_freq LOWEST_CHANNEL_FREQ
                        Set the frequency (Hz) of the lowest channel for band 1. If
                        using a metafits file, this will override the frequency in the
                        metafits
  --coarse_band_width COARSE_BAND_WIDTH
                        Set the width of each coarse band If using a metafits file,
                        this will override the frequency in the metafits
  --num_freq_channels NUM_FREQ_CHANNELS
                        Number of fine frequency channels to simulate - defaults to
                        --coarse_band_width / --freq_res
  --freq_res FREQ_RES   Fine channel frequnecy resolution (Hz) - will default to what
                        is in the metafits

TIME OPTIONS:
  --num_time_steps NUM_TIME_STEPS
                        The number of time steps to simualte. Defaults to how many are
                        inthe metafits if using metafits
  --time_res TIME_RES   Time resolution (s) - will default to what is in the metafits
                        if the metafits if using metafits

OBSERVATION OPTIONS:
  --ra0 RA0             RA of the desired phase centre (deg)
  --dec0 DEC0           Dec of the desired phase centre (deg)
  --date DATE           Initial UTC date of the observatio in format YYYY-MM-
                        DDThh:mm:ss This is used to set the LST and array precession.
                        This is set automatically when reading a metafits but including
                        this will override the date in the metafits

TELESCOPE OPTIONS:
  --latitude LATITUDE   Latitude (deg) of the array - defaults to MWA at -26.7033194444
  --longitude LONGITUDE
                        Longitude (deg) of the array - defaults to MWA at 116.670813889
  --array_height ARRAY_HEIGHT
                        Height (m) of the array above sea level - defaults to MWA at
                        377.0
  --array_layout ARRAY_LAYOUT
                        Instead of reading the array layout from the metafits file,
                        read from a text file. Store antenna positions as offset from
                        array centre, in east, north, height coords (metres)
  --primary_beam PRIMARY_BEAM
                        Which primary beam to use in the simulation.
                        Options are:
                        	 - MWA_FEE (MWA fully embedded element model)
                        	 - MWA_analy (MWA analytic model)
                        	 - Gaussian (Analytic symmetric Gaussian)
                        		 see --gauss_beam_FWHM and
                        		 and --gauss_beam_ref_freq for
                        		 fine control
                        	 - EDA2 (Analytic dipole with a ground mesh) 
                        	 - none (Don't use a primary beam at all)
                        Defaults to --primary_beam=none
  --gauss_beam_FWHM GAUSS_BEAM_FWHM
                        The FWHM of the Gaussian beam in deg - WODEN defaults to using
                        20 deg if this is not set
  --gauss_beam_ref_freq GAUSS_BEAM_REF_FREQ
                        The frequency at which the gauss beam FWHM is set at. If not
                        set, WODEN will default to 150MHz.
  --gauss_ra_point GAUSS_RA_POINT
                        The initial RA (deg) to point the Gaussian beam at. This will
                        be used to calculate an hour angle at which the beam will
                        remain pointed at for the duration of the observation. Defaults
                        to the RA of the metafits if available, or the RA of the phase
                        centre if not
  --gauss_dec_point GAUSS_DEC_POINT
                        The initial Dec (deg) to point the Gaussian beam at. Defaults
                        to the Dec of the metafits if available, or the Dec of the
                        phase centre if not
  --hdf5_beam_path HDF5_BEAM_PATH
                        Location of the hdf5 file holding the FEE beam coefficients
  --MWA_FEE_delays MWA_FEE_DELAYS
                        A list of 16 delays to point the MWA primary beam 
                        model enter as as list like: 
                        	--MWA_FEE_delays=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
                        for a zenith pointing. This is read directly from
                        the metafits if using a metafits file
  --telescope_name TELESCOPE_NAME
                        Name of telescope written out to the uvfits file, defaults to
                        MWA

INPUT/OUTPUT OPTIONS:
  --cat_filename CAT_FILENAME
                        Path to WODEN style sky model
  --metafits_filename METAFITS_FILENAME
                        MWA style metafits file to base the simulation on. Array
                        layout, frequency and time parameters are all set by this
                        option, but can be overridden using other arguments
  --output_uvfits_prepend OUTPUT_UVFITS_PREPEND
                        Prepend name for uvfits - will append band%02d.uvfits %band_num
                        at the end. Defaults to "output".
  --sky_crop_components
                        WODEN will crop out sky model information that is below the
                        horizon for the given LST. By default, for each SOURCE in the
                        sky model, if any COMPONENT is below the horizon, the entire
                        source will be flagged. If --sky_crop_components is included
                        WODEN will include any COMPONENT above the horizon, regardless
                        of which SOURCE it belongs to.

SIMULATOR OPTIONS:
  --remove_phase_tracking
                        By adding this flag, remove the phase tracking of the
                        visibilities - use this to feed uvfits into the RTS
  --no_tidy             Defaults to deleting output binary files from woden and json
                        files. Add this flag to not delete those files
  --chunking_size CHUNKING_SIZE
                        The chunk size to break up the point sources into for
                        processing - defaults to 0 (use default chunking in WODEN)
  --dry_run             Add this to NOT call the WODEN executable - this will just
                        write out the .json file and do nothing else

#!/bin/bash --login
#SBATCH --nodes=1         ##Ask for one node per job in the array
#SBATCH --partition=gpuq  ##Use the GPU queue
#SBATCH --gres=gpu:1      ##Only ask for one GPU per process
#SBATCH --time=00:10:00   ##This simulation takes less than 10 mins per GPU
#SBATCH --account=mwaeor  ##If you aren't MWA EoR, you'll need a different account
#SBATCH --mem=10gb        ##Probably more memory that needed, but some calcs happen on CPU
#SBATCH --ntasks=1        ##Only one process per job in array
#SBATCH --array=1-24      ##Launch 24 jobs, with env variable $SLURM_ARRAY_TASK_ID=1 through 24 inclusive
 
module use /pawsey/mwa/software/python3/modulefiles       
module load singularity                                   ##Load up singularity

mkdir -p data                                             ##If you want to store outputs somewhere else, make a dir

singularity exec --nv path/to/woden/dockerimage run_woden.py \
    --ra0=60.0 --dec0=-27.0 \ ##Set the phase centre of the simulation to ra, dec = 60, -27.0 degrees
    --num_freq_channels=16 --freq_res=80e+3 \ ##Simulate 16 frequency channels per coarse band, of resolution 80kHz
    --num_time_steps=14 --time_res=8.0 \ ##Simulate 14 time steps, of resolution 8 seconds
    --cat_filename=/astro/mwaeor/jline/software/WODEN/woden-srclist_pumav3.txt \ ##Path to a WODEN-style sky model
    --metafits_filename=/astro/mwaeor/jline/software/WODEN/examples/metafits/1136380296_metafits_ppds.fits \ ##Path to a metafits file containing observational settings
    --band_nums=$SLURM_ARRAY_TASK_ID \ ##Use the array job number to simulate different coarse bands - 1 is lowest freq, 24 highest. Uses values in metafits file to set freq.
    --output_uvfits_prepend=./data/MWA_EoR1_FEE \ ##What to name the outputs. For --band_nums=1, this will make ./data/MWA_EoR1_FEE_band01.uvfits
    --primary_beam=MWA_FEE \ ##Which primary beam to use - this one uses the MWA Fully Embedded Element Beam (with no interpolation)
    --sky_crop_components ##WODEN defaults to horizon cropping sky model by SOURCE, but this catalogue happens to only have one giant SOURCE