diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | pilot.sh | 19 |
2 files changed, 13 insertions, 10 deletions
@@ -11,7 +11,6 @@ This script assumes the following: * All relevant input data is available either in either the `$HOME` directory or in a directory henceforth called `$BINDDIR`. Targets of any links in these directories should be accessible to the compute directories, as these will be mounted during relevant jobs. -* The output will be written to a results directory in `$BINDDIR`. * This script will be used with the SLURM queuing system on COSMA5 with the following options: `-p cosma5 -A durham -t 72:00:00`. If these options are not appropriate or if this script is to be run on other SLURM-run clusters one must set `$TOIL_SLURM_ARGS` prior to running. * `$CWL_SINGULARITY_CACHE` is set and the corresponding path contains (a link to) a singularity container `vlbi-cwl.sif`. @@ -33,6 +32,9 @@ Options can be the following: * `--scratch` is a path to local scratch storage where temporary data can be written to (optional). **`--scratch` must be local to the compute node. Nonlocal scratch storage will likely cause the pipeline to fail.** +* `--outdir` is a path relative to which intermediate files and final data products will be written. + Will be created if it does not exist. + If not specified, `$BINDDIR` will be used instead. * `<workflow name>` is the workflow file name without extension, e.g. `delay-calibration` or `concatenate-flag` for the VLBI pipeline or `HBA_calibrator` or `HBA_target` for LINC. ## Notes @@ -18,7 +18,7 @@ set_container() { fi } -opts=$(getopt -o rhf:c:p: --long restart,help,container:,scratch: \ +opts=$(getopt -o rhf:c:p: --long restart,help,container:,scratch:,outdir: \ -n 'pilot' -- "$@") eval set -- "$opts" @@ -33,6 +33,7 @@ while true; do -p ) PIPELINE="${2}"; shift 2 ;; -r | --restart) RESTART="--restart"; shift ;; --scratch) SCRATCH="${2}"; shift 2 ;; + --outdir) OUTDIR="${2}"; shift 2;; * ) shift; break ;; esac done @@ -52,14 +53,14 @@ INPUT_DIR="${2}" PIPELINE_LOG="${HOME}/${WORKFLOW_NAME}.log" -TMP_OUTDIR="${INPUT_DIR}/toil/tmp/tmp/" -JOB_LOG_DIR="${INPUT_DIR}/toil/logs/" -BATCH_LOG_DIR="${TOIL_BATCH_LOGS_DIR:-${INPUT_DIR}/toil/logs}" -WORK_DIR="${INPUT_DIR}/toil/work/" -# TODO: optionally decouple output directory from input directory? -OUTPUT_DIR="${INPUT_DIR}/${WORKFLOW_NAME}_results" -JOBSTORE_DIR="${INPUT_DIR}/toil/${WORKFLOW_NAME}_job/" -STATS_DIR="${OUTPUT_DIR}/stats" +TMP_OUTDIR="${OUTDIR-$INPUT_DIR}/toil/tmp/tmp/" +JOB_LOG_DIR="${OUTDIR-$INPUT_DIR}/toil/logs/" +BATCH_LOG_DIR="${OUTDIR-$TOIL_BATCH_LOGS_DIR:-${INPUT_DIR}/toil/logs}" +WORK_DIR="${OUTDIR-$INPUT_DIR}/toil/work/" + +OUTPUT_DIR="${OUTDIR-$INPUT_DIR}/${WORKFLOW_NAME}_results" +JOBSTORE_DIR="${OUTDIR-$INPUT_DIR}/toil/${WORKFLOW_NAME}_job/" +STATS_DIR="${OUTDIR-$OUTPUT_DIR}/stats" mkdir -p "$JOB_LOG_DIR" mkdir -p "$BATCH_LOG_DIR" |