deadtrees/dvc.yaml

# dvc project pipeline

# year 2017 inference only (disabled for now)
stages:
  createtiles:
    foreach:
      - 2017
      - 2019
    do:
      cmd: >-
        mkdir -p data/processed.images.${item};
        gdal_retile.py 
        -csv locations.csv 
        -v -ps ${source_dim} ${source_dim}
        -co "TILED=YES" -co "COMPRESS=LZW" -co "PREDICTOR=2"
        -targetDir data/processed.images.${item} 
        data/raw/ortho_${item}_ESPG3044.tif
      deps:
      - data/raw/ortho_${item}_ESPG3044.tif
      params:
      - source_dim
      outs:
      - data/processed.images.${item}

  createtiles-nir:
    foreach:
      - 2017
      - 2019
    do:
      cmd: >-
        mkdir -p data/processed.images.${item}.nir;
        gdal_retile.py 
        -csv locations.csv 
        -v -ps ${source_dim} ${source_dim}
        -co "TILED=YES" -co "COMPRESS=LZW" -co "PREDICTOR=2"
        -targetDir data/processed.images.${item}.nir 
        data/raw/ortho_${item}_ESPG3044_NIR.tif
      deps:
      - data/raw/ortho_${item}_ESPG3044_NIR.tif
      params:
      - source_dim
      outs:
      - data/processed.images.${item}.nir

  createmasks:
    cmd: >-
      python scripts/createmasks.py 
      --bbox data/raw/shapefiles/deadtrees_area/deadtrees_area.shp
      data/processed.images.2019 
      data/processed.masks.2019 
      data/raw/shapefiles/deadtrees/deadtrees.shp
    deps:
    - data/processed.images.2019
    - data/raw/shapefiles/deadtrees
    - data/raw/shapefiles/deadtrees_area
    outs:
    - data/processed.masks.2019

  createdataset:
    cmd: >-
      python scripts/createdataset.py 
      data/processed.images.2019 
      data/processed.masks.2019 
      data/dataset 
      --source_dim ${source_dim}
      --tile_size ${createdataset.tile_size}
      --format ${file_type}
    deps:
    - data/processed.images.2019
    - data/processed.masks.2019 
    params:
    - source_dim
    - createdataset.tile_size
    - file_type
    outs:
    - data/dataset/train
    - data/dataset/stats.csv 

  createbalanced:
    cmd: >-
      python scripts/createbalanced.py 
      data/dataset/stats.csv
      data/dataset/train 
      data/dataset/train_balanced 
      data/dataset/train_balanced_short
      --format ${file_type}
      --tmp-dir ./tmp
    deps:
    - data/dataset/train
    - data/dataset/stats.csv
    params:
    - file_type
    outs:
    - data/dataset/train_balanced
    - data/dataset/train_balanced_short

  createextra:
    cmd: >-
      mkdir -p data/extra;
      python scripts/createextra.py 
      data/raw/shapefiles/deadtrees_notdead/deadtrees_notdead.shp
      data/processed.images.2019
      data/dataset/train_balanced
      data/dataset/train_balanced_short 
      --tmp-dir ./tmp;
      mv data/extra/ds2-train-00* data/dataset/train_balanced_short/
      mv data/extra/ds3-train-00* data/dataset/train_balanced_short/
    deps:
    - data/raw/shapefiles/deadtrees_notdead
    - data/processed.images.2019
    - data/dataset/train_balanced
    - data/dataset/train_balanced_short
    outs:
    - data/dataset/train_balanced_short/ds2-train-0*
    - data/dataset/train_balanced_short/ds3-train-0*

  # train:

  # inference
  inference:
    foreach:
      - 2017
      - 2019
    do:
      cmd: >-
        mkdir -p data/predicted.${item};
        mkdir -p data/predicted.${item}_preview;
        stdbuf -i0 -o0 -e0 python deadtrees/deployment/tiler.py --all -o data/predicted.${item} data/processed.images.${item};
        gdal_merge.py 
        -co 'COMPRESS=LZW' 
        -co 'TILED=YES' 
        -o data/predicted_mosaic_${item}.tif 
        data/predicted.${item}/ortho_${item}_ESPG3044_*
      deps:
      - data/processed.images.${item}
      outs:
      - data/predicted.${item}
      - data/predicted.${item}_preview
      - data/predicted_mosaic_${item}.tif