deadtrees/dvc.yaml

# dvc project pipeline

# NOTE: see scripts/README.MD for preprocessing from raw data up to the first
# tracked stage `createtiles`

stages:
  createtiles:
    foreach:
      - 2017
      - 2019
    do:
      cmd: >-
        mkdir -p data/processed.images.${item};
        gdal_retile.py 
        -csv locations.csv 
        -v -ps ${source_dim} ${source_dim}
        -co "TILED=YES" -co "COMPRESS=LZW" -co "PREDICTOR=2" -co "ALPHA=NO" -co "NUM_THREADS=ALL_CPUS"
        -targetDir data/processed.images.${item} 
        data/raw/ortho_ms_${item}_EPSG3044.tif
      deps:
      - data/raw/ortho_ms_${item}_EPSG3044.tif
      params:
      - source_dim
      outs:
      - data/processed.images.${item}

  computestats:
    cmd: >-
      python scripts/computestats.py --frac 0.1
      data/processed.images.2017  
      data/processed.images.2019 
    deps:
      - data/processed.images.2017
      - data/processed.images.2019
    outs:
      - data/processed.images.stats.json
  
  createmasks:
    cmd: >-
      python scripts/createmasks.py 
      data/processed.images.2019 
      data/processed.masks.2019 
      data/raw/shapefiles/deadtrees_2019/deadtrees_2019.shp
      --negativesample data/raw/shapefiles/deadtrees_notdead/deadtrees_notdead.shp
    deps:
    - data/processed.images.2019
    - data/raw/shapefiles/deadtrees_2019
    - data/raw/shapefiles/deadtrees_notdead
    outs:
    - data/processed.masks.2019
    - data/processed.masks.2019.neg_sample

  createdataset:
    cmd: >-
      python scripts/createdataset.py 
      data/processed.images.2019 
      data/processed.masks.2019 
      data/dataset 
      --source_dim ${source_dim}
      --tile_size ${createdataset.tile_size}
      --format ${file_type}
    deps:
    - data/processed.images.2019
    - data/processed.masks.2019 
    - data/processed.masks.2019.neg_sample 
    params:
    - source_dim
    - createdataset.tile_size
    - file_type
    outs:
    - data/dataset/train
    - data/dataset/stats.csv 

  # train: do this manually

  # inference
  inference:
    foreach:
      - 2017
      - 2019
    do:
      cmd: >-
        mkdir -p data/predicted.${item};
        mkdir -p data/predicted.${item}_preview;
        stdbuf -i0 -o0 -e0 python deadtrees/deployment/tiler.py --all -o data/predicted.${item} data/processed.images.${item};
        gdal_merge.py 
        -co "TILED=YES" -co "COMPRESS=LZW" -co "PREDICTOR=2" -co "NUM_THREADS=ALL_CPUS"
        -o data/predicted_mosaic_${item}.tif 
        data/predicted.${item}/ortho_ms_${item}_EPSG3044_*
      deps:
      - data/processed.images.${item}
      - checkpoints/bestmodel.ckpt
      outs:
      - data/predicted.${item}
      - data/predicted.${item}_preview
      - data/predicted_mosaic_${item}.tif

      
  computestatsinference:
    cmd: >-
      python scripts/computestats_inference.py
      data/predicted.2017  
      data/predicted.2019 
    deps:
      - data/predicted.2017
      - data/predicted.2019
    outs:
      - data/predicted.stats.csv