deadtrees/dvc.yaml

# dvc project pipeline

# NOTE: see scripts/README.MD for preprocessing from raw data up to the first
# tracked stage `createtiles`

stages:
  createtiles:
    foreach:
      - 2017
      - 2018
      - 2019
      - 2020
    do:
      cmd: >-
        mkdir -p data/processed.images.${item};
        gdal_retile.py 
        -csv locations.csv 
        -v -ps ${source_dim} ${source_dim}
        -co "TILED=YES" -co "COMPRESS=LZW" -co "PREDICTOR=2" -co "ALPHA=NO" -co "NUM_THREADS=ALL_CPUS"
        -targetDir data/processed.images.${item} 
        data/raw/ortho_ms_${item}_EPSG3044.tif
      deps:
      - data/raw/ortho_ms_${item}_EPSG3044.tif
      params:
      - source_dim
      outs:
      - data/processed.images.${item}

  computestats:
    cmd: >-
      python scripts/computestats.py --frac 0.1
      data/processed.images.2017  
      data/processed.images.2018  
      data/processed.images.2019 
      data/processed.images.2020 
    deps:
      - data/processed.images.2017
      - data/processed.images.2018
      - data/processed.images.2019
      - data/processed.images.2020 
    outs:
      - data/processed.images.stats.json
  
  createmasks:
    foreach:
      - 2017
      - 2018
      - 2019
      - 2020
    do:
      cmd: >-
        python scripts/createmasks.py 
        data/processed.images.${item} 
        data/processed.masks.${item} 
        data/raw/shapefiles/deadtrees_${item}/deadtrees_${item}.shp
      deps:
      - data/processed.images.${item}
      - data/raw/shapefiles/deadtrees_${item}
      outs:
      - data/processed.masks.${item}

  createforestmasks:
    foreach:
      - 2017
      - 2018
      - 2019
      - 2020
    do:
      cmd: >-
        python scripts/createmasks.py 
        data/processed.images.${item} 
        data/processed.lus.${item} 
        data/raw/shapefiles/forestmask/CORINE_forest.shp
        --simple
      deps:
      - data/processed.images.${item}
      - data/raw/shapefiles/forestmask/CORINE_forest.shp
      outs:
      - data/processed.lus.${item}      

  createdataset:
    foreach:
      - 2017
      - 2018
      - 2019
      - 2020
    do:
      cmd: >-
        python scripts/createdataset.py 
        data/processed.images.${item} 
        data/processed.masks.${item} 
        data/processed.lus.${item} 
        data/dataset 
        --subdir train_${item}
        --source_dim ${source_dim}
        --tile_size ${createdataset.tile_size}
        --format ${file_type}
        --stats stats_${item}.csv
      deps:
      - data/processed.images.${item}
      - data/processed.masks.${item} 
      - data/processed.lus.${item} 
      params:
      - source_dim
      - createdataset.tile_size
      - file_type
      outs:
      - data/dataset/train_${item}
      - data/dataset/stats_${item}.csv 


  # we only merge 2017, 2019 for train, val, test - 2018, 2020 are test only
  mergedatasets:
    cmd: >-
      python scripts/mergedatasets.py data/dataset/train_2017 data/dataset/train_2019
    deps:
    - data/dataset/train_2017
    - data/dataset/train_2019
    outs:
    - data/dataset/train
    - data/dataset/val
    - data/dataset/test

  # train: do this manually

  # inference
  # overlap values possible: 32px or 128px
  inference:
    foreach:
      - 2017
      - 2018
      - 2019
      - 2020
    do:
      cmd: >-
        mkdir -p data/predicted.${item};
        stdbuf -i0 -o0 -e0 
        python scripts/inference.py --all --overlap 32 -o data/predicted.${item} data/processed.images.${item} 
        -m checkpoints/earnest-dew-216_epoch_235.ckpt 
        -m checkpoints/fine-lake-207_epoch_279.ckpt
        -m checkpoints/sage-glitter-214_epoch_106.ckpt;
        gdal_merge.py 
        -co "TILED=YES" -co "COMPRESS=LZW" -co "PREDICTOR=2" -co "NUM_THREADS=ALL_CPUS"
        -o data/predicted_mosaic_${item}.tif 
        data/predicted.${item}/ortho_ms_${item}_EPSG3044_*
      deps:
      - data/processed.images.${item}
      - checkpoints/earnest-dew-216_epoch_235.ckpt
      - checkpoints/fine-lake-207_epoch_279.ckpt
      - checkpoints/sage-glitter-214_epoch_106.ckpt
      outs:
      - data/predicted.${item}
      - data/predicted_mosaic_${item}.tif

      
  computestatsinference:
    cmd: >-
      python scripts/computestats_inference.py
      data/predicted.2017  
      data/predicted.2018  
      data/predicted.2019 
      data/predicted.2020  
    deps:
      - data/predicted.2017
      - data/predicted.2018
      - data/predicted.2019
      - data/predicted.2020  
    outs:
      - data/predicted.stats.csv

  computeaggregatemaps:
    cmd: >-
      mkdir data/maps;
      python scripts/aggregate_results.py
      data/processed.lus.2017  
      data/processed.lus.2018  
      data/processed.lus.2019  
      data/processed.lus.2020  
    deps:
      - data/predicted.2017
      - data/predicted.2018
      - data/predicted.2019
      - data/predicted.2020
      - data/processed.lus.2017
      - data/processed.lus.2018
      - data/processed.lus.2019
      - data/processed.lus.2020
    outs:
      - data/maps