Bookdata-tools/import/dvc.yaml

# This file uses the two-stage status design, so new command stages
# need to be accompanied by an entry in the status stage at the end
stages:
  az-ratings:
    cmd: python run.py --rust pcat -t az.raw_ratings -s az-ratings -T import/az-ratings.transcript
      -D az-schema -f CSV data/ratings_Books.csv
    wdir: ..
    deps:
    - data/ratings_Books.csv
    - schemas/az-schema.status
    outs:
    - import/az-ratings.transcript

  bx-ratings:
    cmd: python run.py bx-import -T import/bx-ratings.transcript data/BX-Book-Ratings.csv
    wdir: ..
    deps:
    - data/BX-Book-Ratings.csv
    - schemas/bx-schema.status
    outs:
    - import/bx-ratings.transcript

  gr-authors:
    cmd: python run.py --rust import-json -T import/gr-authors.transcript --stage
      gr-authors -D gr-schema --truncate import/gr-authors.toml data/goodreads_book_authors.json.gz
    wdir: ..
    deps:
    - data/goodreads_book_authors.json.gz
    - import/gr-authors.toml
    - schemas/gr-schema.status
    outs:
    - import/gr-authors.transcript

  gr-book-genres:
    cmd: python run.py --rust import-json -T import/gr-book-genres.transcript --stage
      gr-book-genres -D gr-schema --truncate import/gr-book-genres.toml data/goodreads_book_genres_initial.json.gz
    wdir: ..
    deps:
    - data/goodreads_book_genres_initial.json.gz
    - import/gr-book-genres.toml
    - schemas/gr-schema.status
    outs:
    - import/gr-book-genres.transcript

  gr-book-series:
    cmd: python run.py --rust import-json -T import/gr-book-series.transcript --stage
      gr-book-series -D gr-schema --truncate import/gr-book-series.toml data/goodreads_book_series.json.gz
    wdir: ..
    deps:
    - data/goodreads_book_genres_initial.json.gz
    - import/gr-book-series.toml
    - schemas/gr-schema.status
    outs:
    - import/gr-book-series.transcript

  gr-books:
    cmd: python run.py --rust import-json -T import/gr-books.transcript --stage gr-books
      -D gr-schema --truncate import/gr-books.toml data/goodreads_books.json.gz
    wdir: ..
    deps:
    - data/goodreads_books.json.gz
    - import/gr-books.toml
    - schemas/gr-schema.status
    outs:
    - import/gr-books.transcript

  gr-interactions:
    cmd: python run.py --rust import-json -T import/gr-interactions.transcript --stage
      gr-interactions -D gr-schema --truncate import/gr-interactions.toml data/goodreads_interactions.json.gz
    wdir: ..
    deps:
    - data/goodreads_interactions.json.gz
    - import/gr-interactions.toml
    - schemas/gr-schema.status
    outs:
    - import/gr-interactions.transcript

  gr-works:
    cmd: python run.py --rust import-json -T import/gr-works.transcript --stage gr-works
      -D gr-schema --truncate import/gr-works.toml data/goodreads_book_works.json.gz
    wdir: ..
    deps:
    - data/goodreads_book_works.json.gz
    - import/gr-works.toml
    - schemas/gr-schema.status
    outs:
    - import/gr-works.transcript

  loc-mds-books:
    cmd: python run.py --rust parse-marc --db-schema locmds -t book_marc_field --truncate
      --stage loc-mds-books -D loc-mds-schema --transcript import/loc-mds-books.transcript
      --src-dir data/loc-books --src-prefix BooksAll.2016
    wdir: ..
    deps:
    - data/loc-books
    - schemas/loc-mds-schema.status
    outs:
    - import/loc-mds-books.transcript

  loc-mds-names:
    cmd: python run.py --rust parse-marc --db-schema locmds -t name_marc_field --truncate
      --stage loc-mds-names -D loc-mds-schema --transcript import/loc-mds-names.transcript
      --src-dir data/loc-names --src-prefix Names.2016
    wdir: ..
    deps:
    - data/loc-names
    - schemas/loc-mds-schema.status
    outs:
    - import/loc-mds-names.transcript

  ol-authors:
    cmd: python run.py --rust import-json -T import/ol-authors.transcript --stage
      ol-authors -D ol-schema --truncate import/ol-authors.toml data/ol_dump_authors.txt.gz
    wdir: ..
    deps:
    - data/ol_dump_authors.txt.gz
    - import/ol-authors.toml
    - schemas/ol-schema.status
    outs:
    - import/ol-authors.transcript

  ol-editions:
    cmd: python run.py --rust import-json -T import/ol-editions.transcript --stage
      ol-editions -D ol-schema --truncate import/ol-editions.toml data/ol_dump_editions.txt.gz
    wdir: ..
    deps:
    - data/ol_dump_editions.txt.gz
    - import/ol-editions.toml
    - schemas/ol-schema.status
    outs:
    - import/ol-editions.transcript

  ol-works:
    cmd: python run.py --rust import-json -T import/ol-works.transcript --stage ol-works
      -D ol-schema --truncate import/ol-works.toml data/ol_dump_works.txt.gz
    wdir: ..
    deps:
    - data/ol_dump_works.txt.gz
    - import/ol-works.toml
    - schemas/ol-schema.status
    outs:
    - import/ol-works.transcript

  viaf:
    cmd: python run.py --rust parse-marc --db-schema viaf -t marc_field --truncate
      --stage viaf -D viaf-schema --transcript import/viaf.transcript --line-mode
      data/viaf-clusters-marc21.xml.gz
    wdir: ..
    deps:
    - data/viaf-clusters-marc21.xml.gz
    - schemas/viaf-schema.status
    outs:
    - import/viaf.transcript

  status:
    foreach:
      - az-ratings
      - bx-ratings
      - gr-authors
      - gr-book-genres
      - gr-books
      - gr-book-series
      - gr-interactions
      - gr-works
      - loc-mds-books
      - loc-mds-names
      - ol-authors
      - ol-editions
      - ol-works
      - viaf
    do:
      cmd: python ../run.py stage-status -o ${item}.status ${item}
      always_changed: true
      outs:
      - ${item}.status
      deps:
      - ${item}.transcript