Bookdata-tools/viaf/dvc.lock

schema: '2.0'
stages:
  scan-authors:
    cmd: cargo run --release -- scan-marc -L -o viaf.parquet ../data/viaf-clusters-marc21.xml.gz
    deps:
    - path: ../data/viaf-clusters-marc21.xml.gz
      hash: md5
      md5: 62075083cf4ec72bf0ec841e4f46665f
      size: 14163590455
    - path: ../src/cli/scan_marc.rs
      hash: md5
      md5: 0663aa5a5d2fe2a3c2fdb505170a5cc2
      size: 3934
    - path: ../src/marc
      hash: md5
      md5: 874e3a2ea08a2d41e3c54b9d7a2032c1.dir
      size: 22963
      nfiles: 5
    outs:
    - path: viaf.parquet
      hash: md5
      md5: 9e58fa2d32e55dd94328917aef3c5f16
      size: 15013721675
  author-names:
    cmd: python ../run.py --rust fusion author-names.tcl
    deps:
    - path: author-names.tcl
      md5: d0ebf76118c885cd85ce01ddc87cdb64
      size: 223
    - path: viaf.parquet
      md5: 8c93292c182dcfba2a5960eac57c6faa
      size: 12519632430
    outs:
    - path: author-names.csv.gz
      md5: 3823edd34fc084a9f8cef5cd8ec43da2
      size: 494355416
  author-genders:
    cmd: cargo run --release -- filter-marc --tag=375 --subfield=a --trim --lower
      -n gender -o author-genders.parquet viaf.parquet
    deps:
    - path: ../src/cli/filter_marc.rs
      hash: md5
      md5: 87ea27014ee2006924cc041cbfbcfa1c
      size: 5682
    - path: viaf.parquet
      hash: md5
      md5: 9e58fa2d32e55dd94328917aef3c5f16
      size: 15013721675
    outs:
    - path: author-genders.parquet
      hash: md5
      md5: e5398fd1755b812bf8b43551f56994d7
      size: 134925962
  index-names:
    cmd: cargo run --release -- index-names --marc-authorities viaf.parquet author-name-index.parquet
    deps:
    - path: ../src/cleaning/names
      md5: c5591171b9fef6a491e33b4b7f52ba2c.dir
      size: 10362
      nfiles: 5
    - path: ../src/cli/index_names.rs
      hash: md5
      md5: 3ea83851695cebe5cb78faab379e89ad
      size: 4067
    - path: viaf.parquet
      hash: md5
      md5: 9e58fa2d32e55dd94328917aef3c5f16
      size: 15013721675
    outs:
    - path: author-name-index.csv.gz
      hash: md5
      md5: e224a3148eeb0e2cf5fed19e1d9fd966
      size: 859649637
    - path: author-name-index.parquet
      hash: md5
      md5: 382caf194f20c9f9ba74317c05ae4083
      size: 565011474
  schema@author-name-index:
    cmd: python ../run.py --rust pq-info -o author-name-index.json author-name-index.parquet
    deps:
    - path: author-name-index.parquet
      md5: 28bdb3cdaf4f193087e566662a880b22
      size: 515538267
    outs:
    - path: author-name-index.json
      md5: fa29532090232226cf30bd5c53b2566a
      size: 247
  schema@viaf:
    cmd: python ../run.py --rust pq-info -o viaf.json viaf.parquet
    deps:
    - path: viaf.parquet
      md5: 3b436c52b269e5ec33af33a24c1af1c0
      size: 11540970920
    outs:
    - path: viaf.json
      md5: 5cd18bf6344942be879199d5a1f393a7
      size: 695
  schema@author-genders:
    cmd: python ../run.py --rust pq-info -o author-genders.json author-genders.parquet
    deps:
    - path: author-genders.parquet
      md5: 505a5a8f30082f297cb79995435a82c3
      size: 115121052
    outs:
    - path: author-genders.json
      md5: ef9b90520bd446297359938de27a9de1
      size: 249