Bookdata-tools/loc-mds/dvc.lock

schema: '2.0'
stages:
  scan-books:
    cmd: cargo run --release -- scan-marc --book-mode --glob "../data/loc-books/BooksAll.2016*.xml.gz"
    deps:
    - path: ../data/loc-books
      md5: 1b1e7ab1d98cc81e373dfc53345d4bb7.dir
      size: 3129774145
      nfiles: 43
    - path: ../src/cli/scan_marc.rs
      hash: md5
      md5: 0663aa5a5d2fe2a3c2fdb505170a5cc2
      size: 3934
    - path: ../src/marc
      hash: md5
      md5: 874e3a2ea08a2d41e3c54b9d7a2032c1.dir
      size: 22963
      nfiles: 5
    outs:
    - path: book-authors.parquet
      hash: md5
      md5: 0994ad02a21d496e6ff14c7001147f67
      size: 88233101
    - path: book-fields.parquet
      hash: md5
      md5: 472830445ffa794290a6f1f519af761a
      size: 3066979138
    - path: book-ids.parquet
      hash: md5
      md5: c2651736c069b5836563de8028815deb
      size: 71130687
    - path: book-isbns.parquet
      hash: md5
      md5: a954fd606bfcc6ff9db66af964dd2239
      size: 57538194
  book-authors:
    cmd: python ../run.py --rust fusion book-authors.tcl
    deps:
    - path: book-authors.tcl
      md5: 56396f59e4ea77683b2af8e74d205a4f
      size: 250
    - path: book-fields.parquet
      md5: b305bde2a13f77c0cb62ed586574480c
      size: 3094731732
    outs:
    - path: book-authors.parquet
      md5: c46c8998dfdfd25b91c88b225a436466
      size: 100853758
  scan-names:
    cmd: cargo run --release -- scan-marc --glob "../data/loc-names/Names.2016*.xml.gz"
      -o name-fields.parquet
    deps:
    - path: ../data/loc-names
      md5: fc488a8775561070cced774803fe0d72.dir
      size: 1410755359
      nfiles: 40
    - path: ../src/cli/scan_marc.rs
      hash: md5
      md5: 0663aa5a5d2fe2a3c2fdb505170a5cc2
      size: 3934
    - path: ../src/marc
      hash: md5
      md5: 874e3a2ea08a2d41e3c54b9d7a2032c1.dir
      size: 22963
      nfiles: 5
    outs:
    - path: name-fields.parquet
      hash: md5
      md5: 35b9abf74ec583ac01f226478958bd99
      size: 1574481968
  author-names:
    cmd: python ../run.py --rust fusion author-names.tcl
    deps:
    - path: author-names.tcl
      md5: 29c68172aa2a10d6054b73656c21567c
      size: 193
    - path: name-fields.parquet
      md5: 800b068931bb1a2cba59b85b3ebf471d
      size: 1543059992
    outs:
    - path: author-names.csv.gz
      md5: 9e68c20496a2f62922334701f9e1492f
      size: 87125358
  author-genders:
    cmd: python ../run.py --rust fusion author-genders.tcl
    deps:
    - path: author-genders.tcl
      md5: 58abbe4a110eb782450d9db0e461e087
      size: 197
    - path: name-fields.parquet
      md5: 800b068931bb1a2cba59b85b3ebf471d
      size: 1543059992
    outs:
    - path: author-genders.csv.gz
      md5: 71a6ef56b674c032c5a572d478e66762
      size: 1469682
  book-isbn-ids:
    cmd: cargo run --release -- link-isbn-ids -R rec_id -o loc-mds/book-isbn-ids.parquet
      loc-mds/book-isbns.parquet
    deps:
    - path: book-links/all-isbns.parquet
      hash: md5
      md5: 8803c162ab97efac8b098df7e9252314
      size: 464207536
    - path: loc-mds/book-isbns.parquet
      hash: md5
      md5: a954fd606bfcc6ff9db66af964dd2239
      size: 57538194
    outs:
    - path: loc-mds/book-isbn-ids.parquet
      hash: md5
      md5: 719c232a3bc19f0510607c31f024b1db
      size: 39838250
  loc-clusters:
    cmd: python ../run.py --rust fusion loc-clusters.tcl
    deps:
    - path: ../book-links/cluster-graph-nodes.parquet
      md5: 6d0ebe8ef88733428ac88a222f959b96
      size: 798139218
    - path: loc-clusters.tcl
      md5: 31716e72f6d58f047ea391109be5d806
      size: 221
    outs:
    - path: loc-clusters.parquet
      md5: 0748a8001c63142ebda3d099b6ebd84c
      size: 35754914
  schema@book-fields:
    cmd: python ../run.py --rust pq-info -o book-fields.json book-fields.parquet
    deps:
    - path: book-fields.parquet
      md5: 426ffd30f8659a5b7615dfab372b3ba0
      size: 2763440545
    outs:
    - path: book-fields.json
      md5: f5f037b02b62afd7df6194dee5cd86e5
      size: 693
  schema@book-isbn-ids:
    cmd: python ../run.py --rust pq-info -o book-isbn-ids.json book-isbn-ids.parquet
    deps:
    - path: book-isbn-ids.parquet
      md5: 1022f7b00a26899573dd319637cb098e
      size: 34297513
    outs:
    - path: book-isbn-ids.json
      md5: 71bb49d90e205b89033d2d951f2288ff
      size: 249
  schema@book-ids:
    cmd: python ../run.py --rust pq-info -o book-ids.json book-ids.parquet
    deps:
    - path: book-ids.parquet
      md5: b8e4eb3ee0b6330bcd655a64b1fa89e6
      size: 65210771
    outs:
    - path: book-ids.json
      md5: 82a4c2b33bde2b1835d3786818928495
      size: 605
  schema@book-authors:
    cmd: python ../run.py --rust pq-info -o book-authors.json book-authors.parquet
    deps:
    - path: book-authors.parquet
      md5: 415b0a79bb7bc1c7c6651eaccf0d9d30
      size: 85290815
    outs:
    - path: book-authors.json
      md5: 4f6d1f44d92cd6e4efe52a095097d92e
      size: 252
  schema@book-isbns:
    cmd: python ../run.py --rust pq-info -o book-isbns.json book-isbns.parquet
    deps:
    - path: book-isbns.parquet
      md5: ca03e6c54a490743746f2bc2a73ad3f0
      size: 55680857
    outs:
    - path: book-isbns.json
      md5: e37c1c973ddd9106deba98d15bba761b
      size: 329
  schema@loc-clusters:
    cmd: python ../run.py --rust pq-info -o loc-clusters.json loc-clusters.parquet
    deps:
    - path: loc-clusters.parquet
      md5: 0748a8001c63142ebda3d099b6ebd84c
      size: 35754914
    outs:
    - path: loc-clusters.json
      md5: a9217be376ffffe877ee4f3a2f0dcbb9
      size: 352
  schema@name-fields:
    cmd: python ../run.py --rust pq-info -o name-fields.json name-fields.parquet
    deps:
    - path: name-fields.parquet
      md5: fb0b961cc7a916930e10949d8da5a0df
      size: 1422606012
    outs:
    - path: name-fields.json
      md5: b1397f21e19ad39c1014f823091116d9
      size: 693