Dean
/
Bookdata-tools
mirror of https://github.com/BoiseState/bookdata-tools


  
1

	
2

	
3

	
4

	
5

	
6

	
7

	
8

	
9

	
10

	
11

	
12

	
13

	
14

	
15

	
16

	
17

	
18

	
19

	
20

	
21

	
22

	
23

	
24

	
25

	
26

	
27

	
28

	
29

	
30

	
31

	
32

	
33

	
34

	
35

	
36

	
37

	
38

	
39

	
40

	
41

	
42

	
43

	
44

	
45

	
46

	
47

	
48

	
49

	
50

	
51

	
52

	
53

	
54

	
55

	
56

	
57

	
58

	
59

	
60

	
61

	
62

	
63

	
64

	
65

	
66

	
67

	
68

	
69

	
70

	
71

	
72

	
73

	
74

	
75

	
76

	
77

	
78

	
79

	
80

	
81

	
82

	
            stages:
  scan-authors:
    cmd: python ../run.py --rust openlib scan-authors ../data/openlib/ol_dump_authors.txt.gz
    deps:
    - ../src/cli/openlib.rs
    - ../src/openlib/
    - ../data/openlib/ol_dump_authors.txt.gz
    outs:
    - authors.parquet
    - author-names.parquet

  scan-works:
    cmd: python ../run.py --rust openlib scan-works ../data/openlib/ol_dump_works.txt.gz
    deps:
    - ../src/cli/openlib.rs
    - ../src/openlib/
    - ../data/openlib/ol_dump_works.txt.gz
    - authors.parquet
    outs:
    - works.parquet
    - work-authors.parquet
    - work-subjects.parquet
    - author-ids-after-works.parquet

  scan-editions:
    cmd: python ../run.py --rust openlib scan-editions ../data/openlib/ol_dump_editions.txt.gz
    deps:
    - ../src/cli/openlib.rs
    - ../src/openlib/
    - ../data/openlib/ol_dump_editions.txt.gz
    - authors.parquet
    - works.parquet
    - author-ids-after-works.parquet
    outs:
    - editions.parquet
    - edition-works.parquet
    - edition-isbns.parquet
    - edition-authors.parquet
    - edition-subjects.parquet
    - all-works.parquet
    - all-authors.parquet

  edition-isbn-ids:
    cmd: python run.py --rust link-isbn-ids -R edition -o openlibrary/edition-isbn-ids.parquet openlibrary/edition-isbns.parquet
    wdir: ..
    deps:
    - openlibrary/edition-isbns.parquet
    - book-links/all-isbns.parquet
    outs:
    - openlibrary/edition-isbn-ids.parquet

  work-clusters:
    cmd: python run.py --rust cluster extract-books -n work_id -o openlibrary/work-clusters.parquet OL-W
    wdir: ..
    deps:
    - book-links/cluster-graph-nodes.parquet
    outs:
    - openlibrary/work-clusters.parquet

  schema:
    foreach:
    - authors
    - author-names
    - works
    - work-authors
    - work-subjects
    - work-clusters
    - editions
    - edition-works
    - edition-isbns
    - edition-authors
    - edition-subjects
    - all-works
    - all-authors
    - edition-isbn-ids
    do:
      cmd: python ../run.py --rust pq-info -o ${item}.json ${item}.parquet
      deps:
      - ${item}.parquet
      outs:
      - ${item}.json:
          cache: false