Dean
/
Bookdata-tools
mirror of https://github.com/BoiseState/bookdata-tools


  
1

	
2

	
3

	
4

	
5

	
6

	
7

	
8

	
9

	
10

	
11

	
12

	
13

	
14

	
15

	
16

	
17

	
18

	
19

	
20

	
21

	
22

	
23

	
24

	
25

	
26

	
27

	
28

	
29

	
30

	
31

	
32

	
33

	
34

	
35

	
36

	
37

	
38

	
39

	
40

	
41

	
42

	
43

	
44

	
45

	
46

	
47

	
48

	
49

	
50

	
51

	
52

	
53

	
54

	
55

	
56

	
57

	
58

	
            schema: '2.0'
stages:
  BX:
    cmd: unzip BX-CSV-Dump.zip
    deps:
    - path: BX-CSV-Dump.zip
      md5: 37d647ee9e18ba134ea6d78ee4fe5292
    outs:
    - path: BX-Book-Ratings.csv
      md5: b34fe0534c9b846b8a45f316c60eb92b
    - path: BX-Books.csv
      md5: d50d59b0c40f10d37d379d9fd3fb98ac
    - path: BX-Users.csv
      md5: 6ec3fe6463da9e149d474ed8226612a2
  loc-books:
    cmd: curl -L https://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2016.part[01-43].xml.gz
      -o loc-books/BooksAll.2016.part#1.xml.gz --create-dirs
    outs:
    - path: loc-books
      md5: 1b1e7ab1d98cc81e373dfc53345d4bb7.dir
      size: 3129774145
      nfiles: 43
  loc-names:
    cmd: curl -L https://www.loc.gov/cds/downloads/MDSConnect/Names.2016.part[01-40].xml.gz
      -o loc-names/Names.2016.part#1.xml.gz --create-dirs
    outs:
    - path: loc-names
      md5: fc488a8775561070cced774803fe0d72.dir
      size: 1410755359
      nfiles: 40
  ol-authors:
    cmd: curl -L --retry 100 -o openlib/ol_dump_authors.txt.gz https://openlibrary.org/data/ol_dump_authors_2023-12-31.txt.gz
    outs:
    - path: openlib/ol_dump_authors.txt.gz
      hash: md5
      md5: d174f253b528f74c0d6446f38feb6b63
      size: 598486891
  ol-editions:
    cmd: curl -L --retry 100 -o openlib/ol_dump_editions.txt.gz https://openlibrary.org/data/ol_dump_editions_2023-12-31.txt.gz
    outs:
    - path: openlib/ol_dump_editions.txt.gz
      hash: md5
      md5: 6b13afe5a50fb1aeeb430bcd01164d13
      size: 10078352001
  ol-works:
    cmd: curl -L --retry 100 -o openlib/ol_dump_works.txt.gz https://openlibrary.org/data/ol_dump_works_2023-12-31.txt.gz
    outs:
    - path: openlib/ol_dump_works.txt.gz
      hash: md5
      md5: 0d96507a98acf4dcc5416b10d85b1225
      size: 3200947685
  viaf-clusters:
    cmd: curl -L --retry 100 -o viaf-clusters-marc21.xml.gz https://viaf.org/viaf/data/viaf-20240101-clusters-marc21.xml.gz
    outs:
    - path: viaf-clusters-marc21.xml.gz
      hash: md5
      md5: 62075083cf4ec72bf0ec841e4f46665f
      size: 14163590455