Register
Login
Resources
Docs Blog Datasets Glossary Case Studies Tutorials & Webinars
Product
Data Engine LLMs Platform Enterprise
Pricing Explore
Connect to our Discord channel

Makefile 3.6 KB

You have to be logged in to leave a comment. Sign In
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
  1. .PHONY: clean dirs virtualenv lint requirements push pull reproduce
  2. #################################################################################
  3. # GLOBALS #
  4. #################################################################################
  5. PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
  6. #$(info $$PROJECT_DIR is [${PROJECT_DIR}])
  7. #PYTHON_INTERPRETER = python3
  8. PYTHON_INTERPRETER = python
  9. #################################################################################
  10. # COMMANDS #
  11. #################################################################################
  12. ## Create virtualenv.
  13. ## Activate with the command:
  14. ## source env/bin/activate
  15. virtualenv:
  16. $(PYTHON_INTERPRETER) -m venv env
  17. $(info "Activate with the command 'source env/bin/activate'")
  18. ## Install Python Dependencies.
  19. ## Make sure you activate the virtualenv first!
  20. requirements:
  21. $(PYTHON_INTERPRETER) -m pip install -U pip setuptools wheel
  22. $(PYTHON_INTERPRETER) -m pip install -r requirements.txt
  23. ## Create directories that are ignored by git but required for the project
  24. dirs:
  25. # Windows
  26. mkdir \data\raw data\raw\nasa data\raw\siap data\raw\sniim data\interim data\processed models logs
  27. ## Make dataset from interim to processed
  28. dataset:
  29. $(PYTHON_INTERPRETER) src\data\make_dataset.py data\interim data\processed
  30. ## Delete all compiled Python files
  31. clean:
  32. find . -type f -name "*.py[co]" -delete
  33. find . -type d -name "__pycache__" -delete
  34. ## To use the pre-commit hooks
  35. pre-commit-install:
  36. pre-commit install
  37. setup-data-validation:
  38. cd src/data; great_expectations -y init; great_expectations datasource new
  39. run-data-validation:
  40. cd src/data; $(PYTHON_INTERPRETER) data_validation.py
  41. ## Lint using flake8
  42. lint:
  43. flake8 src
  44. ## Upload Data to default DVC remote
  45. push:
  46. dvc push -r origin
  47. ## Download Data from default DVC remote
  48. pull:
  49. dvc pull -r origin
  50. ## Reproduce the DVC pipeline - recompute any modified outputs such as processed data or trained models
  51. reproduce:
  52. dvc repro
  53. #################################################################################
  54. # PROJECT RULES #
  55. #################################################################################
  56. #################################################################################
  57. # Self Documenting Commands #
  58. #################################################################################
  59. .DEFAULT_GOAL := reproduce
  60. # Inspired by <http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html>
  61. # sed script explained:
  62. # /^##/:
  63. # * save line in hold space
  64. # * purge line
  65. # * Loop:
  66. # * append newline + line to hold space
  67. # * go to next line
  68. # * if line starts with doc comment, strip comment character off and loop
  69. # * remove target prerequisites
  70. # * append hold space (+ newline) to line
  71. # * replace newline plus comments by `---`
  72. # * print line
  73. # Separate expressions are necessary because labels cannot be delimited by
  74. # semicolon; see <http://stackoverflow.com/a/11799865/1968>
  75. .PHONY: help
  76. help:
  77. @echo "$$(tput bold)Available rules:$$(tput sgr0)"
  78. @echo
  79. @sed -n -e "/^## / Missing" $Missing \
  80. | LC_ALL='C' sort --ignore-case \
  81. | awk -F '---' \
  82. -v ncol=$$(tput cols) \
  83. -v indent=19 \
  84. -v col_on="$$(tput setaf 6)" \
  85. -v col_off="$$(tput sgr0)" \
  86. 'Missing \
  87. printf "%s ", words[i]; \
  88. } \
  89. printf "\n"; \
  90. }' \
  91. | more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars')
Tip!

Press p or to see the previous file or, n or to see the next file

Comments

Loading...