Register
Login
Resources
Docs Blog Datasets Glossary Case Studies Tutorials & Webinars
Product
Data Engine LLMs Platform Enterprise
Pricing Explore
Connect to our Discord channel

stage_03_prepare_callback.py 1.7 KB

You have to be logged in to leave a comment. Sign In
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
  1. import argparse
  2. import os
  3. import logging
  4. import time
  5. from src.utils.common import read_yaml_file, create_directories
  6. from src.utils.callbacks import create_and_save_tensorboard_callback,create_and_save_checkpointing_callback
  7. logging.basicConfig(
  8. filename=os.path.join("logs", 'running_logs.log'),
  9. level=logging.INFO,
  10. format="[%(asctime)s: %(levelname)s: %(module)s]: %(message)s",
  11. filemode="a"
  12. )
  13. def prepare_callbacks(config_path: str) -> None:
  14. """prepare and save callbacks as binary
  15. Args:
  16. config_path (str): path to configuration file
  17. """
  18. config = read_yaml_file(config_path)
  19. artifacts = config["artifacts"]
  20. artifacts_dir = artifacts["ARTIFACTS_DIR"]
  21. tensorboard_log_dir = os.path.join(artifacts_dir, artifacts["TENSORBOARD_ROOT_LOG_DIR"])
  22. checkpoint_dir = os.path.join(artifacts_dir, artifacts["CHECKPOINT_DIR"])
  23. callbacks_dir = os.path.join(artifacts_dir, artifacts["CALLBACKS_DIR"])
  24. create_directories([
  25. tensorboard_log_dir,
  26. checkpoint_dir,
  27. callbacks_dir
  28. ])
  29. create_and_save_tensorboard_callback(callbacks_dir, tensorboard_log_dir)
  30. create_and_save_checkpointing_callback(callbacks_dir, checkpoint_dir)
  31. if __name__ == '__main__':
  32. args = argparse.ArgumentParser()
  33. args.add_argument("--config", "-c", default="configs/config.yaml")
  34. parsed_args = args.parse_args()
  35. try:
  36. logging.info("\n********************")
  37. logging.info(">>>>> stage three started <<<<<")
  38. prepare_callbacks(config_path=parsed_args.config)
  39. logging.info(">>>>> stage three completed! callbacks are prepared and saved as binary <<<<<n")
  40. except Exception as e:
  41. logging.exception(e)
  42. raise e
Tip!

Press p or to see the previous file or, n or to see the next file

Comments

Loading...