22_graph_test_2/dvc.yaml

stages:
  etl_instacart:
    cmd: python src/etl_instacart.py data/raw/order_products__prior.csv data/raw/products.csv
    deps:
    - data/raw/order_products__prior.csv
    - data/raw/products.csv
    - src/etl_instacart.py
    - src/tools/etl_tools.py
    outs:
    - data/input/instacart_transformed.csv
  etl_online_retail_ii:
    cmd: python src/etl_online_retail_ii.py data/raw/online_retail_II.xlsx
    deps:
    - data/raw/online_retail_II.xlsx
    - src/etl_online_retail_ii.py
    - src/tools/etl_tools.py
    outs:
    - data/input/online_retail_transformed.csv
  product_name_normalization:
    cmd: python src/names_cleaner.py data/input/
    deps:
    - data/input/
    - src/names_cleaner.py
    - src/tools/preprocess_tools.py
    outs:
    - data/input_names_normalized/
  prepare_embeddings_sets:
    cmd: python src/prepare_embeddings_sets.py -o data/computed_embeddings_sets -p
      params.yaml
    deps:
    - src/prepare_embeddings_sets.py
    params:
    - eimp.embeddings.dataset_size
    - eimp.embeddings.train_size
    - eimp.embeddings.vector_size
    outs:
    - data/computed_embeddings_sets/test/
    - data/computed_embeddings_sets/train/
  prepare_eipt_params:
    cmd: python src/prepare_eipt_params.py -p params.yaml -o data/eipt/testing_params
    deps:
    - src/prepare_eipt_params.py
    params:
    - eimp.model
    outs:
    - data/eipt/testing_params
  fullscan_eipt_index_search:
    cmd: python src/fullscan_eipt_index_searcher.py
    deps:
    - data/computed_embeddings_sets/
    - src/fullscan_eipt_index_searcher.py
    params:
    - eimp.search_params
    outs:
    - data/eipt/metrics/by_model/fullscan/
    - data/eipt/recommendations/fullscan/
  annoy_eipt_index_embeddings:
    cmd: python src/eipt_annoy_embeddings_indexer.py
    deps:
    - data/computed_embeddings_sets/train
    - data/eipt/testing_params/annoy_grid.csv
    - src/eipt_annoy_embeddings_indexer.py
    params:
    - eimp.model.annoy
    - eimp.search_params
    outs:
    - data/eipt/indexer_models/annoy/
    - data/eipt/metrics/by_model/annoy/train.csv
  annoy_eipt_index_search:
    cmd: python src/annoy_eipt_index_searcher.py
    deps:
    - data/computed_embeddings_sets/test/
    - data/eipt/indexer_models/annoy/
    - src/annoy_eipt_index_searcher.py
    params:
    - eimp.model.annoy
    - eimp.search_params
    outs:
    - data/eipt/metrics/by_model/annoy/test.csv
    - data/eipt/recommendations/annoy/
  eipt_analysis:
    cmd: python src/eipt_analysis.py -p params.yaml -o data/eipt/metrics/ -m data/eipt/metrics/by_model/
      -r data/eipt/recommendations -g data/eipt/testing_params/ -t data/eipt/template.md
      -d embedding_indexer_performance_testing.md
    deps:
    - data/eipt/metrics/by_model/
    - data/eipt/template.md
    - data/eipt/testing_params/
    - src/eipt_analysis.py
    params:
    - eimp
    outs:
    - data/eipt/metrics/graphs/
    - data/eipt/metrics/test_metrics_summary.csv
    - data/eipt/metrics/train_metrics_summary.csv
    - embedding_indexer_performance_testing.md
  drop_products_by_name:
    cmd: python3 src/products_removers.py data/input_names_normalized/
    deps:
    - data/input_names_normalized/
    - src/products_removers.py
    - src/tools/preprocess_tools.py
    outs:
    - data/input_preprocessed/
  split_into_subsets_all_data:
    cmd: python3 src/train_test_split.py 0.8 data/input_preprocessed/
    deps:
    - data/input_preprocessed/
    - src/tools/preprocess_tools.py
    - src/train_test_split.py
    outs:
    - data/test_data/
    - data/train_data/
  preprocess_test_add_dummy_orders:
    cmd: python src/prepare_dummy_orders.py data/test_data data/test_final
    deps:
    - data/test_data
    - src/prepare_dummy_orders.py
    - src/tools/preprocess_tools.py
    outs:
    - data/test_final
  preprocess_train_drop_by_size:
    cmd: python src/drop_by_order_size.py data/train_data/ data/train_final/
    deps:
    - data/train_data/
    - src/tools/preprocess_tools.py
    params:
    - order_size_train.max
    - order_size_train.min
    outs:
    - data/train_final/
  input_data_for_random_model:
    cmd: python src/prepare_train_for_random_model.py data/input_preprocessed/ data/train_for_random/
    deps:
    - data/input_preprocessed/
    - src/prepare_train_for_random_model.py
    - src/tools/preprocess_tools.py
    outs:
    - data/train_for_random/

  input_data_for_embedding_model:
    cmd: python src/prepare_train_for_embedding_model.py data/train_final/ data/train_for_embeddings/
    deps:
    - data/train_final/
    - src/prepare_train_for_embedding_model.py
    - src/tools/preprocess_tools.py
    outs:
    - data/train_for_embeddings/
  train_basket_tfidf_perceptron:
    cmd: python src/models/trainer_basket_tfidf_perceptron_embedding.py data/train_for_embeddings/
      data/embedding_models/basket_tfidf_perceptron/
    deps:
    - data/train_for_embeddings/
    - src/models/dataset_iterators.py
    - src/models/neural_network_embedding.py
    - src/models/trainer_basket_tfidf_perceptron_embedding.py
    params:
    - basket_tfidf_perceptron_model.params_grid
    - basket_tfidf_perceptron_model.random_seed
    outs:
    - data/embedding_models/basket_tfidf_perceptron/
  train_random_model:
    cmd: python src/trainer_random_model.py data/train_for_random/ data/random_models/
    deps:
    - data/train_for_random/
    - src/models/trainer_random_model.py
    - src/trainer_random_model.py
    params:
    - random_model.random_seed
    outs:
    - data/random_models/
  build_products_registry:
    cmd: python src/generate_products_registry.py data/input_preprocessed/ data/products_registry/
    deps:
    - data/input_preprocessed/
    - src/generate_products_registry.py
    outs:
    - data/products_registry/