ML-Monitoring/generate_report.py

import datetime as dt
import warnings
import ast
from datetime import datetime

import configparser
import dagshub
from dagshub.upload import Repo
import mlflow
import papermill as pm
from handler import load_data, get_experiment_id
from sklearn import ensemble
import streamlit as st
import os 
import sys

from evidently import ColumnMapping
from evidently.metric_preset import DataDriftPreset, TargetDriftPreset
from evidently.metrics import *
from evidently.report import Report

from evidently.test_preset import DataDriftTestPreset
from evidently.test_suite import TestSuite
from dagshub.streaming import install_hooks

warnings.filterwarnings("ignore")
warnings.simplefilter("ignore")

try:
    install_hooks()
except:
    print("Hooks Loaded")

TARGET = "Close"
PREDICTION = "prediction"


REPO_NAME = " "
USER_NAME = " "

from handler import load_data
st.title('DATA REPORTS')

def setup():
    global REPO_NAME
    global USER_NAME
    config = configparser.ConfigParser()
    config.read('config.ini')
    os.environ['DAGSHUB_TOKEN'] = dagshub.auth.get_token()
    REPO_NAME = config.get('dagshub','repo_name')
    USER_NAME = config.get('dagshub','user_name')
    dagshub.init(REPO_NAME,USER_NAME)
    return config

def generate_model(data, save_local=False):
    numerical_features = ['Open', 'High', 'Low', 'Adj Close', 'Volume']

    reference = data.loc['2017-02-09':'2021-02-09']
    current = data.loc['2021-02-10':'2022-02-07']

    column_mapping = ColumnMapping()
    column_mapping.target = TARGET
    column_mapping.prediction = PREDICTION
    column_mapping.numerical_features = numerical_features

    experiment_id = get_experiment_id('evidently')
    with mlflow.start_run(experiment_id=experiment_id):
        
        regressor = ensemble.RandomForestRegressor(random_state=0, n_estimators=50)
        regressor.fit(reference[numerical_features], reference[TARGET])
        mlflow.sklearn.log_model(regressor, 'models')
        if save_local:
            mlflow.sklearn.save_model(regressor,'model')
            repo = Repo(USER_NAME, REPO_NAME)  
            repo.upload(local_path="model", remote_path="model", versioning="dvc")

   
def load_app(columns):
    print('Loading App....')
    # config_file = st.form('User Details')
    with st.form(key='User Details'):
        user_name = st.text_input('*DagsHub User Name',placeholder="[Required] JaneDoe")
        repo_name = st.text_input('*DagsHub Repo Name', placeholder="[Required] Sample-Repo")
        model_uri = st.text_input("Model URI",placeholder="runs:/000000000000000/model")
        col = st.multiselect('*Pick a column', columns)
        submit = st.form_submit_button('Submit')
    
        if submit:
            st.subheader(f'Writing {user_name}\'s report to {repo_name}')

            config_object = configparser.ConfigParser()
            config_object["dagshub"]={"user_name":user_name,"repo_name":repo_name}
            config_object["mlflow"]={"model_uri":model_uri}
            config_object["data"]={"columns":col}
            with open("config.ini","w") as file_object:
                config_object.write(file_object)
        else:
            st.stop()
    
def load_model(data, model_uri='./model'):
    
    current, reference, mapping= data
    if not model_uri:
        model_uri = './model'
    regressor = mlflow.pyfunc.load_model(model_uri=model_uri)
    reference["prediction"] = regressor.predict(reference[mapping.numerical_features])
    current["prediction"] = regressor.predict(current[mapping.numerical_features])

    return current, reference, mapping

def create_data():
    config = setup()
    data = load_data()
    numerical_features = ast.literal_eval(config.get('data','columns'))
    if TARGET in numerical_features:
        numerical_features.remove(TARGET)
    reference = data.loc['2017-02-09':'2021-02-09']
    current = data.loc['2021-02-10':'2022-02-07']

    column_mapping = ColumnMapping()
    column_mapping.target = TARGET
    column_mapping.prediction = PREDICTION
    column_mapping.numerical_features = numerical_features

    model_uri = config.get('mlflow','model_uri')
    current, reference,column_mapping = load_model((current,reference,column_mapping), model_uri)
    return current, reference, column_mapping

def data_drift_report(data):
    current, reference, _ = data
    report = Report(metrics=[
        DataDriftPreset(),
        ])

    report.run(reference_data=reference, current_data=current)
    return report

def create_report(data, columns=None):
    current, reference, column_mapping = data
    if columns is None:
        columns = reference.columns
    
    values = [RegressionQualityMetric(), DatasetSummaryMetric(), DatasetDriftMetric()]
    for column in columns:
        values.append(ColumnDriftMetric(column_name=column, stattest="wasserstein"),)
    
    values.extend([ColumnSummaryMetric(column_name=TARGET),ColumnSummaryMetric(column_name=PREDICTION)])
    
    data_drift_report = Report(metrics=values)

    data_drift_report.set_batch_size("monthly")

    data_drift_report.run(
        reference_data=reference,
        current_data=current,
        column_mapping=column_mapping,
        )
    return data_drift_report

def create_test_suite(data):
    current, reference, column_mapping = data
    data_drift_test_suite = TestSuite(
        tests=[DataDriftTestPreset()]
    )

    data_drift_test_suite.run(
        reference_data=reference,
        current_data=current,
        column_mapping=column_mapping,
    )
    return data_drift_test_suite

def save_report():
    
    now = datetime.now()
    date_time = str(now.strftime("%d-%m-%Y_%H-%M-%S"))

    output_path = f'./Data_reports/{date_time}.ipynb'

    pm.execute_notebook(
    './data_report.ipynb',
    output_path
    )

    repo = Repo(USER_NAME, REPO_NAME)  
    repo.upload(local_path=output_path, remote_path=output_path, versioning="git")


if __name__ == "__main__":

    # Load Data
    data = load_data()

    # Create a model
    #### NOTE: Fill in the config.ini manually if its empty ####
    # config = setup()
    # generate_model(data,True)

    # Load app to get config
    load_app(data.columns)

    # Create Report
    config = setup()
    save_report()