# import libraries
import numpy as np 
import matplotlib.pyplot as plt
from bs4 import BeautifulSoup as soup
from urllib.request import urlopen as uReq
import pandas as pd
import unidecode
import seaborn as sns
from scipy import stats
import math
import warnings

# set sns color code
sns.set(color_codes=True)

# suprress warnings, I'm doing this so the notebook is more readable
warnings.filterwarnings("ignore")

# main functions used for scraping the premier league website

def remove_special_characters(name):
    """This function removes numbers, apostrophes, and addition signs from a string and returns the cleaned string. 
    This was specifically written with the format of name strings from the premier league match summaries in mind. 
    Strings of the form "john doe 90 + 4' " are returned as "john doe".
    
    :param str name: The full string of text containing the player name.
    :return: A string containing the name with the unnecessary characters stripped.
    :rtype: str 
    
    """
    for c in ['0','1','2','3','4','5','6','7','8','9','+',"'"]:
        name = name.replace(c,'')
    
    # remove trailing spaces
    name.strip()
    
    return name



def remove_team_code(name):
    """This function removes team codes from a string containing names+codes.
    The function is limited in that it only does this for teams that were involved in the premier league
    in the 2017/18 and 2018/19 seasons. 
    
    :param str name: A team name of the form name+code.
    :return: A string containing the team name with the codes removed i.e BurnleyBUR returns Burnley.
    :rtype: str
    
    """
    # team codes
    team_codes = ["BUR","BOU","CRY","WBA","HUD","ARS","LIV","BHA","MUN","WAT","NEW","CHE","SOU",\
                  "MCI","SWA","STK","TOT","LEI","WHU","EVE","WOL","CAR","FUL"]
    
    # look for codes in name, if found, remove
    for c in team_codes:
        name = name.replace(c,'')
        
    return name



def extract_epl_lineups(team_container, home):
    """This function extract lineups information from the matchLineupTeamContainer class 
    as defined on the the premier league results website. 
    
    :param Tag team_container: Contains the matchLineupTeamContainer from the premier league website.
    :param bool home: A flag for whether team_conatiner contains the home team lineup or away team lineup
    :return:  A dictionary with the keys = [ "player1", ...., "player 14"], and the values are [{"name":"John Doe"}....].
             If a team plays with less than 14 players in a game, the remaining slots are left blank (= '').
    :rtype: dict
    
    """
    
    # generate keys for return dict, initialize player names
    if home== True:
        base_string = "home_player"
    else: 
        base_string = "away_player"
    keys = [base_string+str(i+1) for i in range(14)]
    player_names = []

    # extract info from the container
    lineup_container = team_container.find_all("div",{"class":"info"})
    
    # extract player names
    for i in range(len(lineup_container)):
        if i < 11:
            player_names.append(unidecode.unidecode(remove_special_characters(lineup_container[i].div.text)))
        else:
            if lineup_container[i].div.div.text != '':
                player_names.append(unidecode.unidecode(remove_special_characters(lineup_container[i].div.text)))
    
    # create dictionary 
    player_dict = dict(zip(keys,player_names))
    
    # if player12, player13, or player14 don't exits, create and and assign '' as value
    for i in range(12,15):
        if base_string+str(i) not in player_dict:
            player_dict[base_string+str(i)]= ""
            
    return player_dict




def extract_epl_match_details(match_ids):
    """This function extracts match information for games from the premier league results website
    given their match ids. 
   
    :param list match_ids: A list of match ids for the games you're interested in. Can be a list of one or empty. 
    :return: A dictionary containing the match information for every match id. Each unique match
            match id is a key. The value is the corresponding match information which is a list 
            five dictionaries the team names, score, result (encoded as 1 for a home team win, 
            0 for a draw, -1 for an away team win), the home team lineup,
            and the away team line up. The dictionaries are formatted as follows:
            team_names = {"home_team":"Arsenal", "away_team":"Liverpool"}
            score = {"home_goals":2, "away_goals":3}
            result = {"result": 1}
            home_team = {"home_player1":"Petr Cech", ....., "home_player14": "Alex Iwobi"}
            away_team = {"away_player1":"Loris Karius", ...., "away_player14":""}
    :rtype: dict
    
    """
     
    # where the match data is being scraped from
    epl_url = "https://www.premierleague.com/match/"

    # initialize a dictionary to store the results 
    data = dict()

    # this loop extracts the data through each game in turn
    for match_id in match_ids:

        # url for each math
        match_url = epl_url + str(match_id)

        # download page, call and then close client
        uClient = uReq(match_url)
        results_page = uClient.read()
        uClient.close()

        # extract html
        results_soup = soup(results_page, "html.parser")

        # extract home team name container
        home_team_container = results_soup.findAll("div", {"class":"team home"})

        # extract away team name container
        away_team_container = results_soup.findAll("div", {"class":"team away"})

        # extract score container
        score_container = results_soup.findAll("div", {"class":"matchScoreContainer"})

         # extract lineup containers 
        team_lineups_container = results_soup.findAll("div", {"class":"matchLineupTeamContainer"})

        # extract team names from their containers
        team_names = {"home_team": remove_team_code(home_team_container[0].text),\
                      "away_team":remove_team_code(away_team_container[0].text)}

        # extract score from score container
        score = {"home_goals": int(((score_container[0].text).split("-"))[0]),\
                 "away_goals":int(((score_container[0].text).split("-"))[1]) }

        # use score to determine match outcome, Win for home team = 1, Win for away team = -1, draw = 0
        result = {"result":  1 if score["home_goals"] > score["away_goals"] else\
                  -1 if score["away_goals"] > score["home_goals"] else 0 }

        # extract lineups for the home and away team from their containers, this includes substitutes who made appearances
        home_team = extract_epl_lineups(team_lineups_container[0], home = True)
        away_team = extract_epl_lineups(team_lineups_container[1], home = False)

        # append to data dictionary
        data[match_id] = [team_names, score, result, home_team, away_team]

    return data

# main functions used for finding fifa ratings 

def last_name(name):
    """Extract the last name for a player given their full name. 
    
    :param str name: A string of the full name of the player.
    :return: Returns just the last name of the player. If their full name
            is just one name - "Ronaldinho" for example - return what is passed.
    :rtype: str
    
    """
    name = unidecode.unidecode(name)
    names = name.split()
    last_name = names[len(names)-1]
    
    return last_name

def extract_fifa_rating(data_main, data_auxillary, name, club):
    """This function extracts the fifa rating for a player.
    
    :param DataFrame data_main: A master data store; this is the data from kaggle.
    :param DataFrame data_auxillary: A supplementary dataframe that conatains data missing in data_main.
    :param str name: A string containing the name of the player missing. 
    :param str club: A string containing the the player's club.
    :return: The base fifa rating (type int) if found, else returns np.NaN.
    :rtype: int
    
    """
    rating = 0 
    
    # check the main dataset to see if the full name matches
    if data_main[data_main["NAME"]== name]["RATING"].empty == False:
        rating = data_main[data_main["NAME"]== name]["RATING"].min()
    
    # check if the last name and club are enough to match 
    elif data_main[data_main["LAST_NAME"]== last_name(name)][fifa_data["CLUB"]==club]["RATING"].empty == False:
        rating = data_main[data_main["LAST_NAME"]== last_name(name)][fifa_data["CLUB"]==club]["RATING"].min()

    elif data_auxillary[data_auxillary["NAME"] == unidecode.unidecode(name)]["RATING"].empty == False:
        rating = data_auxillary[data_auxillary["NAME"] == unidecode.unidecode(name)]["RATING"].min()
    else:
        rating = np.NaN

    return rating
        

# functions to formet the data (list of dictionaries) appropriate as dataframe

def convert_to_dataframe(data, match_ids):
    """This function converts a list called data (see documentation for 
    extract_epl_match_details for format) and match_ids to create dataframe of 33 columns. 
    The column names are enumerated in the second line of the function.
    
    :param dict data: The data for all the games fromatted as a dictionary.
    :param list match_ids: A list of all the match ids. These are the keys used to access the data parameter.
    :return: DataFrame containing 33 columns with every row corresponding to a single game. The indexes correspond
            to match ids.
    :rtype: DataFrame
    
    """
    
    # index and columns used to initialize return dataframe 
    index = [0]
    columns = ["home_team", "away_team", "home_goals", "away_goals","result", \
              "home_player1","home_player2","home_player3","home_player4","home_player5","home_player6","home_player7",\
               "home_player8","home_player9","home_player10","home_player11","home_player12","home_player13","home_player14",\
              "away_player1","away_player2","away_player3","away_player4","away_player5","away_player6","away_player7",\
               "away_player8","away_player9","away_player10","away_player11","away_player12","away_player13","away_player14"]

    # initialize dataframe with one empty row
    data_df = pd.DataFrame(index=index,columns=columns)

    # append match info for each match in turn to return dataframe 
    for match_id in match_ids:
        temp_df = pd.DataFrame.from_dict(data[match_id][0],orient="index")
        for i in [1,2,3,4]:
            temp_df= pd.concat([temp_df,pd.DataFrame.from_dict(data[match_id][i],orient="index")])
        temp_df = temp_df.T
        data_df = data_df.append(temp_df)

    # remove the empty first row
    data_df = data_df.iloc[1:]

    # make the indices equal to the match ids
    data_df.index = match_ids
    
    return data_df

# This code takes between 15-20 minutes to run to run through all the pages.
# I've modified num_matches_2018 and num_matches_2019 appropriately so that doesn't happen.
# I ran it once with full number of matches, saved the results, and uploaded them to GitHub.
# In the next section, I directly read these files from the GitHub links.

##### Extract Match Info #####

# there are 380 matches played in a season, there are 20 teams and each team plays every other team twice. 
num_matches_2018 = 1 #380
num_matches_2019 = 1 #190

# list of match IDs for the games played last season, I found these by reading through the html on the website
match_ids_2018 = [22342+i for i in range(num_matches_2018)]
match_ids_2019 = [38313+i for i in range(num_matches_2019)]

# extract data for each season
data_2018 = extract_epl_match_details(match_ids_2018)
data_2019 = extract_epl_match_details(match_ids_2019)

# combine data to create a master dictionary
data = {**data_2018, **data_2019}

# convert data from dictionary to dataframe
df_2018 = convert_to_dataframe(data_2018,match_ids_2018)
df_2019 = convert_to_dataframe(data_2019,match_ids_2019)


##### Compile Player Names and Ratings #####

# import master FIFA19 file, includes just fifa ratings 
fifa_data = pd.read_csv("https://raw.githubusercontent.com/kafagy/fifa-FUT-Data/master/FIFA19.csv")

# remove special characters from the name column (accented e's etc)
fifa_data["NAME"] = fifa_data["NAME"].transform(unidecode.unidecode)

# add last name column to dataframe to aid querying
fifa_data["LAST_NAME"] = fifa_data["NAME"].transform(last_name)

# import missing fifa data, this is the  missing data file I created
missing_fifa_data = pd.read_csv("https://raw.githubusercontent.com/shahzoor/FIFA_PROJECT/master/missing_player_ratings_final.csv")

# initiaize dictionary to hold player data
name_team_rating_dict = dict()

# extract player name, club, and rating for every player who played any minutes at all in the epl in the last two seasons
for id in (match_ids_2018+match_ids_2019):
    for i in range(1,15):
        if data[id][3]["home_player"+str(i)] != '':
            name = (data[id][3]["home_player"+str(i)]).strip()
            club = remove_team_code(data[id][0]["home_team"])
            rating = extract_fifa_rating(fifa_data, missing_fifa_data, name, club)
            name_team_rating_dict[name] = [club,rating]
    for j in range(1,15):
        if data[id][4]["away_player"+str(j)] != '':
            name = (data[id][4]["away_player"+str(j)]).strip()
            club = remove_team_code(data[id][0]["away_team"])
            rating = extract_fifa_rating(fifa_data, missing_fifa_data, name, club)
            name_team_rating_dict[name] = [club,rating]

# convert player data dictionary to dataframe
player_data = pd.DataFrame(name_team_rating_dict).T
player_data.columns = ["CLUB", "RATING"]


##### Export Data #####

# export dataframes as csv files
df_2018.to_csv("epl_results_2018", sep = ',')
df_2019.to_csv("epl_results_2019", sep = ',')
player_data.to_csv('epl_player_ratings', sep=',')

match_data_2018 = pd.read_csv("https://raw.githubusercontent.com/shahzoor/FIFA_PROJECT/master/epl_results_2018_full.csv")
match_data_2019 = pd.read_csv("https://raw.githubusercontent.com/shahzoor/FIFA_PROJECT/master/epl_results_2019_first_half.csv")
player_ratings = pd.read_csv("https://raw.githubusercontent.com/shahzoor/FIFA_PROJECT/master/epl_player_ratings.csv")

def make_swarmplot(df, xlabel, ylabel, ax, point_size =3, point_color='darkgrey', box_pal="Blues"):
    """Make a swarmplot with a boxplot super imposed given a dataset and two variables to focus on. The ylabel
    must be categorical. 
    
    :param DataFrame df: Dataframe, must have categorical and numeric data.
    :param str xlabel: Label of column with numeric data, must match df.
    :param str ylabel: Label of column with categorical data, must match df.
    :param axes ax: Axis object from mpl. 
    :param dict box_pal: A custome color palette for the box plots.
    :return: The original axis object with the plot imposed on it.
    :rtype: axes
    
    """

    # make box plot for each category/club
    ax = sns.boxplot(y = ylabel, x = xlabel, data = df, whis=np.inf, ax= ax, palette=box_pal, width=0.3)
    
    # make swarm plot for each category/club
    ax = sns.swarmplot(y = ylabel, x = xlabel, data = df, size = point_size, color = point_color, ax = ax, alpha = 0.85)
    
    # remove the top and right line in graph
    sns.despine()
    
    # set size
    ax.figure.set_size_inches(20,14)
    
    # set labels
    ax.set_xlabel((xlabel.lower()).capitalize())
    ax.set_ylabel((ylabel.lower()).capitalize())
    ax.grid(False)
    
    return ax

# these functions compute the rating averages for teams, the differences, and append them to the match data dataframes 

def compute_team_average(names ,ratings ):
    """This function computes the average rating for a list of players. It accounts for any empty
    strings (i.e. '') in the list.
    
    :param: list names : A list of names of the players.
    :param: DataFrame ratings : An appropriately formatted dataframe containing player ratings.
    :return: The average rating of the players passed in.
    :rtype: float
    
    """
    
    #initialize return variable
    average = 0  
    count = 0 
    
    # only count players who actually appeared, ignore empty places and NaN values
    for name in names:
        if type(name) == str:
            average = average + ratings[ratings["NAME"]==name.strip()]["RATING"].min()
            count = count + 1
            
    # average contains the sum of all player ratings, thus divide
    average = average/count
    
    return average


def add_averages_to_df(match_data,ratings):
    """This function appends the following three coloumns to the match data dataframe:
    home_team_rating, away_team_rating, rating_diff. It returns this modified dataframe.
    
    :param: DataFrame match_data: This dataframe must contain the lineups for each game in the set.
    :param: DataFrame ratings: This dataframe must contain player ratings.
    :return: Appends three columns to the match_data dataframe: one for the average rating for the home team,
            one for the average rating for the away team, and a column for the average difference.
    :rtype: DataFrame        
    
    """

    # initialize lists to store the home team average rating, away team average rating, and average team difference
    home_team_rating = []
    away_team_rating = []
    average_rating_diff =[]
    
    # populate the above list by match
    for i in match_data.index:
        home_team_rating.append(compute_team_average(list(match_data.iloc[i][6:20]), ratings))
        away_team_rating.append(compute_team_average(list(match_data.iloc[i][20:34]), ratings))
        average_rating_diff.append(home_team_rating[i] - away_team_rating[i])
        
    # turn the three lists into series
    home_average_series = pd.Series(home_team_rating)
    away_average_series = pd.Series(away_team_rating)
    average_rating_diff = pd.Series(average_rating_diff)
    
    # append the series as columns into the dataframe
    match_data["home_average_rating"] = home_average_series
    match_data["away_average_rating"] = away_average_series
    match_data["rating_diff"] = average_rating_diff
    
    return match_data

def estimate_conditional_pdf(data, count):
    """This function estimates the conditional probability of one bin in the conditional pdf.
    
    :param DataFrame data: Data frame that satisfies: lower_bound <= rating_diff <= upper_bound.
    :param int count: Number or rows in this dataframe.
    :return: Returns the following three floats: p(win), p(draw), p(loss).
    :rtye float, float, float
    
    """
    
    # count the number of wins 
    # home team wins, home team is higher rated OR away team  wins, away team is higher rated
    wins =  data[data["result"]==1][data["rating_diff"]>0]["rating_diff"].count() 
    wins = wins +  data[data["result"]==-1][data["rating_diff"]<0]["rating_diff"].count()  
    # count the number of draws
    draws = data[data["result"]==0]["rating_diff"].count() # ties
    
    # count the number of losses
    # home team wins, away team is highe rated OR away team wins, home team is higher rated  
    losses = data[data["result"]==1][data["rating_diff"]<0]["rating_diff"].count()  
    losses = losses + data[data["result"]==-1][data["rating_diff"]>0]["rating_diff"].count() 
    
    return wins/count, draws/count, losses/count
    


def estimate_full_conditional_pdf(data, _max, _bins):
    """This function estimates full conditional pdf given data, max differences, and the number of bins (_bins).
    
    :param DataFrame data: The match data dataframe.
    :param float _max: The maximum rating difference between two teams in any game played and part of the data dataframe parameter.
    :param int _bins: The number of bins to be used create the conditional distribution.
    :return: Returns a dataframe containing the conditional pdf.
    :rtype: DataFrame
    
    """
    
    columns = ["Difference","Win", "Draw", "Loss", "Count","Min","Max", "Sum"]
    index = range(_bins)

    pdf = pd.DataFrame(index = index, columns = columns)  
    
    bin_width = _max/_bins
    lower_bound = 0
    upper_bound = bin_width
    
    # creat bins, estimate pdf for each bin in turn
    for i in index:
        # obtain number of rows in bin
        count = data[data["rating_diff"].apply(abs) < upper_bound]\
                [data["rating_diff"].apply(abs) >= lower_bound]["rating_diff"].count()
        
        # deals with zero difference
        if i ==0:
            count= count-data[data["rating_diff"]==0]["rating_diff"].count()
        
        # calculate pdf for => rating difference in [lower_bound, upper_bound]
        p_win, p_draw, p_loss = estimate_conditional_pdf(data[data["rating_diff"].apply(abs) < upper_bound]\
                                                   [data["rating_diff"].apply(abs) >= lower_bound], count)
        
        # update pdf table row
        pdf.loc[i]["Difference"] = "(" + str(lower_bound) + ", " +str(upper_bound)+"]"
        pdf.loc[i]["Win"] = p_win
        pdf.loc[i]["Draw"] = p_draw
        pdf.loc[i]["Loss"] = p_loss
        pdf.loc[i]["Count"] = count
        pdf.loc[i]["Min"] = lower_bound
        pdf.loc[i]["Max"] = upper_bound
        pdf.loc[i]["Sum"] = p_win+p_draw+p_loss
        
        # update bounds
        lower_bound = lower_bound + bin_width
        upper_bound = upper_bound + bin_width
        
    return pdf
            

_max = abs(max(match_data_2018["rating_diff"].agg(["min","max"]), key = abs))
pdf = []
for num in np.arange(math.ceil(_max)*2):
    pdf.append(estimate_full_conditional_pdf(rating_games_2018,math.ceil(_max),num+1))

# extract appropriate probabiilities for a single game
def simulate_single_game(data, pdf):
    """Simulates a single game based on the conditional pdf of matching outcomes on rating differences.
    
    :param DataFrame data: The dataframe row of relevent game.
    :param DataFrame pdf: The dataframe containing the conditional pdf.
    :return: Returns 1 for a home win, -1 for an away win, and zero for a draw.
    :rtype: int
    
    """
    
    difference = data[0]
    u = data[1]
    
    # extract appropriate probabiilities for the game based on the difference
    # i.e. pick out the relevant row of the full pdf
    for i in pdf.index:
        if abs(difference) >=pdf["Min"][i] and abs(difference) < pdf["Max"][i]:
            p_win, p_draw, p_loss = pdf["Win"][i] , pdf["Draw"][i] , pdf["Loss"][i]

    # determine simulated result based on u (a random draw from unif(0,1))
    if u <= p_win:
        if difference > 0 :
            result = 1
        else:
            result = -1
    elif p_win < u <= p_win+p_draw:
        result = 0
    else:
        if difference > 0:
            result = -1
        else:
            result = 1

    return result

def simulate_games(data, pdf):
    """ Simulates all games by making calls to simulate_single_game based on the conditional pdf of matching outcomes on rating differences.
    
    :param DataFrame data: The dataframe row of relevent game.
    :param DataFrame pdf: The dataframe containing the conditional pdf.
    :return: Adds a column to the data dataframe that contains the simulated result.
    :rtype: DataFrame
    
    """
    
    return_df = data.copy()

    # set seed, 42 because of the hitch-hikers guide to the galaxy
    np.random.seed(42)

    # generate ranndom numbers to help simulate each match
    num_matches = len(return_df.index)
    U = np.random.uniform(0,1,num_matches)

    # add random numbers to dataframe
    return_df["U"] = pd.Series(U)

    # simulate all the games
    return_df["simulated_result"] = return_df[["rating_diff","U"]].apply(func=simulate_single_game, pdf= pdf,axis=1)
    
    # remove the U column
    return_df = return_df.drop("U", axis=1)
    
    return return_df

# checks how many times the simulated result equalled the actual result
def score_simulation(data):
    """ This function scores the simulation results. It does this by comparinng the simuulated result to the actual results. The score is equal to the 
    number of matches correctly predicted by the simulation.
    
    :param DataFrame data: A dataframe containing match data and the simulated results for every game.
    :return: The score for the simulation. This is equal to the number of times the simulated result equals the actual/observed results.
    :rtype: int
    
    """
    return data[data["result"]==data["simulated_result"]].count().max()

scores = []
for i in range(len(pdf)):
    scores.append(score_simulation(simulate_games(match_data_2019,pdf[i])))

random_guess_score = score_simulation(simulate_games(match_data_2019,random_pdf))

def format_pdf_for_plot(pdfs, outcome ):
    """This function creates a dataframe that has rows that correspond to the probability of an outcome
    vs the difference in average rating.
    
    :param list pdfs: A list containing all the pdf estimates. Every element in the list is a dataframe.
    :param str outcome: The outcome to focus on: "Win", "Draw", "Loss"
    :return: Probability of outcome vs difference for every pdf, each row corresponds to pdf
    :rtype: DataFrame
    
    """

    out_list = []
    
    for pdf in pdfs:
        q = np.linspace(0,1,len(pdfs))
        for i in pdf.index:
            q[math.floor(pdf["Min"][i]*2):math.ceil(pdf["Max"][i])*2]  = pdf[outcome][i]
        out_list.append(q)
        
    out_df = pd.DataFrame(out_list)
    
    return out_df