import logging
import os
from datetime import date
from io import StringIO
from itertools import groupby

from bs4 import BeautifulSoup
import requests
import pandas as pd
import tenacity

from . import utils, validation
from .notifications import send_report

logger = logging.getLogger(__name__)

url = "http://www.cboe.com/delayedquote/quote-table-download"


def fetch_data(symbols=None):
    """Fetches options data for a given list of symbols"""
    symbols = symbols or _get_all_listed_symbols()
    valid_symbols = _get_all_listed_symbols()

    try:
        form_data = _form_data()
    except requests.ConnectionError as ce:
        msg = "Connection error trying to reach {}".format(url)
        logger.error(msg)
        raise ce
    except Exception as e:
        msg = "Error parsing response"
        logger.error(msg, exc_info=True)
        raise e

    headers = {"Referer": url}
    file_url = "http://www.cboe.com/delayedquote/quotedata.dat"

    symbols = [symbol.upper() for symbol in symbols]
    done = 0
    failed = []

    for symbol in symbols:
        form_data["ctl00$ContentTop$C005$txtTicker"] = symbol

        try:
            assert symbol in valid_symbols
            response = requests.post(url,
                                     data=form_data,
                                     headers=headers,
                                     allow_redirects=False)
            symbol_req = requests.get(file_url,
                                      cookies=response.cookies,
                                      headers=headers)
            symbol_data = symbol_req.text

            if symbol_data == "" or symbol_data.startswith(" <!DOCTYPE"):
                raise Exception
        except AssertionError:
            msg = "Error fetching invalid symbol {} data".format(symbol)
            logger.error(msg, exc_info=True)
        except Exception:
            failed.append(symbol)
            msg = "Error fetching symbol {} data".format(symbol)
            logger.error(msg, exc_info=True)
        else:
            _save_data(symbol, symbol_data)
            done += 1
    retry_failure(failed, done)
    send_report(done, failed, __name__)


# if a symbol failes to scrape try again exponentialy
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=300),
                stop=tenacity.stop_after_attempt(10),
                retry=tenacity.retry_if_exception_type(IOError))
def retry_failure(failed, done):
    form_data = _form_data()
    headers = {"Referer": url}
    file_url = "http://www.cboe.com/delayedquote/quotedata.dat"
    for symbol in failed:
        try:
            response = requests.post(url,
                                     data=form_data,
                                     headers=headers,
                                     allow_redirects=False)

            symbol_req = requests.get(file_url,
                                      cookies=response.cookies,
                                      headers=headers)
            symbol_data = symbol_req.text
            if symbol_data == "" or symbol_data.startswith(" <!DOCTYPE"):
                raise Exception
        except Exception:
            msg = "error fetching symbol {} data".format(symbol)
            logger.error(msg, exc_info=True)
        else:
            _save_data(symbol, symbol_data)
            done += 1
            failed.remove(symbol)


def aggregate_monthly_data(symbols=None):
    """Aggregate daily snapshots into monthly files and validate data"""
    symbols = symbols or _get_all_listed_symbols()
    save_data_path = utils.get_save_data_path()
    scraper_dir = os.path.join(save_data_path, "cboe")
    symbols = [symbol.upper() for symbol in symbols]
    done = 0
    failed = []

    for symbol in symbols:
        daily_dir = os.path.join(scraper_dir, symbol + "_daily")

        if not os.path.exists(daily_dir):
            msg = "Error aggregating data. Dir {} not found.".format(daily_dir)
            logger.error(msg)
            continue

        monthly_dir = os.path.join(scraper_dir, symbol)
        symbol_files = [
            file for file in os.listdir(daily_dir) if file.endswith(".csv")
        ]

        for month, files in groupby(sorted(symbol_files), _monthly_grouper):
            file_names = list(files)
            daily_files = [
                os.path.join(daily_dir, name) for name in file_names
            ]

            try:
                symbol_df = concatenate_files(daily_files)
            except Exception:

                msg = "Error concatenating daily files for period " + month
                logger.error(msg, exc_info=True)

                continue

            date_range = pd.to_datetime(symbol_df["quotedate"].unique())

            if not validation.validate_dates_in_month(symbol, date_range):
                today = pd.Timestamp.today()
                first_date = date_range[0]
                if first_date.year != today.year or first_date.month != today.month:
                    msg = "Some trading dates where missing for symbol {} in period {}".format(
                        symbol, month)
                    logger.error(msg)

                    failed.append(symbol)
                continue

            if not os.path.exists(monthly_dir):
                os.makedirs(monthly_dir)
                logger.debug("Symbol dir %s created", monthly_dir)

            file_name = _monthly_filename(file_names)
            monthly_file = os.path.join(monthly_dir, file_name)
            symbol_df.to_csv(monthly_file, index=False)
            if not validation.validate_aggregate_file(monthly_file,
                                                      daily_files):
                utils.remove_file(monthly_file)
                msg = "Data in {} differs from the daily files".format(
                    monthly_file)
                logger.error(msg)
                continue

            logger.debug("Saved monthly data %s", monthly_file)
            done += 1

            for file in daily_files:
                utils.remove_file(file, logger)

    send_report(done, failed, __name__, op="aggregate")


def _get_all_listed_symbols():
    """Returns array of all listed symbols.
    http://www.cboe.com/publish/scheduledtask/mktdata/cboesymboldir2.csv
    """
    url = 'http://www.cboe.com/publish/scheduledtask/mktdata/cboesymboldir2.csv'
    symbols_df = pd.read_csv(url, skiprows=1)
    return symbols_df["Stock Symbol"].array


def concatenate_files(files):
    """Returns a dataframe of the concatenated data from `files`."""
    df_generator = (pd.read_csv(file) for file in sorted(files))
    return pd.concat(df_generator, ignore_index=True)


def _form_data():
    """Return validation form data"""
    homepage = requests.get(url)
    soup = BeautifulSoup(homepage.content, "lxml")
    data = {
        "__VIEWSTATE": soup.select_one("#__VIEWSTATE")["value"],
        "__EVENTVALIDATION": soup.select_one("#__EVENTVALIDATION")["value"]
    }
    return data


def _save_data(symbol, symbol_data):
    """Saves the contents of `symbol_data` to
    `$SAVE_DATA_PATH/cboe/{symbol}_daily/{symbol}_{%date}.csv`
    """
    filename = date.today().strftime(symbol + "_%Y%m%d.csv")

    save_data_path = utils.get_save_data_path()
    symbol_dir = os.path.join(save_data_path, "cboe", symbol + "_daily")

    if not os.path.exists(symbol_dir):
        os.makedirs(symbol_dir)
        logger.debug("Symbol dir %s created", symbol_dir)
    file_path = os.path.join(symbol_dir, filename)

    if os.path.exists(file_path) and validation.file_hash_matches_data(
            file_path, symbol_data):
        logger.debug("File %s already downloaded", file_path)
    else:
        daily_df = _wrangle_data(symbol, symbol_data)
        daily_df.to_csv(file_path, index=False)
        logger.debug("Saved daily symbol data as %s", file_path)


def _wrangle_data(symbol, symbol_data):
    """Returns a properly formated (_tidy_) dataframe"""
    string_data = StringIO(symbol_data)
    first_line = string_data.readline()
    spot_price = float(first_line.split(",")[-2])
    quote_date = date.today().strftime("%m/%d/%Y")

    data = pd.read_csv(string_data, skiprows=1)
    call_columns = [
        "Calls", "Expiration Date", "Strike", "Last Sale", "Net", "Bid", "Ask",
        "Vol", "Open Int", "IV", "Delta", "Gamma"
    ]
    calls = data[call_columns]

    put_columns = [
        "Puts", "Expiration Date", "Strike", "Last Sale.1", "Net.1", "Bid.1",
        "Ask.1", "Vol.1", "Open Int.1", "IV.1", "Delta.1", "Gamma.1"
    ]
    puts = data[put_columns]

    renamed_columns = [
        "optionroot", "expiration", "strike", "last", "net", "bid", "ask",
        "volume", "openinterest", "impliedvol", "delta", "gamma"
    ]
    calls.columns = renamed_columns
    calls.insert(loc=1, column="type", value="call")
    puts.columns = renamed_columns
    puts.insert(loc=1, column="type", value="put")

    merged = pd.concat([calls, puts])
    merged.insert(loc=0, column="underlying", value=symbol)
    merged.insert(loc=1, column="underlying_last", value=spot_price)
    merged.insert(loc=2, column="exchange", value="CBOE")
    merged.insert(loc=6, column="quotedate", value=quote_date)

    return merged


def _monthly_grouper(filename):
    """Returns `{year}{month}` string. Used to group files by month."""
    basename = filename.split(".")[0]
    file_date = basename.split("_")[1]
    return file_date[:-2]


def _monthly_filename(filenames):
    """Returns filename of monthly aggregate file in the form
    `{symbol}_{start_date}_to_{end_date}.csv`
    """
    sorted_files = list(sorted(filenames))
    first_file = sorted_files[0]
    last_file = sorted_files[-1]
    last_day = last_file.split(".")[0][-8:]  # Get only the date
    file_name = first_file.split(".")[0] + "_to_" + last_day + ".csv"
    return file_name