import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler, MinMaxScaler
import datetime
import paddlets
from paddlets import TSDataset
from paddlets import TimeSeries
from paddlets.models.forecasting import MLPRegressor, LSTNetRegressor
from paddlets.transform import Fill, StandardScaler
from paddlets.metrics import MSE, MAE
from paddlets.analysis import AnalysisReport, Summary
from paddlets.datasets.repository import get_dataset
import warnings


def data_preprocess(data_dir):
    files = os.listdir(data_dir)
    # 第一步，完成数据格式统一
    for f in files:
        # 获取文件路径
        data_file = os.path.join(data_dir, f)
        # 获取文件名后缀
        data_type = os.path.splitext(data_file)[-1]
        # 获取文件名前缀
        data_name = os.path.splitext(data_file)[0]
        # 如果是excel文件，进行转换
        if data_type == '.xlsx':
            # 需要特别注意的是，在读取excel文件时要指定空值的显示方式，否则会在保存时以字符“.”代替，影响后续的数据分析
            data_xls = pd.read_excel(data_file, index_col=0, na_values='')
            data_xls.to_csv(data_name + '.csv', encoding='utf-8')
            # 顺便删除原文件
            os.remove(data_file)
    # 第二步，完成多文件的合并，文件目录要重新更新一次
    files = os.listdir(data_dir)
    for f in files:
        # 获取文件路径
        data_file = os.path.join(data_dir, f)
        # 获取文件名前缀
        data_basename = os.path.basename(data_file)
        # 检查风机数据是否有多个数据文件
        if len(data_basename.split('-')) > 1:
            merge_list = []
            # 找出该风机的所有数据文件
            matches = [f for f in files if (f.find(data_basename.split('-')[0] + '-') > -1)]
            for i in matches:
                # 读取风机这部分数据
                data_df = pd.read_csv(os.path.join(data_dir, i), index_col=False, keep_default_na=False)
                merge_list.append(data_df)
            if len(merge_list) > 0:
                all_data = pd.concat(merge_list, axis=0, ignore_index=True).fillna(".")
                all_data.to_csv(os.path.join(data_dir, data_basename.split('-')[0] + '.csv'), index=False)
            for i in matches:
                # 删除这部分数据文件
                os.remove(os.path.join(data_dir, i))
                # 更新文件目录
            files = os.listdir(data_dir)


data_file = 'E:\Baidu_competition\功率预测竞赛赛题与数据集'
# data_preprocess(data_file)
# 处理文件时，发现xlrd不能使用，版本过高，使用openpyxl即可实现
# warnings不用管
li = ['/01.csv', '/02.csv', '/03.csv', '/04.csv', '/05.csv', '/06.csv', '/07.csv', '/08.csv', '/09.csv', '/10.csv']
#
# # 笨方法，后续读取文件中名字
for i in li:
    tmp = data_file+i
    print(tmp)
    df = pd.read_csv(tmp, parse_dates=['DATATIME'], infer_datetime_format=True,
                     dayfirst=True, dtype={'WINDDIRECTION': np.float64, 'HUMIDITY': np.float64, 'PRESSURE': np.float64})
    df.drop_duplicates(subset=['DATATIME'], keep='first', inplace=True)
    target_cov_dataset = TSDataset.load_from_dataframe(
        df,
        time_col='DATATIME',
        target_cols=['ROUND(A.POWER,0)', 'YD15'],
        observed_cov_cols=['WINDSPEED', 'PREPOWER', 'WINDDIRECTION', 'TEMPERATURE',
                           'HUMIDITY', 'PRESSURE', 'ROUND(A.WS,1)'],
        freq='15min',
        fill_missing_dates=True,
        fillna_method='pre'
    )
    # 时间戳不确定正确
    _, train_dataset = target_cov_dataset.split('2021-05-01 00:00:00')
    train_dataset, val_test_dataset = train_dataset.split('2021-08-01 00:00:00')
    val_dataset, test_dataset = val_test_dataset.split('2021-09-01 00:00:00')
    # 最后一天的工况数据需要预测ROUND(A.POWER,0)和YD15两个字段
    test_dataset, pred_dataset = test_dataset.split('2021-10-01 00:00:00')
    train_dataset.plot(add_data=[val_dataset, test_dataset], labels=['Val', 'Test'])