adv_train_wbc
/
exp2_plot_spatial_STM2+BO.py

# %%
from cv2 import rotate
from utils.plotFunction import test_report
from statistics import mean
from foolbox import accuracy
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.metrics import (
    accuracy_score,
    auc,
    average_precision_score,
    precision_recall_curve,
    roc_auc_score,
    roc_curve,
)

from utils.tools import caculate_all_accuracy, caculate_all_roi, caculate_sAdd_accuracy
CUDA_VISIBLE_DEVICES = 0

plt.switch_backend('agg')

res_dict = {}
res_dict_change = {}

model_dict = {
    # =================== Standard
    "Prob_clean_EfficientNet-B3": "clean_EfficientNet-B3",
    "Prob_spatial_STM+BO_30.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_60.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_90.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_120.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_150.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_180.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    # trans
    "Prob_spatial_STM+BO_1.0_-1_2.0_1000_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_1.0_-1_4.0_1000_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_1.0_-1_6.0_1000_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_1.0_-1_8.0_1000_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_1.0_-1_10.0_1000_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_1.0_-1_12.0_1000_EfficientNet-B3": "tmp",

    # =================== AT_STM
    "Prob_clean_['', 'STM']_120_withoutTrades_EfficientNet-B3": "AT_STM+BO_EfficientNet-B3",
    "Prob_spatial_STM+BO_30.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_60.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_90.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_120.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_150.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_180.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    # trans
    "Prob_spatial_STM+BO_1.0_-1_2.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_1.0_-1_4.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_1.0_-1_6.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_1.0_-1_8.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_1.0_-1_10.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Prob_spatial_STM+BO_1.0_-1_12.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
}


model_dict_change = {
    # =================== Standard
    "Change_spatial_STM+BO_30.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_60.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_90.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_120.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_150.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_180.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    # trans
    "Change_spatial_STM+BO_1.0_-1_2.0_1000_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_1.0_-1_4.0_1000_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_1.0_-1_6.0_1000_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_1.0_-1_8.0_1000_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_1.0_-1_10.0_1000_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_1.0_-1_12.0_1000_EfficientNet-B3": "tmp",

    # =================== AT_STM
    "Change_spatial_STM+BO_30.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_60.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_90.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_120.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_150.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_180.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    # trans
    "Change_spatial_STM+BO_1.0_-1_2.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_1.0_-1_4.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_1.0_-1_6.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_1.0_-1_8.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_1.0_-1_10.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
    "Change_spatial_STM+BO_1.0_-1_12.0_1000_['', 'STM']_120_withoutTrades_EfficientNet-B3": "tmp",
}


color_dict = {
    # 'Prob_clean_ResNet18': '#e30039',  # 亮红

    # "Prob_clean_ResNet50": "#00994e",

    # "Prob_clean_SE-ResNet50Xt": "#00a8e1",

    # "Prob_clean_VGG16": "#fcd300",

    "Prob_clean_EfficientNet-B3": "#99cc00",
    "Prob_clean_['', 'STM']_120_withoutTrades_EfficientNet-B3": "#fcd300",

}


for m in model_dict:
    res_dict[m + '_label'] = pd.read_csv(
        f'./result/midLog/spatial/exp2_spatial_useSTMtrain_useSTM+BOattack/{m}.csv', header=None, index_col=None, sep=' '
    ).to_numpy()[:, 0]
    res_dict[m + '_proba'] = pd.read_csv(
        f'./result/midLog/spatial/exp2_spatial_useSTMtrain_useSTM+BOattack/{m}.csv', header=None, index_col=None, sep=' '
    ).to_numpy()[:, 2]
    res_df = pd.DataFrame(res_dict)

for m in model_dict_change:
    res_dict_change[m + '_label'] = pd.read_csv(
        f'./result/midLog/spatial/exp2_spatial_useSTMtrain_useSTM+BOattack/{m}.csv', header=None, index_col=None, sep=' '
    ).to_numpy()[:, 0]
    res_dict_change[m + '_oriPred'] = pd.read_csv(
        f'./result/midLog/spatial/exp2_spatial_useSTMtrain_useSTM+BOattack/{m}.csv', header=None, index_col=None, sep=' '
    ).to_numpy()[:, 1]
    res_dict_change[m + '_advPred'] = pd.read_csv(
        f'./result/midLog/spatial/exp2_spatial_useSTMtrain_useSTM+BOattack/{m}.csv', header=None, index_col=None, sep=' '
    ).to_numpy()[:, 2]

    res_df_change = pd.DataFrame(res_dict_change)


# %% ROC

baseline_models = ["EfficientNet-B3",
                   "['', 'STM']_120_withoutTrades_EfficientNet-B3"]


def plot_roc(name, labels, predictions, **kwargs):
    fp, tp, _ = roc_curve(labels, predictions)
    # alpha_value = 1 if name == 'Proposed method' else 0.7
    lw = 1.5 if name == 'Proposed method' else 1
    ls = 'dotted' if (name == 'PredPHI' or name == 'PHIAF') else '-'
    plt.plot(
        fp,
        tp,
        label=name + f' (AUC = {roc_auc_score(labels, predictions):.3f})',
        linewidth=lw,
        linestyle=ls,
        **kwargs,
    )
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.grid(True, linestyle='--', alpha=0.5)
    plt.legend(fontsize=13)


for m in baseline_models:
    perfix = "Prob_clean_"
    plot_roc(
        model_dict[perfix+m],
        res_df[perfix+m + '_label'][1:].astype(float).astype(int),
        res_df[perfix+m + '_proba'][1:].astype(float),
        color=color_dict[perfix+m],
    )
plt.title('ROC Curve', fontsize=18)

# plt.savefig('../result/ROC_Curve.eps', dpi=600, format='eps',  bbox_inches = 'tight')

plt.show()

# %% PRC

baseline_models = ["EfficientNet-B3",
                   "['', 'STM']_120_withoutTrades_EfficientNet-B3"]


def plot_prc(name, labels, predictions, **kwargs):
    precision, recall, _ = precision_recall_curve(labels, predictions)
    lw = 1.5 if name == 'Proposed method' else 0.8
    ls = 'dotted' if (name == 'PredPHI' or name == 'PHIAF') else '-'
    plt.plot(
        precision,
        recall,
        label=name +
        f' (AUPR = {average_precision_score(labels, predictions):.3f})',
        linewidth=lw,
        linestyle=ls,
        **kwargs,
    )
    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.xlim([0.4, 1])
    plt.ylim([0, 1])
    plt.grid(True, linestyle='--', alpha=0.5)
    plt.legend(fontsize=13)
    # ax = plt.gca()
    # ax.set_aspect('equal')


for m in baseline_models:
    perfix = "Prob_clean_"
    plot_prc(
        model_dict[perfix+m],
        res_df[perfix+m + '_label'][1:].astype(float).astype(int),
        res_df[perfix+m + '_proba'][1:].astype(float),
        color=color_dict[perfix+m],
    )
plt.title('PR Curve', fontsize=18)

# plt.savefig('../result/PR_Curve.eps', dpi=600, format='eps',  bbox_inches = 'tight')

plt.show()


# %% test result csv 2 df ------------------------------

baseline_models = ["EfficientNet-B3",
                   "['', 'STM']_120_withoutTrades_EfficientNet-B3"]

results = pd.DataFrame()
for m in baseline_models:
    perfix = "Prob_clean_"
    r = test_report(
        # y_test=res_dict[m + '_label'],
        # y_pred_probe=res_dict[m + '_proba'],
        res_df[perfix+m + '_label'][1:].astype(float).astype(int),
        res_df[perfix+m + '_proba'][1:].astype(float),
        print_flag=False,
    )
    results = results.append(pd.DataFrame(r).T, ignore_index=True)
results.columns = [
    'TN',
    'FP',
    'FN',
    'TP',
    'Precision_0',
    'Recall_0',
    'F1_0',
    'Precision_1',
    'Recall_1',
    'F1_1',
    'Accuracy',
    'AUC',
    'AUPR',
    'KS-Value',
    'MCC',
    'G-Mean',
]
results = results.round(3)
results.index = baseline_models
results

# For Latex input

results[
    ['Accuracy', 'AUC', 'Precision_0', 'Precision_1', 'Recall_0', 'Recall_1',
     'F1_0', 'F1_1',
     'TN',
     'FP',
     'FN',
     'TP', ]
]


# %% spatial STM(rotate)


mdb5 = [
    "clean_EfficientNet-B3",
    "spatial_STM+BO_30.0_1000_1.0_-1_EfficientNet-B3",
    "spatial_STM+BO_60.0_1000_1.0_-1_EfficientNet-B3",
    "spatial_STM+BO_90.0_1000_1.0_-1_EfficientNet-B3",
    "spatial_STM+BO_120.0_1000_1.0_-1_EfficientNet-B3",
    "spatial_STM+BO_150.0_1000_1.0_-1_EfficientNet-B3",
    "spatial_STM+BO_180.0_1000_1.0_-1_EfficientNet-B3",

]

mdb55 = [
    # ========== AT_STM
    "clean_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_30.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_60.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_90.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_120.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_150.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_180.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
]


md1 = [mdb5, mdb55]
model_names = ["EfficientNet-B3",
               "['', 'STM']_120_withoutTrades_EfficientNet-B3"]


def plot_diff_rotate_acc(
    name,
    y_label,
    x_label,
    all_accuracy,
    rotate_range,
        **kwargs):
    # print(all_accuracy)
    # lw = 1.5 if name == 'Proposed method' else 0.8
    # ls = 'dotted' if (name == 'PredPHI' or name == 'PHIAF') else '-'
    lw = 1.5
    ls = '-'

    def getName(name):
        if name == "EfficientNet-B3":
            return name
        elif name == "['', 'STM']_120_withoutTrades_EfficientNet-B3":
            return "AT_STM_EfficientNet-B3"

    plt.plot(
        rotate_range,
        all_accuracy,
        label=getName(name),
        marker='v' if name == model_names[0] else 'o',
        markersize='10',
        linewidth=lw,
        linestyle=ls,
        **kwargs,
    )
    plt.xlabel(x_label, fontsize=18)
    plt.ylabel(y_label, fontsize=18)
    # plt.xlim([eps_range[0]-2/255, eps_range[-1]+2/255])
    plt.xticks(rotate_range, ("0", "30", "60", "90",
               "120", "150", "180"), fontsize=18)
    # plt.ylim([0.4, 1])
    plt.yticks([0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1], fontsize=18)
    plt.grid(True, linestyle='--', alpha=0.5)
    plt.legend(fontsize=13)


def plot_diff_rotate_roi(
    name,
    all_roi,
    rotate_range,
        **kwargs):
    # lw = 1.5 if name == 'Proposed method' else 0.8
    # ls = 'dotted' if (name == 'PredPHI' or name == 'PHIAF') else '-'
    lw = 1.5
    ls = '-'

    def getName(name):
        if name == "EfficientNet-B3":
            return name
        elif name == "['', 'STM']_120_withoutTrades_EfficientNet-B3":
            return "AT_STM_EfficientNet-B3"

    plt.plot(
        rotate_range,
        all_roi,
        label=getName(name),
        marker='v' if name == model_names[0] else 'o',
        markersize='10',
        linewidth=lw,
        linestyle=ls,
        **kwargs,
    )
    plt.xlabel('Rotation range', fontsize=18)
    plt.ylabel('Rate of invariance', fontsize=18)
    plt.xticks(rotate_range, ("0", "30", "60", "90",
               "120", "150", "180"), fontsize=18)
    # plt.ylim([0.7, 1])
    plt.yticks([0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1], fontsize=18)
    plt.grid(True, linestyle='--', alpha=0.5)
    plt.legend(fontsize=13)


#  ===============================================Accuracy
for i, mb in enumerate(md1):
    plot_diff_rotate_acc(
        model_names[i],
        y_label="Accuracy",
        x_label="Rotate range",
        all_accuracy=caculate_all_accuracy(mb, res_df),
        rotate_range=[0, 30, 60, 90, 120, 150, 180],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
plt.title('STM+BO Attack', fontsize=18)
plt.savefig("./result/final_result/STMtrain_Eval_STM+BO_Rotate_acc.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()


# =============================================== S+ Accuracy (S+表示正确分类的样本)
for i, mb in enumerate(md1):
    plot_diff_rotate_acc(
        model_names[i],
        y_label="S+ Accuracy",
        x_label="Rotate range",
        all_accuracy=caculate_sAdd_accuracy(mb[1:], res_df_change),
        rotate_range=[0, 30, 60, 90, 120, 150, 180],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
plt.title('STM+BO Attack', fontsize=18)
plt.savefig("./result/final_result/STMtrain_Eval_STM+BO_S+_Rotate_acc.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()


# ===============================================Rate of invariance

# 绘制不变性指标
for (i, mb) in enumerate(md1):
    plot_diff_rotate_roi(
        model_names[i],
        caculate_all_roi(mb[1:], res_df_change),
        rotate_range=[0, 30, 60, 90, 120, 150, 180],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
plt.title('STM+BO Attack', fontsize=18)
plt.savefig("./result/final_result/STMtrain_Eval_STM+BO_Rotate_invariance.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()

# %% spatial STM(transition)


mdb10 = [
    "clean_EfficientNet-B3",
    # trans
    "spatial_STM+BO_1.0_-1_2.0_1000_EfficientNet-B3",
    "spatial_STM+BO_1.0_-1_4.0_1000_EfficientNet-B3",
    "spatial_STM+BO_1.0_-1_6.0_1000_EfficientNet-B3",
    "spatial_STM+BO_1.0_-1_8.0_1000_EfficientNet-B3",
    "spatial_STM+BO_1.0_-1_10.0_1000_EfficientNet-B3",
    "spatial_STM+BO_1.0_-1_12.0_1000_EfficientNet-B3",
]


mdb100 = [
    # ========== AT_STM
    "clean_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_30.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_60.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_90.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_120.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_150.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
    "spatial_STM+BO_180.0_1000_1.0_-1_['', 'STM']_120_withoutTrades_EfficientNet-B3",
]


md2 = [mdb10, mdb100]
model_names = ["EfficientNet-B3",
               "['', 'STM']_120_withoutTrades_EfficientNet-B3"]


def plot_diff_trans_roi(
    name,
    all_roi,
    trans_range,
        **kwargs):
    # lw = 1.5 if name == 'Proposed method' else 0.8
    # ls = 'dotted' if (name == 'PredPHI' or name == 'PHIAF') else '-'
    lw = 1.5
    ls = '-'

    def getName(name):
        if name == "EfficientNet-B3":
            return name
        elif name == "['', 'STM']_120_withoutTrades_EfficientNet-B3":
            return "AT_STM_EfficientNet-B3"

    plt.plot(
        trans_range,
        all_roi,
        label=getName(name),
        marker='v' if name == model_names[0] else 'o',
        markersize='10',
        linewidth=lw,
        linestyle=ls,
        **kwargs,
    )
    plt.xlabel('Transition range', fontsize=18)
    plt.ylabel('Rate of invariance', fontsize=18)
    plt.xticks(trans_range, ("0", "2%", "4%", "6%",
               "8%", "10%", "12%"), fontsize=18)
    # plt.ylim([0.7, 1])
    plt.yticks([0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1], fontsize=18)
    plt.grid(True, linestyle='--', alpha=0.5)
    plt.legend(fontsize=13)


def plot_diff_trans_acc(
    name,
    y_label,
    x_label,
    all_accuracy,
    trans_range,
        **kwargs):
    # lw = 1.5 if name == 'Proposed method' else 0.8
    # ls = 'dotted' if (name == 'PredPHI' or name == 'PHIAF') else '-'
    lw = 1.5
    ls = '-'

    def getName(name):
        if name == "EfficientNet-B3":
            return name
        elif name == "['', 'STM']_120_withoutTrades_EfficientNet-B3":
            return "AT_STM_EfficientNet-B3"

    plt.plot(
        trans_range,
        all_accuracy,
        label=getName(name),
        marker='v' if name == model_names[0] else 'o',
        markersize='10',
        linewidth=lw,
        linestyle=ls,
        **kwargs,
    )
    plt.xlabel(x_label, fontsize=18)
    plt.ylabel(y_label, fontsize=18)
    # plt.xlim([eps_range[0]-2/255, eps_range[-1]+2/255])
    plt.xticks(trans_range, ("0", "2%", "4%", "6%",
               "8%", "10%", "12%"), fontsize=18)
    # plt.ylim([0.7, 1])
    plt.yticks([0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1], fontsize=18)
    plt.grid(True, linestyle='--', alpha=0.5)
    plt.legend(fontsize=13)


# =============================================== Accuracy
for i, mb in enumerate(md2):
    plot_diff_trans_acc(
        model_names[i],
        y_label="Accuracy",
        x_label="Transition range",
        all_accuracy=caculate_all_accuracy(mb, res_df),
        trans_range=[0, 2, 4, 6, 8, 10, 12],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
plt.title('STM+BO Attack', fontsize=18)
plt.savefig("./result/final_result/STMtrain_Eval_STM+BO_Trans_acc.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()

# =============================================== S+ Accuracy (S+表示正确分类的样本)
for i, mb in enumerate(md2):
    plot_diff_trans_acc(
        model_names[i],
        y_label="S+ Accuracy",
        x_label="Transition range",
        all_accuracy=caculate_sAdd_accuracy(mb[1:], res_df_change),
        trans_range=[0, 2, 4, 6, 8, 10, 12],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
plt.title('STM+BO Attack', fontsize=18)
plt.savefig("./result/final_result/STMtrain_Eval_STM+BO_S+_Trans_acc.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()


# =============================================== Rate of invariance
for (i, mb) in enumerate(md2):
    plot_diff_trans_roi(
        model_names[i],
        caculate_all_roi(mb[1:], res_df_change),
        trans_range=[0, 2, 4, 6, 8, 10, 12],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
plt.title('STM+BO Attack', fontsize=18)
plt.savefig("./result/final_result/STMtrain_Eval_STM+BO_Trans_invariance.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()

# %%