adv_train_wbc
/
(F)exp1_plot_spatial_STM.py

# %%
from cv2 import rotate
from utils.plotFunction import test_report
from statistics import mean
from foolbox import accuracy
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.metrics import (
    accuracy_score,
    auc,
    average_precision_score,
    precision_recall_curve,
    roc_auc_score,
    roc_curve,
)

from utils.tools import caculate_all_accuracy, caculate_all_roi, caculate_sAdd_accuracy
from func import plot_diff_rotate_acc, plot_diff_rotate_roi, plot_diff_trans_acc, plot_diff_trans_roi
from param import color_dict, advModel_labels, baseModel_marks


CUDA_VISIBLE_DEVICES = 1
plt.switch_backend('agg')

res_dict = {}
res_dict_change = {}

model_dict = {
    "Prob_clean_ResNet18": "clean ResNet18",
    # rotate
    "Prob_spatial_STM_30.0_1000_1.0_-1_ResNet18": "tmp",
    "Prob_spatial_STM_60.0_1000_1.0_-1_ResNet18": "tmp",
    "Prob_spatial_STM_90.0_1000_1.0_-1_ResNet18": "tmp",
    "Prob_spatial_STM_120.0_1000_1.0_-1_ResNet18": "tmp",
    "Prob_spatial_STM_150.0_1000_1.0_-1_ResNet18": "tmp",
    "Prob_spatial_STM_180.0_1000_1.0_-1_ResNet18": "tmp",
    # trans
    "Prob_spatial_STM_1.0_-1_2.0_1000_ResNet18": "tmp",
    "Prob_spatial_STM_1.0_-1_4.0_1000_ResNet18": "tmp",
    "Prob_spatial_STM_1.0_-1_6.0_1000_ResNet18": "tmp",
    "Prob_spatial_STM_1.0_-1_8.0_1000_ResNet18": "tmp",
    "Prob_spatial_STM_1.0_-1_10.0_1000_ResNet18": "tmp",
    "Prob_spatial_STM_1.0_-1_12.0_1000_ResNet18": "tmp",

    "Prob_clean_ResNet50": "tmp",
    "Prob_spatial_STM_30.0_1000_1.0_-1_ResNet50": "tmp",
    "Prob_spatial_STM_60.0_1000_1.0_-1_ResNet50": "tmp",
    "Prob_spatial_STM_90.0_1000_1.0_-1_ResNet50": "tmp",
    "Prob_spatial_STM_120.0_1000_1.0_-1_ResNet50": "tmp",
    "Prob_spatial_STM_150.0_1000_1.0_-1_ResNet50": "tmp",
    "Prob_spatial_STM_180.0_1000_1.0_-1_ResNet50": "tmp",
    # trans
    "Prob_spatial_STM_1.0_-1_2.0_1000_ResNet50": "tmp",
    "Prob_spatial_STM_1.0_-1_4.0_1000_ResNet50": "tmp",
    "Prob_spatial_STM_1.0_-1_6.0_1000_ResNet50": "tmp",
    "Prob_spatial_STM_1.0_-1_8.0_1000_ResNet50": "tmp",
    "Prob_spatial_STM_1.0_-1_10.0_1000_ResNet50": "tmp",
    "Prob_spatial_STM_1.0_-1_12.0_1000_ResNet50": "tmp",


    "Prob_clean_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_30.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_60.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_90.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_120.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_150.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_180.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    # trans
    "Prob_spatial_STM_1.0_-1_2.0_1000_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_1.0_-1_4.0_1000_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_1.0_-1_6.0_1000_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_1.0_-1_8.0_1000_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_1.0_-1_10.0_1000_SE-ResNet50Xt": "tmp",
    "Prob_spatial_STM_1.0_-1_12.0_1000_SE-ResNet50Xt": "tmp",


    "Prob_clean_VGG16": "tmp",
    "Prob_spatial_STM_30.0_1000_1.0_-1_VGG16": "tmp",
    "Prob_spatial_STM_60.0_1000_1.0_-1_VGG16": "tmp",
    "Prob_spatial_STM_90.0_1000_1.0_-1_VGG16": "tmp",
    "Prob_spatial_STM_120.0_1000_1.0_-1_VGG16": "tmp",
    "Prob_spatial_STM_150.0_1000_1.0_-1_VGG16": "tmp",
    "Prob_spatial_STM_180.0_1000_1.0_-1_VGG16": "tmp",
    # trans
    "Prob_spatial_STM_1.0_-1_2.0_1000_VGG16": "tmp",
    "Prob_spatial_STM_1.0_-1_4.0_1000_VGG16": "tmp",
    "Prob_spatial_STM_1.0_-1_6.0_1000_VGG16": "tmp",
    "Prob_spatial_STM_1.0_-1_8.0_1000_VGG16": "tmp",
    "Prob_spatial_STM_1.0_-1_10.0_1000_VGG16": "tmp",
    "Prob_spatial_STM_1.0_-1_12.0_1000_VGG16": "tmp",

    "Prob_clean_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_30.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_60.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_90.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_120.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_150.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_180.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    # trans
    "Prob_spatial_STM_1.0_-1_2.0_1000_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_1.0_-1_4.0_1000_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_1.0_-1_6.0_1000_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_1.0_-1_8.0_1000_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_1.0_-1_10.0_1000_EfficientNet-B3": "tmp",
    "Prob_spatial_STM_1.0_-1_12.0_1000_EfficientNet-B3": "tmp",


}


model_dict_change = {
    "Change_spatial_STM_30.0_1000_1.0_-1_ResNet18": "tmp",
    "Change_spatial_STM_60.0_1000_1.0_-1_ResNet18": "tmp",
    "Change_spatial_STM_90.0_1000_1.0_-1_ResNet18": "tmp",
    "Change_spatial_STM_120.0_1000_1.0_-1_ResNet18": "tmp",
    "Change_spatial_STM_150.0_1000_1.0_-1_ResNet18": "tmp",
    "Change_spatial_STM_180.0_1000_1.0_-1_ResNet18": "tmp",
    # trans
    "Change_spatial_STM_1.0_-1_2.0_1000_ResNet18": "tmp",
    "Change_spatial_STM_1.0_-1_4.0_1000_ResNet18": "tmp",
    "Change_spatial_STM_1.0_-1_6.0_1000_ResNet18": "tmp",
    "Change_spatial_STM_1.0_-1_8.0_1000_ResNet18": "tmp",
    "Change_spatial_STM_1.0_-1_10.0_1000_ResNet18": "tmp",
    "Change_spatial_STM_1.0_-1_12.0_1000_ResNet18": "tmp",


    "Change_spatial_STM_30.0_1000_1.0_-1_ResNet50": "tmp",
    "Change_spatial_STM_60.0_1000_1.0_-1_ResNet50": "tmp",
    "Change_spatial_STM_90.0_1000_1.0_-1_ResNet50": "tmp",
    "Change_spatial_STM_120.0_1000_1.0_-1_ResNet50": "tmp",
    "Change_spatial_STM_150.0_1000_1.0_-1_ResNet50": "tmp",
    "Change_spatial_STM_180.0_1000_1.0_-1_ResNet50": "tmp",
    # trans
    "Change_spatial_STM_1.0_-1_2.0_1000_ResNet50": "tmp",
    "Change_spatial_STM_1.0_-1_4.0_1000_ResNet50": "tmp",
    "Change_spatial_STM_1.0_-1_6.0_1000_ResNet50": "tmp",
    "Change_spatial_STM_1.0_-1_8.0_1000_ResNet50": "tmp",
    "Change_spatial_STM_1.0_-1_10.0_1000_ResNet50": "tmp",
    "Change_spatial_STM_1.0_-1_12.0_1000_ResNet50": "tmp",


    "Change_spatial_STM_30.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    "Change_spatial_STM_60.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    "Change_spatial_STM_90.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    "Change_spatial_STM_120.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    "Change_spatial_STM_150.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    "Change_spatial_STM_180.0_1000_1.0_-1_SE-ResNet50Xt": "tmp",
    # trans
    "Change_spatial_STM_1.0_-1_2.0_1000_SE-ResNet50Xt": "tmp",
    "Change_spatial_STM_1.0_-1_4.0_1000_SE-ResNet50Xt": "tmp",
    "Change_spatial_STM_1.0_-1_6.0_1000_SE-ResNet50Xt": "tmp",
    "Change_spatial_STM_1.0_-1_8.0_1000_SE-ResNet50Xt": "tmp",
    "Change_spatial_STM_1.0_-1_10.0_1000_SE-ResNet50Xt": "tmp",
    "Change_spatial_STM_1.0_-1_12.0_1000_SE-ResNet50Xt": "tmp",


    "Change_spatial_STM_30.0_1000_1.0_-1_VGG16": "tmp",
    "Change_spatial_STM_60.0_1000_1.0_-1_VGG16": "tmp",
    "Change_spatial_STM_90.0_1000_1.0_-1_VGG16": "tmp",
    "Change_spatial_STM_120.0_1000_1.0_-1_VGG16": "tmp",
    "Change_spatial_STM_150.0_1000_1.0_-1_VGG16": "tmp",
    "Change_spatial_STM_180.0_1000_1.0_-1_VGG16": "tmp",
    # trans
    "Change_spatial_STM_1.0_-1_2.0_1000_VGG16": "tmp",
    "Change_spatial_STM_1.0_-1_4.0_1000_VGG16": "tmp",
    "Change_spatial_STM_1.0_-1_6.0_1000_VGG16": "tmp",
    "Change_spatial_STM_1.0_-1_8.0_1000_VGG16": "tmp",
    "Change_spatial_STM_1.0_-1_10.0_1000_VGG16": "tmp",
    "Change_spatial_STM_1.0_-1_12.0_1000_VGG16": "tmp",


    "Change_spatial_STM_30.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Change_spatial_STM_60.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Change_spatial_STM_90.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Change_spatial_STM_120.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Change_spatial_STM_150.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    "Change_spatial_STM_180.0_1000_1.0_-1_EfficientNet-B3": "tmp",
    # trans
    "Change_spatial_STM_1.0_-1_2.0_1000_EfficientNet-B3": "tmp",
    "Change_spatial_STM_1.0_-1_4.0_1000_EfficientNet-B3": "tmp",
    "Change_spatial_STM_1.0_-1_6.0_1000_EfficientNet-B3": "tmp",
    "Change_spatial_STM_1.0_-1_8.0_1000_EfficientNet-B3": "tmp",
    "Change_spatial_STM_1.0_-1_10.0_1000_EfficientNet-B3": "tmp",
    "Change_spatial_STM_1.0_-1_12.0_1000_EfficientNet-B3": "tmp",


}


for m in model_dict:
    res_dict[m + '_label'] = pd.read_csv(
        f'./result/midLog/spatial/exp1_spatial_STM/{m}.csv', header=None, index_col=None, sep=' '
    ).to_numpy()[:, 0]
    res_dict[m + '_proba'] = pd.read_csv(
        f'./result/midLog/spatial/exp1_spatial_STM/{m}.csv', header=None, index_col=None, sep=' '
    ).to_numpy()[:, 2]
    res_df = pd.DataFrame(res_dict)

for m in model_dict_change:
    res_dict_change[m + '_label'] = pd.read_csv(
        f'./result/midLog/spatial/exp1_spatial_STM/{m}.csv', header=None, index_col=None, sep=' '
    ).to_numpy()[:, 0]
    res_dict_change[m + '_oriPred'] = pd.read_csv(
        f'./result/midLog/spatial/exp1_spatial_STM/{m}.csv', header=None, index_col=None, sep=' '
    ).to_numpy()[:, 1]
    res_dict_change[m + '_advPred'] = pd.read_csv(
        f'./result/midLog/spatial/exp1_spatial_STM/{m}.csv', header=None, index_col=None, sep=' '
    ).to_numpy()[:, 2]

    res_df_change = pd.DataFrame(res_dict_change)


# %% spatial STM(rotate)+BO


mdb1 = [
    "clean_ResNet18",
    "spatial_STM_30.0_1000_1.0_-1_ResNet18",
    "spatial_STM_60.0_1000_1.0_-1_ResNet18",
    "spatial_STM_90.0_1000_1.0_-1_ResNet18",
    "spatial_STM_120.0_1000_1.0_-1_ResNet18",
    "spatial_STM_150.0_1000_1.0_-1_ResNet18",
    "spatial_STM_180.0_1000_1.0_-1_ResNet18",

]

mdb2 = [
    "clean_ResNet50",
    "spatial_STM_30.0_1000_1.0_-1_ResNet50",
    "spatial_STM_60.0_1000_1.0_-1_ResNet50",
    "spatial_STM_90.0_1000_1.0_-1_ResNet50",
    "spatial_STM_120.0_1000_1.0_-1_ResNet50",
    "spatial_STM_150.0_1000_1.0_-1_ResNet50",
    "spatial_STM_180.0_1000_1.0_-1_ResNet50",
]

mdb3 = [
    "clean_SE-ResNet50Xt",
    "spatial_STM_30.0_1000_1.0_-1_SE-ResNet50Xt",
    "spatial_STM_60.0_1000_1.0_-1_SE-ResNet50Xt",
    "spatial_STM_90.0_1000_1.0_-1_SE-ResNet50Xt",
    "spatial_STM_120.0_1000_1.0_-1_SE-ResNet50Xt",
    "spatial_STM_150.0_1000_1.0_-1_SE-ResNet50Xt",
    "spatial_STM_180.0_1000_1.0_-1_SE-ResNet50Xt",
]

mdb4 = [
    "clean_VGG16",
    "spatial_STM_30.0_1000_1.0_-1_VGG16",
    "spatial_STM_60.0_1000_1.0_-1_VGG16",
    "spatial_STM_90.0_1000_1.0_-1_VGG16",
    "spatial_STM_120.0_1000_1.0_-1_VGG16",
    "spatial_STM_150.0_1000_1.0_-1_VGG16",
    "spatial_STM_180.0_1000_1.0_-1_VGG16",
]

mdb5 = [
    "clean_EfficientNet-B3",
    "spatial_STM_30.0_1000_1.0_-1_EfficientNet-B3",
    "spatial_STM_60.0_1000_1.0_-1_EfficientNet-B3",
    "spatial_STM_90.0_1000_1.0_-1_EfficientNet-B3",
    "spatial_STM_120.0_1000_1.0_-1_EfficientNet-B3",
    "spatial_STM_150.0_1000_1.0_-1_EfficientNet-B3",
    "spatial_STM_180.0_1000_1.0_-1_EfficientNet-B3",

]


md1 = [mdb1, mdb2, mdb3, mdb4, mdb5]
model_names = ["ResNet18", "ResNet50", "VGG16","SE-ResNet50Xt", "EfficientNet-B3"]


#  ===============================================Accuracy
for i, mb in enumerate(md1):
    plot_diff_rotate_acc(
        model_names[i],
        y_label="Accuracy",
        x_label="Rotate range",
        all_accuracy=caculate_all_accuracy(mb, res_df),
        rotate_range=[0, 30, 60, 90, 120, 150, 180],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
# plt.title('STM Attack', fontsize=18)
plt.savefig("./result/final_result/stmRotate_acc.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()


# =============================================== S+ Accuracy (S+表示正确分类的样本)
for i, mb in enumerate(md1):
    plot_diff_rotate_acc(
        model_names[i],
        y_label="S+ Accuracy",
        x_label="Rotate range",
        all_accuracy=caculate_sAdd_accuracy(mb[1:], res_df_change),
        rotate_range=[0, 30, 60, 90, 120, 150, 180],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
# plt.title('STM Attack', fontsize=18)
plt.savefig("./result/final_result/S+_stmRotate_acc.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()


# ===============================================Rate of invariance

# 绘制不变性指标
for (i, mb) in enumerate(md1):
    plot_diff_rotate_roi(
        model_names[i],
        caculate_all_roi(mb[1:], res_df_change),
        rotate_range=[0, 30, 60, 90, 120, 150, 180],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
# plt.title('STM Attack', fontsize=18)
plt.savefig("./result/final_result/stmRotate_invariance.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()

# %% spatial STM(transition)+BO


mdb6 = [
    "clean_ResNet18",
    # trans
    "spatial_STM_1.0_-1_2.0_1000_ResNet18",
    "spatial_STM_1.0_-1_4.0_1000_ResNet18",
    "spatial_STM_1.0_-1_6.0_1000_ResNet18",
    "spatial_STM_1.0_-1_8.0_1000_ResNet18",
    "spatial_STM_1.0_-1_10.0_1000_ResNet18",
    "spatial_STM_1.0_-1_12.0_1000_ResNet18",
]

mdb7 = [
    "clean_ResNet50",
    # trans
    "spatial_STM_1.0_-1_2.0_1000_ResNet50",
    "spatial_STM_1.0_-1_4.0_1000_ResNet50",
    "spatial_STM_1.0_-1_6.0_1000_ResNet50",
    "spatial_STM_1.0_-1_8.0_1000_ResNet50",
    "spatial_STM_1.0_-1_10.0_1000_ResNet50",
    "spatial_STM_1.0_-1_12.0_1000_ResNet50",
]

mdb8 = [
    "clean_SE-ResNet50Xt",
    # trans
    "spatial_STM_1.0_-1_2.0_1000_SE-ResNet50Xt",
    "spatial_STM_1.0_-1_4.0_1000_SE-ResNet50Xt",
    "spatial_STM_1.0_-1_6.0_1000_SE-ResNet50Xt",
    "spatial_STM_1.0_-1_8.0_1000_SE-ResNet50Xt",
    "spatial_STM_1.0_-1_10.0_1000_SE-ResNet50Xt",
    "spatial_STM_1.0_-1_12.0_1000_SE-ResNet50Xt",
]

mdb9 = [
    "clean_VGG16",
    # trans
    "spatial_STM_1.0_-1_2.0_1000_VGG16",
    "spatial_STM_1.0_-1_4.0_1000_VGG16",
    "spatial_STM_1.0_-1_6.0_1000_VGG16",
    "spatial_STM_1.0_-1_8.0_1000_VGG16",
    "spatial_STM_1.0_-1_10.0_1000_VGG16",
    "spatial_STM_1.0_-1_12.0_1000_VGG16",

]

mdb10 = [
    "clean_EfficientNet-B3",
    # trans
    "spatial_STM_1.0_-1_2.0_1000_EfficientNet-B3",
    "spatial_STM_1.0_-1_4.0_1000_EfficientNet-B3",
    "spatial_STM_1.0_-1_6.0_1000_EfficientNet-B3",
    "spatial_STM_1.0_-1_8.0_1000_EfficientNet-B3",
    "spatial_STM_1.0_-1_10.0_1000_EfficientNet-B3",
    "spatial_STM_1.0_-1_12.0_1000_EfficientNet-B3",

]


md2 = [mdb6, mdb7, mdb8, mdb9, mdb10]
model_names = ["ResNet18", "ResNet50", "VGG16", "SE-ResNet50Xt", "EfficientNet-B3"]


# =============================================== Accuracy
for i, mb in enumerate(md2):
    plot_diff_trans_acc(
        model_names[i],
        y_label="Accuracy",
        x_label="Transition range",
        all_accuracy=caculate_all_accuracy(mb, res_df),
        trans_range=[0, 2, 4, 6, 8, 10, 12],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
# plt.title('STM Attack', fontsize=18)
plt.savefig("./result/final_result/stmTrans_acc.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()

# =============================================== S+ Accuracy (S+表示正确分类的样本)
for i, mb in enumerate(md2):
    plot_diff_trans_acc(
        model_names[i],
        y_label="S+ Accuracy",
        x_label="Transition range",
        all_accuracy=caculate_sAdd_accuracy(mb[1:], res_df_change),
        trans_range=[0, 2, 4, 6, 8, 10, 12],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
# plt.title('STM Attack', fontsize=18)
plt.savefig("./result/final_result/S+_stmTrans_acc.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()


# =============================================== Rate of invariance
for (i, mb) in enumerate(md2):
    plot_diff_trans_roi(
        model_names[i],
        caculate_all_roi(mb[1:], res_df_change),
        trans_range=[0, 2, 4, 6, 8, 10, 12],
        color=color_dict["Prob_clean_"+model_names[i]],
    )
# plt.title('STM Attack', fontsize=18)
plt.savefig("./result/final_result/stmTrans_invariance.eps",
            dpi=600, format='eps',  bbox_inches='tight')
plt.show()

# %%