在linear regression基础上修改了loss和gradient函数，添加了sigmoid函数，添加了测试集结果输出，acc、精确率、召回率计算、ROC曲线绘制、auc计算等

实验过程：（遇到各种问题后解决的过程）

1.修改线性回归为逻辑回归，主要是增加sigmoid函数、修改了梯度下降的公式，逻辑回归主体迭代过程不变

2.刚开始运行时初始值还是用了上次线性回归的数据，即w=1，b=60，然后得到了曲线方向完全相反的结果，运行测试集发现loss大概在21左右，感觉误差过大后根据测试集loss输出和预测结果的整体loss调整之后将b逐步调小，b=5左右得到比较好的训练结果。

在这之中又发现了过拟合的问题，具体如下：

根据绘制的counter图知，梯度下降的最优结果区间大致在这里：

但如果进一步增大迭代次数，训练集的loss进一步减小，但测试集的loss开始上升，为避免过拟合最终选择初始值w=1，b=5，iter=10000，learningrate=0.001

3.接下来的问题和一个报错有关：

简单来说就是log（0）->无穷的情况，询问老师后增加一个小正数可以解决，随后完善loss函数，这也是第二个问题中图片右上全部是白色的原因

至此，逻辑回归主体部分已经完成，开始着手绘制roc曲线

4.绘制roc曲线调用sklearn的roc_courve函数，计算auc调用auc函数，在调用roc_courve函数时遇到了一点小问题，在先前计算precision和recall时，提前将y_pred进行了四舍五入的处理，导致roc曲线只有一个拐点，查阅资料后发现了这个问题，并修改测试集输出函数如下：

roc曲线问题 修改后的函数

5.最后的运行结果：Acc=0.800，precision=0.8333，recall=0.8333，auc=0.9583，图上是更详细的结果

画图：

import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import roc_curve, auc



xdata = [8., 3., 9., 7., 16., 05., 3., 10., 4., 6.]
ydata = [0, 0, 1, 0, 1, 0, 0, 1, 0, 0]

xtest = [5., 4.5, 9.8, 8., 22., 17., 3., 19., 20, 30]
ytest = [0, 0, 0, 1, 1, 1, 0, 1, 1, 1]
plt.cla()

w_init = 1.
b_init = 5.

w_change = [w_init]
b_change = [b_init]


def sigmoid(x):
    return 1. / (1. + np.exp(-x))


def loss(w, b):
    res = 0
    exp = 1e-8
    for i in range(len(xdata)):
        y_pred = sigmoid(w * xdata[i] + b)
        res += - (1 - ydata[i]) * np.log(1 - y_pred+exp)-ydata[i] * np.log(y_pred+exp)
    return res / len(xdata)


def grad(w, b):
    w_grad = 0
    b_grad = 0
    for i in range(len(xdata)):
        y_pred = sigmoid(w * xdata[i] + b)
        w_grad += -(ydata[i] - y_pred) * xdata[i]
        b_grad += -(ydata[i] - y_pred)
    return w_grad, b_grad


def logistic_regression(w_pred, b_pred, lr=0.001, iter=10000):
    for i in range(iter):
        gradf = grad(w=w_pred, b=b_pred)
        w_pred = float(w_pred - learning_rate * gradf[0])
        b_pred = float(b_pred - learning_rate * gradf[1])
        w_change.append(w_pred)
        b_change.append(b_pred)
    return w_pred, b_pred


def test_set(xtest, ytest, w, b):
    ytest_pred = []
    tp = fp = tn = fn = 0
    for i in range(len(xtest)):
        y_pred = sigmoid(w * xtest[i] + b)
        ytest_pred.append(y_pred)
        if y_pred > 0.5:
            y_pred = 1
        else:
            y_pred = 0
        if ytest[i] == y_pred & y_pred == 1:
            tp += 1
        elif ytest[i] == y_pred & y_pred == 0:
            tn += 1
        elif ytest[i] != y_pred & y_pred == 1:
            fp += 1
        elif ytest[i] != y_pred & y_pred == 0:
            fn += 1
    return ytest_pred, tp, fp, tn, fn

def test_loss(w, b):
    res = 0
    exp = 1e-8
    for i in range(len(xtest)):
        y_pred = sigmoid(w * xtest[i] + b)
        res += - (1 - ytest[i]) * np.log(1 - y_pred+exp)-ytest[i] * np.log(y_pred+exp)
    return res / len(xtest)

print("data set:")
print(xdata)
print(ydata)
plt.figure(1)
plt.scatter(xdata, ydata,label="training set")
plt.title("output")
plt.xlabel("x")
plt.ylabel("y")

# 测试集
print("test set:")
print(xtest)
print(ytest)
plt.scatter(xtest, ytest, marker='v',label="test set")

print("initial input:")
print("w = " + str(w_init))
print("b = " + str(b_init))

learning_rate = 0.001
iter = 10000

print("Logistic Regression: learning_rate = " + str(learning_rate) + ",iteration = " + str(iter))
w_pred, b_pred = logistic_regression(w_pred=w_init, b_pred=b_init, lr=learning_rate, iter=iter)
print("result: w_pred = " + str(w_pred) + ", b_pred = " + str(b_pred))
print("loss = "+str(loss(w_pred,b_pred)))
x = np.arange(0, 40, 0.1)
y = sigmoid(w_pred * x + b_pred)
plt.plot(x, y)
plt.legend()
plt.show()

# ROC曲线
plt.figure(2)
ytest_pred, tp, fp, tn, fn = test_set(xtest, ytest, w_pred, b_pred)
acc = float(tp + tn) / float(len(xtest))
precision = float(tp) / float(tp + fp)
recall = float(tp) / float(tp + fn)
print("y_pred:")
print(ytest_pred)
print("test loss = "+str(test_loss(w_pred,b_pred)))
print("tp = "+str(tp)+" fp = "+str(fp)+" tn = "+str(tn)+" fn = "+str(fn))
print("acc = " + str(acc))
print("precision = " + str(precision))
print("recall = " + str(recall))

fpr, tpr, thresholds = roc_curve(ytest, ytest_pred,pos_label=1)
print("fpr = "+str(fpr))
print("tpr = "+str(tpr))
roc_auc = auc(fpr, tpr)
print("auc = "+str(roc_auc))
plt.plot(fpr, tpr, 'k--', label='ROC (area = {0:.2f})'.format(roc_auc), lw=2)
plt.xlabel("FPR")
plt.ylabel("TPR")
plt.show()



# counter图
plt.figure(3)
x = np.arange(-1, 5, 0.1)
y = np.arange(-20, 20, 0.2)
# 对x,y网格化，得到X,Y
X, Y = np.meshgrid(x, y)
# 等高线
Z = loss(X, Y)
fig, ax = plt.subplots(figsize=(8, 8), dpi=100)
# 画contour图
CS = ax.contourf(X, Y, Z, 100)
# 画轮廓线
CS = ax.contour(X, Y, Z, 100, colors='white')

plt.scatter(w_change, b_change)
plt.title("predict change")
plt.xlabel("w_change")
plt.ylabel("b_change")
plt.tight_layout()
plt.show()