1. from sklearn.feature_extraction.text import TfidfVectorizer
    2. import os
    3. from sklearn.model_selection import train_test_split
    4. from sklearn.linear_model import LogisticRegression
    5. from sklearn import metrics
    6. import urllib
    8. import matplotlib.pyplot as plt
    9. #import sys
    10. #reload(sys)
    11. #sys.setdefaultencoding('utf8')
    14. def loadFile(name):
    15.     directory = str(os.getcwd())
    16.     filepath = os.path.join(directory, name)
    17.     with open(filepath,'r', encoding='utf8') as f:
    18.         data = f.readlines()
    19.     data = list(set(data))
    20.     result = []
    21.     for d in data:
    22.         d = str(urllib.parse.unquote(d))   #converting url encoded data to simple string
    23.         result.append(d)
    24.     return result
    27. badQueries=loadFile("d:/pythonLab/badqueries.txt")
    28. validQueries=loadFile("d:/pythonLab/goodqueries.txt")
    30. badQueries = list(set(badQueries))
    31. validQueries = list(set(validQueries))
    32. allQueries = badQueries + validQueries
    33. yBad = [1 for i in range(0, len(badQueries))]  #labels, 1 for malicious and 0 for clean
    34. yGood = [0 for i in range(0, len(validQueries))]
    35. y = yBad + yGood
    36. queries = allQueries
    38. ## min_df代表忽略低于频率为0的词条，analyzer定义特征为词（word）或n-gram字符，如果传递给它的调用被用于抽取未处理输入源文件的特征序列，sublinear_tf代表应用线性缩放TF，例如，使用1+log(tf)覆盖tf，ngram_range参数要提取的n-gram的n-values的下限和上限范围，在min_n <= n <= max_n区间的n的全部值
    39. vectorizer = TfidfVectorizer(min_df = 0.0, analyzer="char", sublinear_tf=True, ngram_range=(1,3)) #converting data to vectors
    40. X = vectorizer.fit_transform(queries)
    42. X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) #splitting data
    44. badCount = len(badQueries)
    45. validCount = len(validQueries)
    48. ## class_weight ： dict或'balanced'，默认值：无
    49. ## 与表单中的类关联的权重。如果没有给出，所有课程都应该有一个重量。{class_label: weight}
    50. ## “平衡”模式使用y的值自动调整与输入数据中的类频率成反比的权重。n_samples / (n_classes * np.bincount(y))
    51. ## 请注意，如果指定了sample_weight，这些权重将与sample_weight（通过fit方法传递）相乘。
    52. lgs = LogisticRegression(class_weight={1: 2 * validCount / badCount, 0: 1.0}) # class_weight='balanced')
    53. lgs.fit(X_train, y_train) #training our model
    55. ##############
    56. # Evaluation #
    57. ##############
    59. predicted = lgs.predict(X_test)
    61. fpr, tpr, _ = metrics.roc_curve(y_test, (lgs.predict_proba(X_test)[:, 1]))
    62. auc = metrics.auc(fpr, tpr)
    64. print("Bad samples: %d" % badCount)
    65. print("Good samples: %d" % validCount)
    66. print("Baseline Constant negative: %.6f" % (validCount / (validCount + badCount)))
    67. print("------------")
    68. print("Accuracy: %f" % lgs.score(X_test, y_test))  #checking the accuracy
    69. print("Precision: %f" % metrics.precision_score(y_test, predicted))
    70. print("Recall: %f" % metrics.recall_score(y_test, predicted))
    71. print("F1-Score: %f" % metrics.f1_score(y_test, predicted))
    72. print("AUC: %f" % auc)