1. #ifndef READER_H
    2. #define READER_H
    3. #include "Setting.h"
    4. #include "Triple.h"
    5. #include <cstdlib>
    6. #include <algorithm>
    7. #include <iostream>
    8. #include <cmath>
    10. INT *freqRel, *freqEnt;
    11. INT *lefHead, *rigHead;
    12. INT *lefTail, *rigTail;
    13. INT *lefRel, *rigRel;
    14. REAL *left_mean, *right_mean;
    15. REAL *prob;
    17. Triple *trainList;
    18. Triple *trainHead;
    19. Triple *trainTail;
    20. Triple *trainRel;
    22. INT *testLef, *testRig;
    23. INT *validLef, *validRig;
    25. extern "C" __declspec(dllexport)
    26. // 不知道干什么用的，提前确定阈值？
    27. void importProb(REAL temp){
    28.     if (prob != NULL)
    29.         free(prob);
    30.     FILE *fin;
    31.     fin = fopen((inPath + "kl_prob.txt").c_str(), "r");
    32.     printf("Current temperature:%f\n", temp);
    33.     prob = (REAL *)calloc(relationTotal * (relationTotal - 1), sizeof(REAL));
    34.     INT tmp;
    35.     for (INT i = 0; i < relationTotal * (relationTotal - 1); ++i){
    36.         tmp = fscanf(fin, "%f", &prob[i]);
    37.     }
    38.     REAL sum = 0.0;
    39.     for (INT i = 0; i < relationTotal; ++i) {
    40.         for (INT j = 0; j < relationTotal-1; ++j){
    41.             REAL tmp = exp(-prob[i * (relationTotal - 1) + j] / temp);
    42.             sum += tmp;
    43.             prob[i * (relationTotal - 1) + j] = tmp;
    44.         }
    45.         for (INT j = 0; j < relationTotal-1; ++j){
    46.             prob[i*(relationTotal-1)+j] /= sum;
    47.         }
    48.         sum = 0;
    49.     }
    50.     fclose(fin);
    51. }
    53. extern "C" __declspec(dllexport)
    54. void importTrainFiles() {
    56.     printf("The toolkit is importing datasets.\n");
    57.     FILE *fin;
    58.     int tmp;
    60.     // 打开关系数据集，第一行存有所有关系的数量
    61.     if (rel_file == "")
    62.         fin = fopen((inPath + "relation2id.txt").c_str(), "r");
    63.     else
    64.         fin = fopen(rel_file.c_str(), "r");
    65.     tmp = fscanf(fin, "%ld", &relationTotal);
    66.     printf("The total of relations is %ld.\n", relationTotal);
    67.     fclose(fin);
    69.     // 打开实体数据集，第一行存有所有实体的数量
    70.     if (ent_file == "")
    71.         fin = fopen((inPath + "entity2id.txt").c_str(), "r");
    72.     else
    73.         fin = fopen(ent_file.c_str(), "r");
    74.     tmp = fscanf(fin, "%ld", &entityTotal);
    75.     printf("The total of entities is %ld.\n", entityTotal);
    76.     fclose(fin);
    78.     // 打开三元组数据集，第一行存有所有三元组的数量
    79.     if (train_file == "")
    80.         fin = fopen((inPath + "train2id.txt").c_str(), "r");
    81.     else
    82.         fin = fopen(train_file.c_str(), "r");
    83.     tmp = fscanf(fin, "%ld", &trainTotal);
    85.     // train开头的都分配与训练集一样的长度
    86.     trainList = (Triple *)calloc(trainTotal, sizeof(Triple));
    87.     trainHead = (Triple *)calloc(trainTotal, sizeof(Triple));
    88.     trainTail = (Triple *)calloc(trainTotal, sizeof(Triple));
    89.     trainRel = (Triple *)calloc(trainTotal, sizeof(Triple));
    91.     // freq开头的长度与实体/关系数量一致，用于存储每个实体/关系出现的频率
    92.     freqRel = (INT *)calloc(relationTotal, sizeof(INT));
    93.     freqEnt = (INT *)calloc(entityTotal, sizeof(INT));
    95.     // 将三元组数据集中的三元组存在trainList中
    96.     for (INT i = 0; i < trainTotal; i++) {
    97.         tmp = fscanf(fin, "%ld", &trainList[i].h);
    98.         tmp = fscanf(fin, "%ld", &trainList[i].t);
    99.         tmp = fscanf(fin, "%ld", &trainList[i].r);
    100.     }
    101.     fclose(fin);
    103.     // 按照头实体ID的大小，对trainList进行排序，若头实体ID相等，则判断关系ID；若头实体、关系都相等，则判断尾实体ID；并以升序的方式排列
    104.     std::sort(trainList, trainList + trainTotal, Triple::cmp_head);
    106.     // 遍历一遍三元组训练集，统计每个实体及关系出现的频率，并把相同的内容复制到trainHead、trainTail、trainRel中
    107.     tmp = trainTotal; trainTotal = 1;
    108.     trainHead[0] = trainTail[0] = trainRel[0] = trainList[0];
    109.     freqEnt[trainList[0].t] += 1;
    110.     freqEnt[trainList[0].h] += 1;
    111.     freqRel[trainList[0].r] += 1;
    112.     for (INT i = 1; i < tmp; i++)
    113.         if (trainList[i].h != trainList[i - 1].h || trainList[i].r != trainList[i - 1].r || trainList[i].t != trainList[i - 1].t) {
    114.             trainHead[trainTotal] = trainTail[trainTotal] = trainRel[trainTotal] = trainList[trainTotal] = trainList[i];
    115.             trainTotal++;
    116.             freqEnt[trainList[i].t]++;
    117.             freqEnt[trainList[i].h]++;
    118.             freqRel[trainList[i].r]++;
    119.         }
    121.     // 现在为什么要额外三个train开头的列表就明了了，原来trainHead、trainTail、trainRel分别是按照三者ID大小来排序的列表，以便进行二分查找
    122.     std::sort(trainHead, trainHead + trainTotal, Triple::cmp_head);
    123.     std::sort(trainTail, trainTail + trainTotal, Triple::cmp_tail);
    124.     std::sort(trainRel, trainRel + trainTotal, Triple::cmp_rel);
    125.     printf("The total of train triples is %ld.\n", trainTotal);
    127.     // 初始化这六个列表是因为三元组列表中同一实体或关系可能同时出现多次，那么我们希望快速确定实体或关系在排序好的三元组列表中出现的位置
    128.     lefHead = (INT *)calloc(entityTotal, sizeof(INT));
    129.     rigHead = (INT *)calloc(entityTotal, sizeof(INT));
    130.     lefTail = (INT *)calloc(entityTotal, sizeof(INT));
    131.     rigTail = (INT *)calloc(entityTotal, sizeof(INT));
    132.     lefRel = (INT *)calloc(entityTotal, sizeof(INT));
    133.     rigRel = (INT *)calloc(entityTotal, sizeof(INT));
    135.     // 对rigHead、rigTail、rigRel数组全部初始化为-1
    136.     memset(rigHead, -1, sizeof(INT)*entityTotal);
    137.     memset(rigTail, -1, sizeof(INT)*entityTotal);
    138.     memset(rigRel, -1, sizeof(INT)*entityTotal);
    141.     // lefTail存储的是对应实体在trainTail有序列表中第一个下标位置，相反rigTail存储的是对应实体在trainTail有序列表中最后一个下标位置
    142.     // 其他如Head、Rel同理
    143.     for (INT i = 1; i < trainTotal; i++) {
    144.         if (trainTail[i].t != trainTail[i - 1].t) {
    145.             rigTail[trainTail[i - 1].t] = i - 1;
    146.             lefTail[trainTail[i].t] = i;
    147.         }
    148.         if (trainHead[i].h != trainHead[i - 1].h) {
    149.             rigHead[trainHead[i - 1].h] = i - 1;
    150.             lefHead[trainHead[i].h] = i;
    151.         }
    152.         if (trainRel[i].h != trainRel[i - 1].h) {
    153.             rigRel[trainRel[i - 1].h] = i - 1;
    154.             lefRel[trainRel[i].h] = i;
    155.         }
    156.     }
    157.     lefHead[trainHead[0].h] = 0;
    158.     rigHead[trainHead[trainTotal - 1].h] = trainTotal - 1;
    159.     lefTail[trainTail[0].t] = 0;
    160.     rigTail[trainTail[trainTotal - 1].t] = trainTotal - 1;
    161.     lefRel[trainRel[0].h] = 0;
    162.     rigRel[trainRel[trainTotal - 1].h] = trainTotal - 1;
    164.     // 以下内容只有在进行伯努利采样时才用得到
    165.     left_mean = (REAL *)calloc(relationTotal,sizeof(REAL));
    166.     right_mean = (REAL *)calloc(relationTotal,sizeof(REAL));
    167.     // 遍历所有entity
    168.     for (INT i = 0; i < entityTotal; i++) {
    169.         for (INT j = lefHead[i] + 1; j <= rigHead[i]; j++)
    170.             // 相邻训练头实体对应的关系不等情况下，对头实体的入边+1
    171.             // 其实由于trainHead是根据Head与Relation一起排列的，所以Head相同的一段其relation也是有序的，所以这个操作就是将在当前Head为指定Entity时出现过的关系次数统一加一
    172.             if (trainHead[j].r != trainHead[j - 1].r)
    173.                 left_mean[trainHead[j].r] += 1.0;
    174.         if (lefHead[i] <= rigHead[i])
    175.             left_mean[trainHead[lefHead[i]].r] += 1.0;
    176.         for (INT j = lefTail[i] + 1; j <= rigTail[i]; j++)
    177.             // 如果左实体的大小小于等于右实体的大小，则以左实体对应的出边+1
    178.             // 与前者相似，只不过这次看的是尾实体，即计算的是关系的出度
    179.             if (trainTail[j].r != trainTail[j - 1].r)
    180.                 right_mean[trainTail[j].r] += 1.0;
    181.         if (lefTail[i] <= rigTail[i])
    182.             right_mean[trainTail[lefTail[i]].r] += 1.0;
    183.     }
    185.     // 左均值即关系的个数除以关系的入边数量，右均值即关系的个数除以关系的出边数量
    186.     for (INT i = 0; i < relationTotal; i++) {
    187.         left_mean[i] = freqRel[i] / left_mean[i];
    188.         right_mean[i] = freqRel[i] / right_mean[i];
    189.     }
    190. }
    192. Triple *testList;
    193. Triple *validList;
    194. Triple *tripleList;
    196. extern "C" __declspec(dllexport)
    197. void importTestFiles() {
    198.     FILE *fin;
    199.     INT tmp;
    201.     if (rel_file == "")
    202.         fin = fopen((inPath + "relation2id.txt").c_str(), "r");
    203.     else
    204.         fin = fopen(rel_file.c_str(), "r");
    205.     tmp = fscanf(fin, "%ld", &relationTotal);
    206.     fclose(fin);
    208.     if (ent_file == "")
    209.         fin = fopen((inPath + "entity2id.txt").c_str(), "r");
    210.     else
    211.         fin = fopen(ent_file.c_str(), "r");
    212.     tmp = fscanf(fin, "%ld", &entityTotal);
    213.     fclose(fin);
    215.     FILE* f_kb1, * f_kb2, * f_kb3;
    216.     if (train_file == "")
    217.         f_kb2 = fopen((inPath + "train2id.txt").c_str(), "r");
    218.     else
    219.         f_kb2 = fopen(train_file.c_str(), "r");
    220.     if (test_file == "")
    221.         f_kb1 = fopen((inPath + "test2id.txt").c_str(), "r");
    222.     else
    223.         f_kb1 = fopen(test_file.c_str(), "r");
    224.     if (valid_file == "")
    225.         f_kb3 = fopen((inPath + "valid2id.txt").c_str(), "r");
    226.     else
    227.         f_kb3 = fopen(valid_file.c_str(), "r");
    228.     tmp = fscanf(f_kb1, "%ld", &testTotal);
    229.     tmp = fscanf(f_kb2, "%ld", &trainTotal);
    230.     tmp = fscanf(f_kb3, "%ld", &validTotal);
    231.     tripleTotal = testTotal + trainTotal + validTotal;
    232.     testList = (Triple *)calloc(testTotal, sizeof(Triple));
    233.     validList = (Triple *)calloc(validTotal, sizeof(Triple));
    234.     tripleList = (Triple *)calloc(tripleTotal, sizeof(Triple));
    235.     for (INT i = 0; i < testTotal; i++) {
    236.         tmp = fscanf(f_kb1, "%ld", &testList[i].h);
    237.         tmp = fscanf(f_kb1, "%ld", &testList[i].t);
    238.         tmp = fscanf(f_kb1, "%ld", &testList[i].r);
    239.         tripleList[i] = testList[i];
    240.     }
    241.     for (INT i = 0; i < trainTotal; i++) {
    242.         tmp = fscanf(f_kb2, "%ld", &tripleList[i + testTotal].h);
    243.         tmp = fscanf(f_kb2, "%ld", &tripleList[i + testTotal].t);
    244.         tmp = fscanf(f_kb2, "%ld", &tripleList[i + testTotal].r);
    245.     }
    246.     for (INT i = 0; i < validTotal; i++) {
    247.         tmp = fscanf(f_kb3, "%ld", &tripleList[i + testTotal + trainTotal].h);
    248.         tmp = fscanf(f_kb3, "%ld", &tripleList[i + testTotal + trainTotal].t);
    249.         tmp = fscanf(f_kb3, "%ld", &tripleList[i + testTotal + trainTotal].r);
    250.         validList[i] = tripleList[i + testTotal + trainTotal];
    251.     }
    252.     fclose(f_kb1);
    253.     fclose(f_kb2);
    254.     fclose(f_kb3);
    256.     std::sort(tripleList, tripleList + tripleTotal, Triple::cmp_head);
    257.     std::sort(testList, testList + testTotal, Triple::cmp_rel2);
    258.     std::sort(validList, validList + validTotal, Triple::cmp_rel2);
    259.     printf("The total of test triples is %ld.\n", testTotal);
    260.     printf("The total of valid triples is %ld.\n", validTotal);
    262.     testLef = (INT *)calloc(relationTotal, sizeof(INT));
    263.     testRig = (INT *)calloc(relationTotal, sizeof(INT));
    264.     memset(testLef, -1, sizeof(INT) * relationTotal);
    265.     memset(testRig, -1, sizeof(INT) * relationTotal);
    266.     for (INT i = 1; i < testTotal; i++) {
    267.     if (testList[i].r != testList[i-1].r) {
    268.         testRig[testList[i-1].r] = i - 1;
    269.         testLef[testList[i].r] = i;
    270.     }
    271.     }
    272.     testLef[testList[0].r] = 0;
    273.     testRig[testList[testTotal - 1].r] = testTotal - 1;
    275.     validLef = (INT *)calloc(relationTotal, sizeof(INT));
    276.     validRig = (INT *)calloc(relationTotal, sizeof(INT));
    277.     memset(validLef, -1, sizeof(INT)*relationTotal);
    278.     memset(validRig, -1, sizeof(INT)*relationTotal);
    279.     for (INT i = 1; i < validTotal; i++) {
    280.     if (validList[i].r != validList[i-1].r) {
    281.         validRig[validList[i-1].r] = i - 1;
    282.         validLef[validList[i].r] = i;
    283.     }
    284.     }
    285.     validLef[validList[0].r] = 0;
    286.     validRig[validList[validTotal - 1].r] = validTotal - 1;
    287. }
    289. INT* head_lef;
    290. INT* head_rig;
    291. INT* tail_lef;
    292. INT* tail_rig;
    293. INT* head_type;
    294. INT* tail_type;
    296. extern "C" __declspec(dllexport)
    297. void importTypeFiles() {
    299.     head_lef = (INT *)calloc(relationTotal, sizeof(INT));
    300.     head_rig = (INT *)calloc(relationTotal, sizeof(INT));
    301.     tail_lef = (INT *)calloc(relationTotal, sizeof(INT));
    302.     tail_rig = (INT *)calloc(relationTotal, sizeof(INT));
    303.     INT total_lef = 0;
    304.     INT total_rig = 0;
    305.     FILE* f_type = fopen((inPath + "type_constrain.txt").c_str(),"r");
    306.     INT tmp;
    307.     tmp = fscanf(f_type, "%ld", &tmp);
    308.     for (INT i = 0; i < relationTotal; i++) {
    309.         INT rel, tot;
    310.         tmp = fscanf(f_type, "%ld %ld", &rel, &tot);
    311.         for (INT j = 0; j < tot; j++) {
    312.             tmp = fscanf(f_type, "%ld", &tmp);
    313.             total_lef++;
    314.         }
    315.         tmp = fscanf(f_type, "%ld%ld", &rel, &tot);
    316.         for (INT j = 0; j < tot; j++) {
    317.             tmp = fscanf(f_type, "%ld", &tmp);
    318.             total_rig++;
    319.         }
    320.     }
    321.     fclose(f_type);
    322.     head_type = (INT *)calloc(total_lef, sizeof(INT)); 
    323.     tail_type = (INT *)calloc(total_rig, sizeof(INT));
    324.     total_lef = 0;
    325.     total_rig = 0;
    326.     f_type = fopen((inPath + "type_constrain.txt").c_str(),"r");
    327.     tmp = fscanf(f_type, "%ld", &tmp);
    328.     for (INT i = 0; i < relationTotal; i++) {
    329.         INT rel, tot;
    330.         tmp = fscanf(f_type, "%ld%ld", &rel, &tot);
    331.         head_lef[rel] = total_lef;
    332.         for (INT j = 0; j < tot; j++) {
    333.             tmp = fscanf(f_type, "%ld", &head_type[total_lef]);
    334.             total_lef++;
    335.         }
    336.         head_rig[rel] = total_lef;
    337.         std::sort(head_type + head_lef[rel], head_type + head_rig[rel]);
    338.         tmp = fscanf(f_type, "%ld%ld", &rel, &tot);
    339.         tail_lef[rel] = total_rig;
    340.         for (INT j = 0; j < tot; j++) {
    341.             tmp = fscanf(f_type, "%ld", &tail_type[total_rig]);
    342.             total_rig++;
    343.         }
    344.         tail_rig[rel] = total_rig;
    345.         std::sort(tail_type + tail_lef[rel], tail_type + tail_rig[rel]);
    346.     }
    347.     fclose(f_type);
    348. }
    351. #endif