article-matching/linkEval.py at master · newsgac/article-matching · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
#!/usr/bin/env python3
"""
    linkEval.py: evaluate metadata to article text links based on gold data
    usage: linkEval.py -g goldFile -t testFile
    20190107 erikt(at)xs4all.nl
"""

import csv
import getopt
import re
import sys

COMMAND = sys.argv.pop(0)
CSVPATTERN = r"\.csv$"
FIELDARTICLEID = "Artikel ID"
FIELDARTICLETEXT = "Identifier"
FIELDPREDICTION = "Prediction"
NONE = "None"
OPTIONSTRING = "g:t:"
OPTIONG = "-g"
OPTIONT = "-t"
SEPARATORCOMMA = ","
SEPARATORTAB = "\t"

def processArgs(argv):
    goldFileName = ""
    testFileName = ""
    try:
        opts,args = getopt.getopt(argv,OPTIONSTRING)
    except Exception as e:
        sys.exit(COMMAND+": error: "+str(e))
    for opt,arg in opts:
        if opt == OPTIONG: goldFileName = arg
        elif opt == OPTIONT: testFileName = arg
    return(goldFileName,testFileName)

def readDataCsv(fileName):
    data = {}
    try:
        inFile = open(fileName,"r")
        csvReader = csv.DictReader(inFile,delimiter=SEPARATORCOMMA)
        for row in csvReader:
            try:
                data[row[FIELDARTICLEID]] = row[FIELDARTICLETEXT]
            except Exception as e:
                sys.exit(COMMAND+": error: "+str(e))
        inFile.close()
    except Exception as e: sys.exit(COMMAND+": error: "+str(e))
    return(data)

def readDataTsv(fileName):
    data = {}
    try:
        inFile = open(fileName,"r")
        csvReader = csv.DictReader(inFile,delimiter=SEPARATORTAB)
        for row in csvReader:
            try:
                data[row[FIELDARTICLEID]] = row[FIELDPREDICTION]
            except Exception as e:
                sys.exit(COMMAND+": error: "+str(e))
        inFile.close()
    except Exception as e: sys.exit(COMMAND+": error: "+str(e))
    return(data)

def readData(fileName):
    if re.search(CSVPATTERN,fileName): data = readDataCsv(fileName)
    else: data = readDataTsv(fileName)
    return(data)

def compareData(goldData,testData):
    correct,wrong,noLabel,unknown,missed = 0,0,0,0,0
    for testId in testData:
        if not testId in goldData: unknown += 1
        elif testData[testId] == NONE: noLabel += 1
        elif testData[testId] == goldData[testId]: correct += 1
        else: wrong += 1
    for goldId in goldData:
        if not goldId in testData: missed += 1
    return(correct,wrong,noLabel,unknown,missed)

def printLine(key,total,freq):
    print("{0:11s}".format(key+":"),end="")
    if total == "": print("{0:4s}".format(total),end="")
    else: print("{0:4d}".format(total),end="")
    if freq != "": print(" {0:4.1f}%".format(100.0*freq),end="")
    print("\n",end="")

def printResults(correct,wrong,noLabel,unknown,missed):
    total = correct+wrong+noLabel+unknown
    printLine("articles",total,"")
    printLine("correct",correct,correct/total)
    printLine("wrong",wrong,wrong/total)
    printLine("no label",noLabel,noLabel/total)
    printLine("missed",missed,missed/total)
    printLine("unknown",unknown,unknown/total)
    printLine("precision","",correct/(correct+wrong))
    printLine("recall","",correct/(correct+wrong+noLabel+missed))

def main(argv):
    goldFileName,testFileName = processArgs(argv)
    goldData = readData(goldFileName)
    testData = readData(testFileName)
    correct,wrong,noLabel,unknown,missed = compareData(goldData,testData)
    printResults(correct,wrong,noLabel,unknown,missed)

if __name__ == "__main__":
    sys.exit(main(sys.argv))