File size: 5,178 Bytes
3d1f2c9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 |
import numpy as np
import zipfile
import argparse
import json
import sys
from tqdm import tqdm
sys.path.append("sn_calibration")
sys.path.append("sn_calibration/src")
from evaluate_camera import get_polylines, scale_points, evaluate_camera_prediction
from evaluate_extremities import mirror_labels
def evaluate(gt_zip, prediction_zip, width=960, height=540):
gt_archive = zipfile.ZipFile(gt_zip, 'r')
prediction_archive = zipfile.ZipFile(prediction_zip, 'r')
gt_jsons = gt_archive.namelist()
accuracies = []
precisions = []
recalls = []
dict_errors = {}
per_class_confusion_dict = {}
total_frames = 0
missed = 0
for gt_json in tqdm(gt_jsons):
#split, name = gt_json.split("/")
#pred_name = f"{split}/camera_{name}"
pred_name = f"camera_{gt_json}"
total_frames += 1
if pred_name not in prediction_archive.namelist():
missed += 1
continue
prediction = prediction_archive.read(pred_name)
prediction = json.loads(prediction.decode("utf-8"))
gt = gt_archive.read(gt_json)
gt = json.loads(gt.decode('utf-8'))
line_annotations = scale_points(gt, width, height)
img_groundtruth = line_annotations
img_prediction = get_polylines(prediction, width, height,
sampling_factor=0.9)
confusion1, per_class_conf1, reproj_errors1 = evaluate_camera_prediction(img_prediction,
img_groundtruth,
5)
confusion2, per_class_conf2, reproj_errors2 = evaluate_camera_prediction(img_prediction,
mirror_labels(img_groundtruth),
5)
accuracy1, accuracy2 = 0., 0.
if confusion1.sum() > 0:
accuracy1 = confusion1[0, 0] / confusion1.sum()
if confusion2.sum() > 0:
accuracy2 = confusion2[0, 0] / confusion2.sum()
if accuracy1 > accuracy2:
accuracy = accuracy1
confusion = confusion1
per_class_conf = per_class_conf1
reproj_errors = reproj_errors1
else:
accuracy = accuracy2
confusion = confusion2
per_class_conf = per_class_conf2
reproj_errors = reproj_errors2
accuracies.append(accuracy)
if confusion[0, :].sum() > 0:
precision = confusion[0, 0] / (confusion[0, :].sum())
precisions.append(precision)
if (confusion[0, 0] + confusion[1, 0]) > 0:
recall = confusion[0, 0] / (confusion[0, 0] + confusion[1, 0])
recalls.append(recall)
for line_class, errors in reproj_errors.items():
if line_class in dict_errors.keys():
dict_errors[line_class].extend(errors)
else:
dict_errors[line_class] = errors
for line_class, confusion_mat in per_class_conf.items():
if line_class in per_class_confusion_dict.keys():
per_class_confusion_dict[line_class] += confusion_mat
else:
per_class_confusion_dict[line_class] = confusion_mat
results = {}
results["completeness"] = (total_frames - missed) / total_frames
results["meanRecall"] = np.mean(recalls)
results["meanPrecision"] = np.mean(precisions)
results["meanAccuracies"] = np.mean(accuracies)
results["finalScore"] = results["completeness"] * results["meanAccuracies"]
for line_class, confusion_mat in per_class_confusion_dict.items():
class_accuracy = confusion_mat[0, 0] / confusion_mat.sum()
class_recall = confusion_mat[0, 0] / (confusion_mat[0, 0] + confusion_mat[1, 0])
class_precision = confusion_mat[0, 0] / (confusion_mat[0, 0] + confusion_mat[0, 1])
results[f"{line_class}Precision"] = class_precision
results[f"{line_class}Recall"] = class_recall
results[f"{line_class}Accuracy"] = class_accuracy
return results
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Evaluation camera calibration task')
parser.add_argument('-s', '--soccernet', default="/home/fmg/data/SN23/calibration-2023/test_secret.zip", type=str,
help='Path to the zip groundtruth folder')
parser.add_argument('-p', '--prediction', default="/home/fmg/results/SN23-tests/test.zip",
required=False, type=str,
help="Path to the zip prediction folder")
parser.add_argument('--width', type=int, default=960)
parser.add_argument('--height', type=int, default=540)
args = parser.parse_args()
results = evaluate(args.soccernet, args.prediction, args.width, args.height)
for key in results.keys():
print(f"{key}: {results[key]}")
|