Associative Tracker

dataloader.py

@@ -117,7 +117,7 @@ class YOLOFrameDataset(Dataset):
         self.shape = np.ceil(np.array(shape) * img_size / stride + pad).astype(int) * stride
 
         self.batch_indices = []
-        for i in range(0,n,batch_size): 
+        for i in range(0, n, batch_size): 
             self.batch_indices.append((i, min(n, i+batch_size)))
 
     

inference.py

@@ -239,7 +239,7 @@ def format_predictions(image_shapes, outputs, width, height, gp=None, batch_size
          
     return all_preds, real_width, real_height
 
-def do_confidence_boost(inference, safe_preds, gp=None, batch_size=BATCH_SIZE, conf_power=1, conf_decay=1, verbose=True):
+def do_confidence_boost(inference, safe_preds, gp=None, batch_size=BATCH_SIZE, boost_power=1, boost_decay=1, verbose=True):
     """
     Args:
         frames_dir: a directory containing frames to be evaluated
@@ -254,7 +254,8 @@ def do_confidence_boost(inference, safe_preds, gp=None, batch_size=BATCH_SIZE, c
 
 
     boost_cutoff = 0.01
-    boost_range = math.floor(math.sqrt(1/conf_decay * math.log(conf_power / boost_cutoff)))
+    boost_range = math.floor(math.sqrt(1/boost_decay * math.log(boost_power / boost_cutoff)))
+    boost_scale = boost_power * math.exp(-boost_decay)
     
     outputs = []
     with tqdm(total=len(inference), desc="Running confidence boost", ncols=0, disable=not verbose) as pbar:
@@ -286,7 +287,7 @@ def do_confidence_boost(inference, safe_preds, gp=None, batch_size=BATCH_SIZE, c
                         temp_frame = next_batch[idx - len(infer)]
                     
                     if temp_frame is not None:
-                        boost_frame(safe_frame, temp_frame, dt, power=conf_power, decay=conf_decay)
+                        boost_frame(safe_frame, temp_frame, dt, scale=boost_scale, decay=boost_decay)
 
             pbar.update(1*batch_size)
           
@@ -297,7 +298,7 @@ def boost_frame(safe_frame, base_frame, dt, power=1, decay=1):
     ious = box_iou(boxes, safe_boxes)
     score = torch.matmul(ious, safe_frame[:, 4])
     # score = iou(safe_box, base_box) * confidence(safe_box)
-    base_frame[:, 4] *= 1 + power*(score)*math.exp(-decay*dt*dt)
+    base_frame[:, 4] *= 1 + power*(score)*math.exp(-decay*(dt*dt-1))
     return base_frame
 
 def do_tracking(all_preds, image_meter_width, image_meter_height, gp=None, max_age=MAX_AGE, iou_thres=IOU_THRES, min_hits=MIN_HITS, min_length=MIN_LENGTH, verbose=True): 
@@ -328,6 +329,35 @@ def do_tracking(all_preds, image_meter_width, image_meter_height, gp=None, max_a
 
     return json_data
 
+def do_associative_tracking(raw_detections, image_meter_width, image_meter_height, gp=None, max_age=MAX_AGE, iou_thres=IOU_THRES, min_hits=MIN_HITS, min_length=MIN_LENGTH, verbose=True):
+
+    if (gp): gp(0, "Tracking...")
+
+    print(len(raw_detections))
+
+    # Initialize tracker
+    clip_info = {
+        'start_frame': 0,
+        'end_frame': len(raw_detections),
+        'image_meter_width': image_meter_width,
+        'image_meter_height': image_meter_height
+    }
+    tracker = Tracker(clip_info, args={ 'max_age': max_age, 'min_hits': 0, 'iou_threshold': iou_thres}, min_hits=min_hits)
+    
+    # Run tracking
+    with tqdm(total=len(all_preds), desc="Running tracking", ncols=0, disable=not verbose) as pbar:
+        for i, key in enumerate(sorted(all_preds.keys())):
+            if gp: gp(i / len(all_preds), pbar.__str__())
+            boxes = all_preds[key]
+            if boxes is not None:
+                tracker.update(boxes)
+            else:
+                tracker.update()
+            pbar.update(1)
+
+    json_data = tracker.finalize(min_length=min_length)
+
+    return json_data
 
 @patch('json.encoder.c_make_encoder', None)
 def json_dump_round_float(some_object, out_path, num_digits=4):

lib/fish_eye/associative.py

@@ -0,0 +1,246 @@
+"""
+    SORT: A Simple, Online and Realtime Tracker
+    Copyright (C) 2016-2020 Alex Bewley alex@bewley.ai
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+from filterpy.kalman import KalmanFilter
+import numpy as np
+
+def linear_assignment(cost_matrix):
+    try:
+        import lap
+        _, x, y = lap.lapjv(cost_matrix, extend_cost=True)
+        return np.array([[y[i],i] for i in x if i >= 0]) #
+    except ImportError:
+        from scipy.optimize import linear_sum_assignment
+        x, y = linear_sum_assignment(cost_matrix)
+        return np.array(list(zip(x, y)))
+
+
+def iou_batch(bb_test, bb_gt):                                                                                                   
+    """                                                                                                                      
+    From SORT: Computes IOU between two bboxes in the form [l,t,w,h]                                                         
+    """                                                                                                                      
+    bb_gt = np.expand_dims(bb_gt, 0)                                                                                         
+    bb_test = np.expand_dims(bb_test, 1)                                                                                     
+                                                                                                                            
+    xx1 = np.maximum(bb_test[..., 0], bb_gt[..., 0])                                                                         
+    yy1 = np.maximum(bb_test[..., 1], bb_gt[..., 1])                                                                         
+    xx2 = np.minimum(bb_test[..., 2], bb_gt[..., 2])                                                                         
+    yy2 = np.minimum(bb_test[..., 3], bb_gt[..., 3])                                                                         
+    w = np.maximum(0., xx2 - xx1)                                                                                            
+    h = np.maximum(0., yy2 - yy1)                                                                                            
+    wh = w * h                                                                                                               
+    o = wh / ((bb_test[..., 2] - bb_test[..., 0]) * (bb_test[..., 3] - bb_test[..., 1])                                      
+        + (bb_gt[..., 2] - bb_gt[..., 0]) * (bb_gt[..., 3] - bb_gt[..., 1]) - wh)                                              
+    return(o)  
+
+
+def convert_bbox_to_z(bbox):
+    """
+    Takes a bounding box in the form [x1,y1,x2,y2] and returns z in the form
+        [x,y,s,r] where x,y is the centre of the box and s is the scale/area and r is
+        the aspect ratio
+    """
+    w = bbox[2] - bbox[0]
+    h = bbox[3] - bbox[1]
+    x = bbox[0] + w/2.
+    y = bbox[1] + h/2.
+    s = w * h    #scale is just area
+    r = w / float(h)
+    return np.array([x, y, s, r]).reshape((4, 1))
+
+
+def convert_x_to_bbox(x,score=None):
+    """
+    Takes a bounding box in the centre form [x,y,s,r] and returns it in the form
+        [x1,y1,x2,y2] where x1,y1 is the top left and x2,y2 is the bottom right
+    """
+    w = np.sqrt(x[2] * x[3])
+    h = x[2] / w
+    if(score==None):
+        return np.array([x[0]-w/2.,x[1]-h/2.,x[0]+w/2.,x[1]+h/2.]).reshape((1,4))
+    else:
+        return np.array([x[0]-w/2.,x[1]-h/2.,x[0]+w/2.,x[1]+h/2.,score]).reshape((1,5))
+
+
+class KalmanBoxTracker(object):
+    """
+    This class represents the internal state of individual tracked objects observed as bbox.
+    """
+    count = 0
+    def __init__(self,bbox):
+        """
+        Initialises a tracker using initial bounding box.
+        """
+        #define constant velocity model
+        self.kf = KalmanFilter(dim_x=7, dim_z=4) 
+        self.kf.F = np.array([[1,0,0,0,1,0,0],[0,1,0,0,0,1,0],[0,0,1,0,0,0,1],[0,0,0,1,0,0,0],  [0,0,0,0,1,0,0],[0,0,0,0,0,1,0],[0,0,0,0,0,0,1]])
+        self.kf.H = np.array([[1,0,0,0,0,0,0],[0,1,0,0,0,0,0],[0,0,1,0,0,0,0],[0,0,0,1,0,0,0]])
+
+        self.kf.R[2:,2:] *= 10.
+        self.kf.P[4:,4:] *= 1000. #give high uncertainty to the unobservable initial velocities
+        self.kf.P *= 10.
+        self.kf.Q[-1,-1] *= 0.01
+        self.kf.Q[4:,4:] *= 0.01
+
+        self.kf.x[:4] = convert_bbox_to_z(bbox)
+        self.time_since_update = 0
+        self.id = KalmanBoxTracker.count
+        KalmanBoxTracker.count += 1
+        self.history = []
+        self.hits = 0
+        self.hit_streak = 0
+        self.age = 0
+
+    def update(self,bbox):
+        """
+        Updates the state vector with observed bbox.
+        """
+        self.time_since_update = 0
+        self.history = []
+        self.hits += 1
+        self.hit_streak += 1
+        self.kf.update(convert_bbox_to_z(bbox))
+
+    def predict(self):
+        """
+        Advances the state vector and returns the predicted bounding box estimate.
+        """
+        if((self.kf.x[6]+self.kf.x[2])<=0):
+            self.kf.x[6] *= 0.0
+        self.kf.predict()
+        self.age += 1
+        if(self.time_since_update>0):
+            self.hit_streak = 0
+        self.time_since_update += 1
+        self.history.append(convert_x_to_bbox(self.kf.x))
+        return self.history[-1]
+
+    def get_state(self):
+        """
+        Returns the current bounding box estimate.
+        """
+        return convert_x_to_bbox(self.kf.x)
+
+
+def associate_detections_to_trackers(detections,trackers,iou_threshold = 0.3):
+    """
+    Assigns detections to tracked object (both represented as bounding boxes)
+
+    Returns 3 lists of matches, unmatched_detections and unmatched_trackers
+    """
+    if(len(trackers)==0):
+        return np.empty((0,2),dtype=int), np.arange(len(detections)), np.empty((0,5),dtype=int)
+
+    iou_matrix = iou_batch(detections, trackers)
+
+    if min(iou_matrix.shape) > 0:
+        a = (iou_matrix > iou_threshold).astype(np.int32)
+        if a.sum(1).max() == 1 and a.sum(0).max() == 1:
+            matched_indices = np.stack(np.where(a), axis=1)
+        else:
+            matched_indices = linear_assignment(-iou_matrix)
+    else:
+        matched_indices = np.empty(shape=(0,2))
+
+    unmatched_detections = []
+    for d, _ in enumerate(detections):
+        if(d not in matched_indices[:,0]):
+            unmatched_detections.append(d)
+    unmatched_trackers = []
+    for t, _ in enumerate(trackers):
+        if(t not in matched_indices[:,1]):
+            unmatched_trackers.append(t)
+
+  #filter out matched with low IOU
+    matches = []
+    for m in matched_indices:
+        if(iou_matrix[m[0], m[1]]<iou_threshold):
+            unmatched_detections.append(m[0])
+            unmatched_trackers.append(m[1])
+        else:
+            matches.append(m.reshape(1,2))
+    if(len(matches)==0):
+        matches = np.empty((0,2),dtype=int)
+    else:
+        matches = np.concatenate(matches,axis=0)
+
+    return matches, np.array(unmatched_detections), np.array(unmatched_trackers)
+
+
+class Associate(object):
+    def __init__(self, max_age=1, min_hits=3, iou_threshold=0.3):
+        """
+        Sets key parameters for SORT
+        """
+        self.max_age = max_age
+        self.min_hits = min_hits
+        self.iou_threshold = iou_threshold
+        self.trackers = []
+        self.frame_count = 0
+
+    def update(self, dets=(np.empty((0, 5)), np.empty((0, 5)))):
+        """
+        Params:
+        dets - a numpy array of detections in the format [[x1,y1,x2,y2,score],[x1,y1,x2,y2,score],...]
+        Requires: this method must be called once for each frame even with empty detections (use np.empty((0, 5)) for frames without detections).
+        Returns the a similar array, where the last column is the object ID.
+
+        NOTE: The number of objects returned may differ from the number of detections provided.
+        """
+        self.frame_count += 1
+        # get predicted locations from existing trackers.
+
+        low_dets = dets[0]
+        high_dets = dets[1]
+
+        trks = np.zeros((len(self.trackers), 5))
+        to_del = []
+        ret = []
+        for t, trk in enumerate(trks):
+            pos = self.trackers[t].predict()[0]
+            trk[:] = [pos[0], pos[1], pos[2], pos[3], 0]
+            if np.any(np.isnan(pos)):
+                to_del.append(t)
+        trks = np.ma.compress_rows(np.ma.masked_invalid(trks))
+        for t in reversed(to_del):
+            self.trackers.pop(t)
+        high_matched, unmatched_high_dets, unmatched_trks = associate_detections_to_trackers(high_dets, trks, self.iou_threshold)
+
+        low_matched, unmatched_low_dets, unmatched_trks = associate_detections_to_trackers(low_dets, unmatched_trks, self.iou_threshold)
+
+        # update matched trackers with assigned detections
+        for m in high_matched:
+            self.trackers[m[1]].update(high_dets[m[0], :])
+        for m in low_matched:
+            self.trackers[m[1]].update(low_dets[m[0], :])
+
+        # create and initialise new trackers for unmatched detections
+        for i in unmatched_high_dets:
+            trk = KalmanBoxTracker(high_dets[i,:])
+            self.trackers.append(trk)
+        i = len(self.trackers)
+        for trk in reversed(self.trackers):
+            d = trk.get_state()[0]
+            if (trk.time_since_update < 1) and (trk.hit_streak >= self.min_hits or self.frame_count <= self.min_hits):
+                ret.append(np.concatenate((d,[trk.id+1])).reshape(1,-1)) # +1 as MOT benchmark requires positive
+            i -= 1
+            # remove dead tracklet
+            if(trk.time_since_update > self.max_age):
+                self.trackers.pop(i)
+        if(len(ret)>0):
+            return np.concatenate(ret)
+        return np.empty((0,5))

scripts/track_detection.py

@@ -110,6 +110,9 @@ def track(in_loc_dir, out_loc_dir, metadata_path, seq, config, verbose):
 
     all_preds, real_width, real_height = format_predictions(image_shapes, outputs, width, height, verbose=verbose)
 
+    print(len(all_preds))
+    print(all_preds[0][0])
+
     results = do_tracking(all_preds, image_meter_width, image_meter_height, min_length=config['min_length'], max_age=config['max_age'], iou_thres=config['iou_threshold'], min_hits=config['min_hits'], verbose=verbose)
 
     mot_rows = []