Unsyncronization on image classification of an image topic

edit

Hello.

I am trying to use Tensorflow lite to implement image classification. To do this I subscribe to the RGB image topic of a Realsense D435. Here I declared my subscription and publications:

def main(self):
    rospy.init_node('tensorflow_ros')
    rospy.loginfo("Iniciado nodo tensorflow_ros")

    print("[INFO] Loading model...")
    self.labels = detector.load_labels("/home/pi/catkin_ws/src/tensorflow_ros/src/coco_labels.txt")
    self.interpreter = Interpreter("/home/pi/catkin_ws/src/tensorflow_ros/src/ssd_mobilenet_v3_small_coco.tflite")
    self.interpreter.allocate_tensors()
    _, input_height, input_width, _ = self.interpreter.get_input_details()[0]['shape']
    self.image_width = input_width
    self.image_height = input_height
    print("[INFO] Model loaded.")
    self.bridge = CvBridge()

    # Publications
    self.classification_pub = rospy.Publisher("image_classified", Image, queue_size=100)
    # Subscriber
    rospy.Subscriber("/camera/color/image_raw", Image, self.callback_image)

    rospy.spin()

Here I process the image and publish the image:

def callback_image(self, msg):

        start = time.time()

        rgb_image = self.bridge.imgmsg_to_cv2(msg, "rgb8")

        rgb_image = cv2.resize(rgb_image, (self.image_width, self.image_height))
        results = detector.detect_objects(self.interpreter, rgb_image, self.threshold)
        detector.annotate_objects(rgb_image, results, self.labels)

        fps = round(1 / (time.time() - start), 2)
        image = cv2.resize(rgb_image, (640, 480))
        cv2.rectangle(rgb_image, (0, 0), (170, 30), (int(0), int(0), int(0)), -1)
        cv2.putText(rgb_image, "FPS:" + str(fps), (5, 25), cv2.FONT_HERSHEY_SIMPLEX, 1, (int(255), int(0), int(0)))

        # Publish topic/home/sara/catkin_ws/src/tensorflow_ros/script
        self.classification_pub.publish(self.bridge.cv2_to_imgmsg(rgb_image, "rgb8"))

The problem is that the classified image have a delay of 4-5 seconds, but it doesnt takes that tame to classify (aproximatelly 100 ms) so I think is a problem of unsyncronization. The framerate is 7-8 fps more or less. What do you think and how can I solve it?

I have tried other way to do this without using the realsense_ros package, by the following way:

def main(self):
    rospy.init_node('tensorflow_ros')
    rospy.loginfo("Iniciado nodo tensorflow_ros")

    # Publications
    self.rgb_pub = rospy.Publisher("/camera/color/image", Image, queue_size=100)
    self.rgb_camerainfo_pub = rospy.Publisher("/camera/color/camera_info", CameraInfo, queue_size=100)
    self.depth_pub = rospy.Publisher("/camera/depth/image", Image, queue_size=100)
    self.depth_camerainfo_pub = rospy.Publisher("/camera/depth/camera_info", CameraInfo, queue_size=100)
    self.pointcloud_pub = rospy.Publisher("pointcloud", PointCloud2, queue_size=100)
    classification_pub = rospy.Publisher("image_classified", Image, queue_size=100)

    print("[INFO] Starting streaming...")

    pipeline = rs.pipeline()
    config = rs.config()
    config.enable_stream(rs.stream.color, self.width, self.height, rs.format.rgb8, 30)
    config.enable_stream(rs.stream.depth, self.width, self.height, rs.format.z16, 30)
    pipeline.start(config)

    print("[INFO] Camera ready.")

    print("[INFO] Loading model...")

    labels = detector.load_labels("/home/pi/catkin_ws/src/tensorflow_ros/src/coco_labels.txt")
    interpreter = Interpreter("/home/pi/catkin_ws/src/tensorflow_ros/src/ssd_mobilenet_v3_small_coco.tflite")
    interpreter.allocate_tensors()
    _, input_height, input_width, _ = interpreter.get_input_details()[0]['shape']
    self.image_width = input_width
    self.image_height = input_height

    print("[INFO] Model loaded.")

    self.bridge = CvBridge()

    try:
        while not rospy.is_shutdown():
            start = time.time()
            self.pc = rs.pointcloud()
            frames = pipeline.wait_for_frames()
            depth_frame = frames.get_depth_frame()
            color_frame = frames.get_color_frame()

            if not depth_frame or not color_frame:
                continue

            rgb_image = detector.get_color_image(color_frame)
            detector.get_depth_image(depth_frame)

            detector.get_pointcloud(depth_frame)

            #Classification
            image = cv2.resize(rgb_image, (self.image_width, self.image_height))
            results = detector.detect_objects(interpreter, image, self.threshold)
            detector ...