Unsyncronization on image classification of an image topic
Hello.
I am trying to use Tensorflow lite to implement image classification. To do this I subscribe to the RGB image topic of a Realsense D435. Here I declared my subscription and publications:
def main(self):
rospy.init_node('tensorflow_ros')
rospy.loginfo("Iniciado nodo tensorflow_ros")
print("[INFO] Loading model...")
self.labels = detector.load_labels("/home/pi/catkin_ws/src/tensorflow_ros/src/coco_labels.txt")
self.interpreter = Interpreter("/home/pi/catkin_ws/src/tensorflow_ros/src/ssd_mobilenet_v3_small_coco.tflite")
self.interpreter.allocate_tensors()
_, input_height, input_width, _ = self.interpreter.get_input_details()[0]['shape']
self.image_width = input_width
self.image_height = input_height
print("[INFO] Model loaded.")
self.bridge = CvBridge()
# Publications
self.classification_pub = rospy.Publisher("image_classified", Image, queue_size=100)
# Subscriber
rospy.Subscriber("/camera/color/image_raw", Image, self.callback_image)
rospy.spin()
Here I process the image and publish the image:
def callback_image(self, msg):
start = time.time()
rgb_image = self.bridge.imgmsg_to_cv2(msg, "rgb8")
rgb_image = cv2.resize(rgb_image, (self.image_width, self.image_height))
results = detector.detect_objects(self.interpreter, rgb_image, self.threshold)
detector.annotate_objects(rgb_image, results, self.labels)
fps = round(1 / (time.time() - start), 2)
image = cv2.resize(rgb_image, (640, 480))
cv2.rectangle(rgb_image, (0, 0), (170, 30), (int(0), int(0), int(0)), -1)
cv2.putText(rgb_image, "FPS:" + str(fps), (5, 25), cv2.FONT_HERSHEY_SIMPLEX, 1, (int(255), int(0), int(0)))
# Publish topic/home/sara/catkin_ws/src/tensorflow_ros/script
self.classification_pub.publish(self.bridge.cv2_to_imgmsg(rgb_image, "rgb8"))
The problem is that the classified image have a delay of 4-5 seconds, but it doesnt takes that tame to classify (aproximatelly 100 ms) so I think is a problem of unsyncronization. The framerate is 7-8 fps more or less. What do you think and how can I solve it?
I have tried other way to do this without using the realsense_ros package, by the following way:
def main(self):
rospy.init_node('tensorflow_ros')
rospy.loginfo("Iniciado nodo tensorflow_ros")
# Publications
self.rgb_pub = rospy.Publisher("/camera/color/image", Image, queue_size=100)
self.rgb_camerainfo_pub = rospy.Publisher("/camera/color/camera_info", CameraInfo, queue_size=100)
self.depth_pub = rospy.Publisher("/camera/depth/image", Image, queue_size=100)
self.depth_camerainfo_pub = rospy.Publisher("/camera/depth/camera_info", CameraInfo, queue_size=100)
self.pointcloud_pub = rospy.Publisher("pointcloud", PointCloud2, queue_size=100)
classification_pub = rospy.Publisher("image_classified", Image, queue_size=100)
print("[INFO] Starting streaming...")
pipeline = rs.pipeline()
config = rs.config()
config.enable_stream(rs.stream.color, self.width, self.height, rs.format.rgb8, 30)
config.enable_stream(rs.stream.depth, self.width, self.height, rs.format.z16, 30)
pipeline.start(config)
print("[INFO] Camera ready.")
print("[INFO] Loading model...")
labels = detector.load_labels("/home/pi/catkin_ws/src/tensorflow_ros/src/coco_labels.txt")
interpreter = Interpreter("/home/pi/catkin_ws/src/tensorflow_ros/src/ssd_mobilenet_v3_small_coco.tflite")
interpreter.allocate_tensors()
_, input_height, input_width, _ = interpreter.get_input_details()[0]['shape']
self.image_width = input_width
self.image_height = input_height
print("[INFO] Model loaded.")
self.bridge = CvBridge()
try:
while not rospy.is_shutdown():
start = time.time()
self.pc = rs.pointcloud()
frames = pipeline.wait_for_frames()
depth_frame = frames.get_depth_frame()
color_frame = frames.get_color_frame()
if not depth_frame or not color_frame:
continue
rgb_image = detector.get_color_image(color_frame)
detector.get_depth_image(depth_frame)
detector.get_pointcloud(depth_frame)
#Classification
image = cv2.resize(rgb_image, (self.image_width, self.image_height))
results = detector.detect_objects(interpreter, image, self.threshold)
detector ...