OpenCV + MediaPipe实现眼睛虹膜检测

OpenCV学堂

发布于 2021-11-25 15:01:13

2.5K0

发布于 2021-11-25 15:01:13

文章被收录于专栏：贾志刚-OpenCV学堂

虹膜检测的基本原理

MediaPipe的人脸landmark提供了468个点位的人脸点云数据，这些数据的编号图示如下：

根据编号，很容筛选出左眼与右眼所在的区域所有点，然后完成区域的截取，然后调用虹膜检测模型完成检测，MediaPipe SDK的 python版本是不支持虹膜检测，这个比较坑，所有我从github上发现了一个别人训练好的模型，大小只有1MB，地址如下：

https://github.com/ItchyHiker/Iris_Landmarks_PyTorch

它检测得到虹膜的32个点位，图示如下：

导出它的ONNX格式模型，输入与输出图示如下：

RGB顺序，然后减去均值127，除以127。

这个模型实现非常简洁，就是基于残差结构的多路合并，最后通过均值池化链接，最终预测32点位坐标，值在0~1之间。感兴趣的可以自己去看模型源码！

代码演示

首先通过MediaPipe完成人脸的468点位landmark提取，然后分别提取左右眼睛周围点位，根据点位求得外接矩形ROI大小，然后试用ROI左右眼睛图象，通过OpenCV直接预测左右眼睛的虹膜的landmark 32点坐标，绘制即可。

其中人脸landmark检测代码如下：

drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
cap = cv2.VideoCapture("D:/images/video/face_mesh.mp4")
with mp_face_mesh.FaceMesh(
    max_num_faces=4,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5) as face_mesh:
  while cap.isOpened():
    success, frame = cap.read()
    if not success:
      print("Ignoring empty camera frame.")
      # If loading a video, use 'break' instead of 'continue'.
      break

    h, w, c = frame.shape
    # image = cv2.resize(frame, (w //2, h//2))
    # frame = cv2.flip(frame, 1)
    image = np.copy(frame)
    h2, w2, c2 = image.shape
    # To improve performance, optionally mark the image as not writeable to
    # pass by reference.
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    results = face_mesh.process(image)

    # Draw the face mesh annotations on the image.
    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
    left_eyes = []
    right_eyes = []
    if results.multi_face_landmarks:
      for face_landmarks in results.multi_face_landmarks:
        for idx, landmark in enumerate(face_landmarks.landmark):
          if idx == 168 or idx == 197: # middle
            x1 = np.int(landmark.x * w2)
            y1 = np.int(landmark.y * h2)
            left_eyes.append((x1, y1))
            right_eyes.append((x1, y1))
            # cv2.circle(image, (x1, y1), 4, (0, 0, 255), 4, cv2.LINE_AA)
          if idx == 162 or idx == 111:  # left
            x1 = np.int(landmark.x * w2)
            y1 = np.int(landmark.y * h2)
            left_eyes.append((x1, y1))
            # cv2.circle(image, (x1, y1), 4, (255, 0, 255), 4, cv2.LINE_AA)
          if idx == 445 or idx == 448: # right
            x1 = np.int(landmark.x * w2)
            y1 = np.int(landmark.y * h2)
            right_eyes.append((x1, y1))
            # cv2.circle(image, (x1, y1), 4, (0, 255, 255), 4, cv2.LINE_AA)
        right_box = cv2.boundingRect(np.asarray(right_eyes))
        left_box = cv2.boundingRect(np.asarray(left_eyes))
        detect_iris(image, right_box, left_box)
    cv2.imshow('MediaPipe Face Mesh', image)
    if cv2.waitKey(5) & 0xFF == 27:
      cv2.imwrite("D:/iris_detect_result.png", image)
      break
cap.release()
cv2.waitKey(0)
cv2.destroyAllWindows()

虹膜landmark检测代码如下：

net = cv2.dnn.readNetFromONNX("iris_lnet.onnx")

def detect_iris(image, right_box, left_box):
  left_roi = image[left_box[1]:left_box[1] + left_box[3], left_box[0]:left_box[0] + left_box[2]]
  lh, lw, lc = left_roi.shape
  right_roi = image[right_box[1]:right_box[1]+right_box[3],right_box[0]:right_box[0]+right_box[2]]
  rh, rw, rc = right_roi.shape
  left_blob = cv2.dnn.blobFromImage(left_roi, 0.00787, (160, 80), (127.0, 127.0, 127.0), True)
  right_blob = cv2.dnn.blobFromImage(right_roi, 0.00787, (160, 80), (127.0, 127.0, 127.0), True)
  net.setInput(left_blob)
  outs = net.forward()
  pts = np.reshape(outs, (-1, 2))
  for pt in pts:
    x = np.int(pt[0] * lw)
    y = np.int(pt[1] * lh)
    cv2.circle(left_roi, (x, y), 1, (0, 0, 255), 0)
  net.setInput(right_blob)
  outs = net.forward()
  pts = np.reshape(outs, (-1, 2))
  for pt in pts:
    x = np.int(pt[0] * rw)
    y = np.int(pt[1] * rh)
    cv2.circle(right_roi, (x, y), 1, (0, 0, 255), 0)

运行结果如下：