/** * Copyright 2020 (C) Hailo Technologies Ltd. * All rights reserved. * * Hailo Technologies Ltd. ("Hailo") disclaims any warranties, including, but not limited to, * the implied warranties of merchantability and fitness for a particular purpose. * This software is provided on an "AS IS" basis, and Hailo has no obligation to provide maintenance, * support, updates, enhancements, or modifications. * * You may use this software in the development of any project. * You shall not reproduce, modify or distribute this software without prior written permission. **/ /** * @ file switch_hefs_example.cpp * This example demonstrates basic usage of HailoRT streaming api over multiple networks, using vstreams. * It loads a folder of images and tries to detect faces in them. Once it found a face it will switch to a different * model that will do age and gender recognition. **/ #include "c_common.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" #include "hailo/hailort.h" #pragma GCC diagnostic pop #include "yolov5.hpp" #include #include #include #include #include #include #include #define MAX_HEF_PATH_LEN (255) #define MAX_EDGE_LAYERS (15) #define HEF_COUNT (2) #define MAX_FACES (15) #define MAX_BATCH (MAX_FACES) cv::Mat g_frame[1300]; uint32_t i = 0; enum hse_output_index { HSE_OUTPUT_AGE_VECTOR = 0, HSE_OUTPUT_GENDER }; typedef struct write_thread_args_t { hailo_input_vstream input_vstream; uint8_t *src_data[MAX_BATCH]; size_t src_frame_size; uint16_t frames_count; hailo_status status; } write_thread_args_t; typedef struct read_thread_args_t { hailo_output_vstream output_vstream; uint8_t *dst_data[MAX_BATCH]; size_t dst_frame_size; uint16_t frames_count; hailo_status status; } read_thread_args_t; /**********************************************************************************/ /* WRITE_TO_DEVICE function */ /* input: args - */ /* * - the virtual stream to send the data */ /* * - a ptr to the data to write */ /* * - the data length and the */ /* */ /* output: status */ /* * - Success or error code */ /**********************************************************************************/ void* write_to_device(void *args) { hailo_status status = HAILO_UNINITIALIZED; write_thread_args_t *write_args = (write_thread_args_t*)args; // looping over all frames that we sent to the device and calling the write api - hailo_vstream_write_raw_buffer for (uint32_t frame = 0; frame < write_args->frames_count; frame++) { // Write data status = hailo_vstream_write_raw_buffer(write_args->input_vstream, write_args->src_data[frame], write_args->src_frame_size); REQUIRE_SUCCESS(status, l_exit, "Failed writing input frame to device"); } status = HAILO_SUCCESS; l_exit: write_args->status = status; return NULL; } /**********************************************************************************/ /* READ_FROM_DEVICE function */ /* input: args - */ /* * - the virtual stream to receive the data */ /* * - a ptr to the data to read */ /* * - the data length and the */ /* */ /* output: status */ /* * - Success or error code */ /**********************************************************************************/ void* read_from_device(void *args) { hailo_status status = HAILO_UNINITIALIZED; read_thread_args_t *read_args = (read_thread_args_t*)args; for (uint32_t i = 0; i < read_args->frames_count; i++) { // Read data status = hailo_vstream_read_raw_buffer(read_args->output_vstream, read_args->dst_data[i], read_args->dst_frame_size); REQUIRE_SUCCESS(status, l_exit, "Failed reading output frame from device"); // Process data here } status = HAILO_SUCCESS; l_exit: read_args->status = status; return NULL; } /**********************************************************************************/ /* CREATE_INPUT_VSTREAM_THREAD function */ /* input: args - */ /* * - the input virtual stream */ /* * - a ptr to the source data location */ /* * - number of frames */ /* */ /* output: status */ /* * - Success or error code */ /* */ /* the function initializes the values for the write arguments and creates the */ /* input thread which writes(sends the inference data) to the device */ /**********************************************************************************/ hailo_status create_input_vstream_thread(hailo_input_vstream input_vstream, uint8_t **src_data, size_t src_frame_size, uint16_t frames_count, pthread_t *input_thread, write_thread_args_t *write_args) { hailo_status status = HAILO_SUCCESS; // Success oriented int pthread_create_res = 0; for (uint8_t i = 0; i < frames_count; i++) { write_args->src_data[i] = src_data[i]; } write_args->src_frame_size = src_frame_size; write_args->input_vstream = input_vstream; write_args->frames_count = frames_count; write_args->status = HAILO_UNINITIALIZED; // Run write pthread_create_res = pthread_create(input_thread, NULL, write_to_device, write_args); REQUIRE_ACTION(0 == pthread_create_res, status = HAILO_INTERNAL_FAILURE, l_exit, "Failed creating thread"); l_exit: return status; } /**********************************************************************************/ /* CREATE_OUTPUT_VSTREAM_THREAD function */ /* input: args - */ /* * - the output virtual stream */ /* * - a ptr to the dst data location */ /* * - dst frame size */ /* */ /* output: status */ /* * - Success or error code */ /* */ /* the function initializes the values for the read arguments and creates the */ /* output thread which reads(receives the processed data) from the Hailo device */ /**********************************************************************************/ hailo_status create_output_vstream_thread(hailo_output_vstream output_vstream, uint8_t **dst_data, size_t dst_frame_size, uint16_t frames_count, pthread_t *output_thread, read_thread_args_t *read_args) { hailo_status status = HAILO_SUCCESS; // Success oriented int pthread_create_res = 0; for (uint8_t i = 0; i < frames_count; i++) { read_args->dst_data[i] = dst_data[i]; } read_args->dst_frame_size = dst_frame_size; read_args->output_vstream = output_vstream; read_args->frames_count = frames_count; read_args->status = HAILO_UNINITIALIZED; // Run read pthread_create_res = pthread_create(output_thread, NULL, read_from_device, read_args); REQUIRE_ACTION(0 == pthread_create_res, status = HAILO_INTERNAL_FAILURE, l_exit, "Failed creating thread"); l_exit: return status; } /**********************************************************************************/ /* BUILD_STREAMS function */ /* input: args - */ /* * - network group - all hefs defined in a group */ /* * - input vstreams and frame sizes */ /* * - output vstreams and frame sizes */ /* * - destination data */ /* * - frame count */ /* */ /* output: status */ /* * - Success or error code */ /* */ /* the function initializes the values for the read arguments and creates the */ /* output thread which reads(receives the processed data) from the Hailo device */ /**********************************************************************************/ hailo_status build_streams(hailo_configured_network_group network_group, hailo_input_vstream *input_vstreams, size_t *input_frame_sizes, hailo_output_vstream *output_vstreams, size_t *output_frame_sizes, uint8_t *(*dst_data)[MAX_BATCH], size_t *num_output_streams, uint8_t frames_count) { hailo_status status = HAILO_UNINITIALIZED; hailo_input_vstream_params_by_name_t input_vstream_params[MAX_EDGE_LAYERS]; hailo_output_vstream_params_by_name_t output_vstream_params[MAX_EDGE_LAYERS]; size_t input_vstream_size = 1; // Make sure it can hold amount of vstreams for hailo_make_input/output_vstream_params size_t output_vstream_size = MAX_EDGE_LAYERS; // prepare all input vstreams param data in advance status = hailo_make_input_vstream_params(network_group, true, HAILO_FORMAT_TYPE_AUTO, input_vstream_params, &input_vstream_size); REQUIRE_SUCCESS(status, l_exit, "Failed making input virtual stream params"); // prepare all output vstreams param data in advance status = hailo_make_output_vstream_params(network_group, true, HAILO_FORMAT_TYPE_AUTO, output_vstream_params, &output_vstream_size); REQUIRE_SUCCESS(status, l_exit, "Failed making output virtual stream params"); *num_output_streams = output_vstream_size; // create all input vstreams data in advance status = hailo_create_input_vstreams(network_group, input_vstream_params, input_vstream_size, input_vstreams); REQUIRE_SUCCESS(status, l_exit, "Failed creating virtual stream"); // create all output vstreams data in advance status = hailo_create_output_vstreams(network_group, output_vstream_params, output_vstream_size, output_vstreams); REQUIRE_SUCCESS(status, l_release_input_vstream, "Failed creating virtual stream"); for (size_t i = 0; i < input_vstream_size; i++) { status = hailo_get_input_vstream_frame_size(input_vstreams[i], &input_frame_sizes[i]); REQUIRE_SUCCESS(status, l_clear_buffers, "Failed getting input virtual stream frame size"); } for (size_t i = 0; i < output_vstream_size; i++) { status = hailo_get_output_vstream_frame_size(output_vstreams[i], &output_frame_sizes[i]); REQUIRE_SUCCESS(status, l_clear_buffers, "Failed getting input virtual stream frame size"); for (uint8_t j = 0; j < frames_count; j++) { dst_data[i][j] = (uint8_t*)malloc(output_frame_sizes[i]); REQUIRE_ACTION(NULL != dst_data[i], status = HAILO_OUT_OF_HOST_MEMORY, l_clear_buffers, "Out of memory"); } } status = HAILO_SUCCESS; goto l_exit; l_clear_buffers: for (size_t i = 0; i < output_vstream_size; i++) { for (uint8_t j = 0; j < frames_count; j++) { FREE(dst_data[i][j]); } } (void)hailo_release_output_vstreams(output_vstreams, output_vstream_size); l_release_input_vstream: (void)hailo_release_input_vstreams(input_vstreams, input_vstream_size); l_exit: return status; } /**********************************************************************************/ /* RUN_NETWORK function */ /* input: args - */ /* * - network group - specific model to run */ /* * - input vstreams and frame sizes */ /* * - output vstreams and frame sizes */ /* * - destination data */ /* * - frame count */ /* */ /* output: status */ /* * - Success or error code */ /* */ /* the function loads a specific model into the Hailo8 device and runs inference */ /* it uses all the pre-prepared data in order to switch between models very fast */ /**********************************************************************************/ hailo_status run_network(hailo_configured_network_group network_group, hailo_input_vstream input_vstream, uint8_t **input_data, uint16_t frames_count, size_t input_frame_size, hailo_output_vstream *output_vstreams, size_t num_output_vstreams, uint8_t *(*dst_data)[MAX_BATCH], size_t *output_frame_size) { hailo_status status = HAILO_UNINITIALIZED; write_thread_args_t write_args; read_thread_args_t read_args[MAX_EDGE_LAYERS]; hailo_activated_network_group activated_network_group = NULL; pthread_t input_vstream_thread; pthread_t output_vstream_threads[MAX_EDGE_LAYERS]; printf("-I- Running network. Input frame size: %lu\n", input_frame_size); // Activating the specific model we would like to run within the network group status = hailo_activate_network_group(network_group, NULL, &activated_network_group); REQUIRE_SUCCESS(status, l_exit, "Failed activate network group"); // create a thread for the input vstream status = create_input_vstream_thread(input_vstream, input_data, input_frame_size, frames_count, &input_vstream_thread, &write_args); // create threads for the output vstreams for (size_t i = 0; i < num_output_vstreams; i++) { status = create_output_vstream_thread(output_vstreams[i], dst_data[i], output_frame_size[i], frames_count, &output_vstream_threads[i], &read_args[i]); } pthread_join(input_vstream_thread, NULL); for (size_t i = 0; i < num_output_vstreams; i++) { pthread_join(output_vstream_threads[i], NULL); } status = hailo_deactivate_network_group(activated_network_group); REQUIRE_SUCCESS(status, l_exit, "Failed activate network group"); status = HAILO_SUCCESS; l_exit: return status; } /**********************************************************************************/ /* GET_AGE function */ /* input: args - */ /* * - age probability vector */ /* * - vecor length */ /* */ /* output: status */ /* * - estimated age */ /* */ /* the function is a post processing function of the age gender model. */ /* it gets the top two ages and calculate weighted average on their age based on */ /* their probabilities. */ /**********************************************************************************/ uint8_t get_age(uint8_t *age_probability_vector, uint8_t vector_length) { uint8_t max_index = 0; uint8_t second_max_index = 0; uint32_t max_index_probabilty = 0; uint32_t second_max_index_probability = 0; uint32_t sum_of_max_two = 0; for (uint8_t i = 1; i < vector_length; i++) { if (age_probability_vector[max_index] < age_probability_vector[i]) { second_max_index = max_index; max_index = i; } } // keep the #1 probability - multiplied by the age max_index_probabilty = max_index*age_probability_vector[max_index]; // keep the #2 probability - multiplied by the age second_max_index_probability = second_max_index*age_probability_vector[second_max_index]; // keep the sum of #1+#2 probability sum_of_max_two = age_probability_vector[max_index]+age_probability_vector[second_max_index]; // return the average of these two top age weighted by their probabilities return ( (max_index_probabilty + second_max_index_probability) / sum_of_max_two) + 1; } int main(int argc, char **argv) { hailo_status status = HAILO_UNINITIALIZED; hailo_device device = NULL; hailo_hef hef[HEF_COUNT] = {NULL}; hailo_configure_params_t configure_params = {}; hailo_configured_network_group network_groups[HEF_COUNT] = {NULL}; size_t network_groups_size = 1; hailo_input_vstream input_vstreams[HEF_COUNT][MAX_EDGE_LAYERS]; hailo_output_vstream output_vstreams[HEF_COUNT][MAX_EDGE_LAYERS]; size_t input_frame_size[HEF_COUNT][MAX_EDGE_LAYERS]; size_t output_frame_size[HEF_COUNT][MAX_EDGE_LAYERS]; // Initialize 2d array to all NULL uint8_t *dst_data[HEF_COUNT][MAX_EDGE_LAYERS][MAX_BATCH] = {NULL}; size_t num_output_vstreams[HEF_COUNT] = {0}; uint8_t hef_index = 0; cv::Mat input_image; cv::Mat resized_input; cv::Mat input_image_rgb; uint8_t *faces_input[MAX_FACES] = {NULL}; uint8_t faces_count = 0; uint8_t file_counter = 0; uint8_t anchor_index = 0; hailo_vstream_info_t yolo_output_stream_info[ANCHORS_NUM] = {}; hailo_vstream_info_t age_stream_info = {}; float32_t bbox_array[MAX_BOXES][6] = {0.0f}; uint32_t box_index = 0; uint32_t dets_count = 0; uint32_t dets_count_after_nms = 0; cv::Mat cropped_image; cv::Mat resized_image[MAX_FACES]; char cropped_path[] = {'f','i','l','e','_','1','i','m','a','g','e','_','0','.','p','n','g','\0'}; std::vector file_names; char HEF_FILES[HEF_COUNT][MAX_HEF_PATH_LEN] = {"yolov5m_vehicles_bicycles_faces_acc.hef", "hse_age_gender_mobilenet_v2.hef"}; uint8_t HEF_MAX_BATCH_COUNT[HEF_COUNT] = {1, MAX_FACES}; cv::Mat pp_frame(640, 640, CV_8UC3); // define the input folder as images std::string source ="images/"; printf("Usage: %s \n", source.c_str()); cv::glob(source.c_str(), file_names, false); // Create the PCIE device - look for a device connected status = hailo_create_pcie_device(NULL, &device); REQUIRE_SUCCESS(status, l_release_image, "Failed to create pcie_device"); /*******************************/ /* MODEL PREPARATION PART */ /*******************************/ // For each one of the models we would like to run, init the config params and build the streams for (hef_index = 0; hef_index < HEF_COUNT; hef_index++) { status = hailo_create_hef_file(&hef[hef_index], HEF_FILES[hef_index]); REQUIRE_SUCCESS(status, l_release_hef, "Failed creating hef file %s", HEF_FILES[hef_index]); status = hailo_init_configure_params(hef[hef_index], HAILO_STREAM_INTERFACE_PCIE, &configure_params); REQUIRE_SUCCESS(status, l_release_hef, "Failed init configure params"); status = hailo_configure_device(device, hef[hef_index], &configure_params, &network_groups[hef_index], &network_groups_size); REQUIRE_SUCCESS(status, l_release_hef, "Failed configuring devcie"); REQUIRE_ACTION(network_groups_size == 1, status = HAILO_INVALID_ARGUMENT, l_release_hef, "Unexpected network group size"); status = build_streams(network_groups[hef_index], input_vstreams[hef_index], input_frame_size[hef_index], output_vstreams[hef_index], output_frame_size[hef_index], dst_data[hef_index], &num_output_vstreams[hef_index], HEF_MAX_BATCH_COUNT[hef_index]); REQUIRE_SUCCESS(status, l_release_vstreams, "Failed building streams"); } // get the output streams info for all outputs for the detection model for (anchor_index = 0; ANCHORS_NUM > anchor_index; anchor_index++) { status = hailo_get_output_vstream_info(output_vstreams[0][anchor_index], &(yolo_output_stream_info[anchor_index])); REQUIRE_SUCCESS(status, l_release_vstreams, "Failed getting Yolo vstreams info"); } // get the output streams info for the age and gender model status = hailo_get_output_vstream_info(output_vstreams[1][HSE_OUTPUT_AGE_VECTOR], &age_stream_info); REQUIRE_SUCCESS(status, l_release_vstreams, "Failed getting age vstream info"); /*******************************/ /* INFERENCE PART */ /*******************************/ // loop on all files in the file folder for (std::string file : file_names) { file_counter++; printf("\nFILE NUMBER: %d\n\n",file_counter); // read the image from the file input_image = cv::imread(file); cv::resize(input_image, g_frame[i], cv::Size(640, 640), 1); input_image_rgb = g_frame[i]; /******************************/ /* Run YoloV5 detection */ /******************************/ status = run_network(network_groups[0], input_vstreams[0][0], &(input_image_rgb.data), 1, input_frame_size[0][0], output_vstreams[0], num_output_vstreams[0], dst_data[0], output_frame_size[0]); REQUIRE_SUCCESS(status, l_release_vstreams, "Failed to run YoloV5m"); printf("-I- YoloV5 ran successfully.\n"); // Process Yolo raw results into bounding boxes. for (anchor_index = 0; ANCHORS_NUM > anchor_index ; anchor_index++) { extract_boxes(dst_data[0][anchor_index][0], yolo_output_stream_info[anchor_index].quant_info.qp_zp, yolo_output_stream_info[anchor_index].quant_info.qp_scale, g_feature_map_size[anchor_index], g_anchors[anchor_index], CONFIDENCE_THRESHOLD, &box_index, bbox_array); } dets_count = box_index; dets_count_after_nms = dets_count; printf("-I- Detections before NMS: %u.\n", dets_count); for (uint32_t i = 0; i < box_index; ++i) { if (bbox_array[i][4] > CONFIDENCE_THRESHOLD) { for (uint32_t j = i + 1; j < box_index; ++j) { if ((bbox_array[i][5] == bbox_array[j][5]) && (bbox_array[j][4] > CONFIDENCE_THRESHOLD)) { if (iou_calc_c(&bbox_array[i][0], &bbox_array[j][0]) >= IOU_THRESHOLD) { bbox_array[j][4] = 0; dets_count_after_nms -= 1; } } } } } box_index = 0; printf("-I- Detections after NMS: %u.\n", dets_count_after_nms); faces_count = 0; for(uint32_t j = 0; (j < dets_count) && (faces_count < MAX_FACES); j++) { // Validate detection is a face and that the entire box is inside the image borders. if (0 == bbox_array[j][4]) { continue; } printf("Class ID: %f)\n", bbox_array[j][CLASS_ID]); if ((3 == bbox_array[j][CLASS_ID]) && (0 <= bbox_array[j][YMIN]) && (0 <= bbox_array[j][XMIN]) && (IMAGE_SIZE > bbox_array[j][YMAX]) && (IMAGE_SIZE > bbox_array[j][XMAX])) { printf("Face %u at (%f, %f), (%f, %f)\n", faces_count, bbox_array[j][XMIN], bbox_array[j][YMIN], bbox_array[j][XMAX], bbox_array[j][YMAX]); cropped_image = input_image_rgb(cv::Range(bbox_array[j][YMIN], bbox_array[j][YMAX]), cv::Range(bbox_array[j][XMIN], bbox_array[j][XMAX])); cropped_path[5] = '0' + file_counter; cropped_path[12] = '0' + faces_count; cv::imwrite(cropped_path, cropped_image); cv::resize(cropped_image, resized_image[faces_count], cv::Size(224,224), cv::INTER_LINEAR); faces_input[faces_count] = resized_image[faces_count].data; faces_count++; } } /******************************/ /* Run Age/Gender recognition */ /******************************/ // if any faces were found run the age/gender recognition otherwise go to the next file if (0 < faces_count) { status = run_network(network_groups[1], input_vstreams[1][0], faces_input, faces_count, input_frame_size[1][0], output_vstreams[1], num_output_vstreams[1], dst_data[1], output_frame_size[1]); REQUIRE_SUCCESS(status, l_release_vstreams, "Failed to run Age/Gender model"); printf("-I- HSE ran successfully.\n"); // for each face print the gender and age recognition for (uint8_t face_index = 0; face_index < faces_count; face_index++) { printf("Face %u:\n", face_index); if (0.6 <= fix_scale(dst_data[1][HSE_OUTPUT_GENDER][face_index][0], age_stream_info.quant_info.qp_scale, age_stream_info.quant_info.qp_zp)) { printf("\tMale - "); } else { printf("\tFemale - "); } printf("%u\n", get_age(dst_data[1][HSE_OUTPUT_AGE_VECTOR][face_index], output_frame_size[1][HSE_OUTPUT_AGE_VECTOR])); } } } status = HAILO_SUCCESS; goto l_release_vstreams; l_release_vstreams: for (hef_index = 0; hef_index < HEF_COUNT; hef_index++) { (void)hailo_release_output_vstreams(output_vstreams[hef_index], num_output_vstreams[hef_index]); (void)hailo_release_input_vstreams(input_vstreams[hef_index], 1); for (hef_index = 0; hef_index < HEF_COUNT; hef_index++) { // TODO: Add proper release of Mat inputs? for (size_t i = 0; i < num_output_vstreams[hef_index]; i++) { if (NULL != dst_data[hef_index] && NULL != dst_data[hef_index][i]) { for (uint8_t j; j < MAX_BATCH; j++) { FREE(dst_data[hef_index][i][j]); } } } } } l_release_hef: for (hef_index = 0; hef_index < HEF_COUNT; hef_index++) { if (NULL != hef[hef_index]) { (void)hailo_release_hef(hef[hef_index]); } } (void)hailo_release_device(device); l_release_image: input_image.release(); //l_exit: return status; }