diff --git a/cpp_inference/CMakeLists.txt b/cpp_inference/CMakeLists.txt new file mode 100644 index 0000000..dc7a7bd --- /dev/null +++ b/cpp_inference/CMakeLists.txt @@ -0,0 +1,36 @@ +cmake_minimum_required(VERSION 3.1) +project(hailo_demo) + +set(HAILORT_ROOT $ENV{HAILORT_ROOT}) +set(ARCH $ENV{ARCH}) +set(HAILORT_LIB $ENV{HAILORT_ROOT}/lib/${ARCH}/libhailort.so.$ENV{HAILORT_LIB_VER}) +set(HAILORT_INCLUDE_DIR "$ENV{HAILORT_ROOT}/include") +#set(COMPILE_OPTIONS_CPP -Werror -g -O0 -std=c++2a) +#set(COMPILE_OPTIONS_CPP -Wall -Werror -O3 -DNDEBUG -std=c++2a) +set(COMPILE_OPTIONS_CPP -Wall -Werror -pedantic -g -O3) + +message(STATUS "HAILORT_ROOT: $ENV{HAILORT_ROOT}") +message(STATUS "HAILORT_LIB: ${HAILORT_LIB}") +message(STATUS "HAILORT_INCLUDE_DIR: ${HAILORT_INCLUDE_DIR}") + +include_directories(${HAILORT_INCLUDE_DIR} ./) + +find_package(Threads) +find_package( OpenCV REQUIRED ) +message(STATUS "opencv libraries: ${OpenCV_LIBS}") + +message(STATUS "HAILORT_LIB_VER: $ENV{HAILORT_LIB_VER}") +message(STATUS "ARCH: ${ARCH}") + +file(GLOB SOURCES + ./*.hpp + ./*.cpp +) + +add_executable(${PROJECT_NAME} ${SOURCES}) + +target_compile_options(${PROJECT_NAME} PRIVATE ${COMPILE_OPTIONS_CPP}) + +target_link_libraries(${PROJECT_NAME} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${PROJECT_NAME} ${HAILORT_LIB}) +target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS}) diff --git a/cpp_inference/build.sh b/cpp_inference/build.sh new file mode 100644 index 0000000..98d59a4 --- /dev/null +++ b/cpp_inference/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +HAILORT_ROOT=~/HailoRT_v4.4.0/Hailort/Linux/Installer/platform/hailort + + +#export CXX=g++-9 +HAILORT_LIB_VER=4.4.0 HAILORT_ROOT=${HAILORT_ROOT} ARCH=x86_64 cmake -S. -Bbuild +cmake --build build + +if [[ -f "hailort.log" ]]; then + rm hailort.log +fi + +cp *.hef build +cp -r images build \ No newline at end of file diff --git a/cpp_inference/c_common.h b/cpp_inference/c_common.h new file mode 100644 index 0000000..29817b8 --- /dev/null +++ b/cpp_inference/c_common.h @@ -0,0 +1,50 @@ +/** + * Copyright 2020 (C) Hailo Technologies Ltd. + * All rights reserved. + * + * Hailo Technologies Ltd. ("Hailo") disclaims any warranties, including, but not limited to, + * the implied warranties of merchantability and fitness for a particular purpose. + * This software is provided on an "AS IS" basis, and Hailo has no obligation to provide maintenance, + * support, updates, enhancements, or modifications. + * + * You may use this software in the development of any project. + * You shall not reproduce, modify or distribute this software without prior written permission. + **/ +/** + * @ file example_common.h + * Common macros and defines used by Hailort Examples + **/ + +#ifndef _EXAMPLE_COMMON_H_ +#define _EXAMPLE_COMMON_H_ + +#include +#include + + +#define FREE(var) \ + do { \ + if (NULL != (var)) { \ + free(var); \ + var = NULL; \ + } \ + } while(0) + +#define REQUIRE_ACTION(cond, action, label, ...) \ + do { \ + if (!(cond)) { \ + printf(__VA_ARGS__); \ + printf("\n"); \ + action; \ + goto label; \ + } \ + } while(0) + +#define REQUIRE_SUCCESS(status, label, ...) REQUIRE_ACTION((HAILO_SUCCESS == (status)), , label, __VA_ARGS__) + +#define ARRAY_LENGTH(__array) (sizeof((__array)) / sizeof((__array)[0])) + +#define NSEC_IN_SEC (1e+9) + + +#endif /* _EXAMPLE_COMMON_H_ */ diff --git a/cpp_inference/hse_age_gender_mobilenet_v2.hef b/cpp_inference/hse_age_gender_mobilenet_v2.hef new file mode 100644 index 0000000..ae52e4b Binary files /dev/null and b/cpp_inference/hse_age_gender_mobilenet_v2.hef differ diff --git a/cpp_inference/images/faces.jpg b/cpp_inference/images/faces.jpg new file mode 100644 index 0000000..d182750 Binary files /dev/null and b/cpp_inference/images/faces.jpg differ diff --git a/cpp_inference/images/nothing.jpg b/cpp_inference/images/nothing.jpg new file mode 100644 index 0000000..358e834 Binary files /dev/null and b/cpp_inference/images/nothing.jpg differ diff --git a/cpp_inference/images/pexels-jopwell-2422290_640_640.jpg b/cpp_inference/images/pexels-jopwell-2422290_640_640.jpg new file mode 100644 index 0000000..2120458 Binary files /dev/null and b/cpp_inference/images/pexels-jopwell-2422290_640_640.jpg differ diff --git a/cpp_inference/images/pexels-rodnae-productions-7502593_640_640.jpg b/cpp_inference/images/pexels-rodnae-productions-7502593_640_640.jpg new file mode 100644 index 0000000..8c7d231 Binary files /dev/null and b/cpp_inference/images/pexels-rodnae-productions-7502593_640_640.jpg differ diff --git a/cpp_inference/images/pexels-yan-krukov-7691691_640_640.jpg b/cpp_inference/images/pexels-yan-krukov-7691691_640_640.jpg new file mode 100644 index 0000000..cab0717 Binary files /dev/null and b/cpp_inference/images/pexels-yan-krukov-7691691_640_640.jpg differ diff --git a/cpp_inference/images/z_old_couple.jpg b/cpp_inference/images/z_old_couple.jpg new file mode 100644 index 0000000..203eb47 Binary files /dev/null and b/cpp_inference/images/z_old_couple.jpg differ diff --git a/cpp_inference/readme.md b/cpp_inference/readme.md new file mode 100644 index 0000000..7afc6bf --- /dev/null +++ b/cpp_inference/readme.md @@ -0,0 +1,128 @@ + + +# YOLOV5 Age Gender Model Demo + +This example demonstrates basic usage of HailoRT streaming API over multiple networks, using vstreams. +It loads a folder of images and tries to detect faces in them. Once it found a face it will switch to a different model that will do age and gender recognition. + +## Setup on Ubuntu 20.04 + +### OpenCV + +To install OpenCV run: + +```bash +sudo apt install libopencv-dev python3-opencv +``` + +To verify the installation run: + +```bash +python3 -c "import cv2; print(cv2.__version__)" +``` + +### Hailo-8 + +Confirm the Hailo-8 PCIe Module has been detected + +```bash +sudo update-pciids +lspci +``` +
04:00.0 Co-processor: Hailo Technologies Ltd. Hailo-8 AI Processor (rev 01)
+ +### HailoRT + +Install HailoRT available from https://hailo.ai/developer-zone/ and confirm the driver is working. + +Enter virtual environment + +```bash +source hailo_platform_venv/bin/activate +``` + +Check Hailo firmware + +```bash +hailo fw-control identify +``` + +
(hailo) Running command 'fw-control' with 'hailortcli'
+Identifying board
+Control Protocol Version: 2
+Firmware Version: 4.4.0 (release,app)
+Logger Version: 0
+Board Name: Hailo-8
+Device Architecture: HAILO8_B0
+Serial Number: HAILO00000000000
+Part Number: HM218B1C2FA
+Product Name: HAILO-8 AI ACCELERATOR M.2 M KEY MODULE
+
+ +### Building the demo + +Modify the following line in build.sh to fit your HailoRT installation. + +```bash +HAILORT_ROOT=~/HailoRT_v4.4.0/Hailort/Linux/Installer/platform/hailort +``` + +Build the demo + +```bash +./build.sh +``` + +After building the hailo_demo, the script will copy the two HEF files and the images directory into the build folder. Run the demo. + +```bash +cd build +./hailo_demo.sh +``` + +
-I- Running network. Input frame size: 1228800
+-I- YoloV5 ran successfully.
+-I- Detections before NMS: 100.
+-I- Detections after NMS: 9.
+Class ID: 3.000000)
+Face 0 at (68.490112, 61.253448), (180.168640, 208.362183)
+Class ID: 3.000000)
+Face 1 at (268.007507, 64.514343), (375.192413, 202.842468)
+Class ID: 3.000000)
+Face 2 at (449.910217, 62.940426), (556.207397, 204.165405)
+Class ID: 3.000000)
+Face 3 at (75.827576, 257.073730), (180.360580, 398.298706)
+Class ID: 3.000000)
+Face 4 at (258.935303, 256.697327), (369.707764, 397.922302)
+Class ID: 3.000000)
+Face 5 at (456.254761, 257.156647), (567.933289, 408.756989)
+Class ID: 3.000000)
+Face 6 at (74.192513, 450.853729), (180.489777, 593.538330)
+Class ID: 3.000000)
+Face 7 at (256.249298, 451.983093), (366.119324, 594.667725)
+Class ID: 3.000000)
+Face 8 at (455.748230, 451.876953), (561.161499, 596.028809)
+-I- Running network. Input frame size: 150528
+-I- HSE ran successfully.
+Face 0:
+	Male - 26
+Face 1:
+	Female - 23
+Face 2:
+	Female - 23
+Face 3:
+	Male - 27
+Face 4:
+	Female - 29
+Face 5:
+	Male - 29
+Face 6:
+	Female - 23
+Face 7:
+	Female - 29
+Face 8:
+	Female - 27
+
+ diff --git a/cpp_inference/switch_hefs_example.cpp b/cpp_inference/switch_hefs_example.cpp new file mode 100644 index 0000000..e42172f --- /dev/null +++ b/cpp_inference/switch_hefs_example.cpp @@ -0,0 +1,615 @@ +/** + * Copyright 2020 (C) Hailo Technologies Ltd. + * All rights reserved. + * + * Hailo Technologies Ltd. ("Hailo") disclaims any warranties, including, but not limited to, + * the implied warranties of merchantability and fitness for a particular purpose. + * This software is provided on an "AS IS" basis, and Hailo has no obligation to provide maintenance, + * support, updates, enhancements, or modifications. + * + * You may use this software in the development of any project. + * You shall not reproduce, modify or distribute this software without prior written permission. + **/ +/** + * @ file switch_hefs_example.cpp + * This example demonstrates basic usage of HailoRT streaming api over multiple networks, using vstreams. + * It loads a folder of images and tries to detect faces in them. Once it found a face it will switch to a different + * model that will do age and gender recognition. + **/ + +#include "c_common.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#include "hailo/hailort.h" +#pragma GCC diagnostic pop +#include "yolov5.hpp" + +#include +#include +#include +#include +#include +#include +#include + + +#define MAX_HEF_PATH_LEN (255) +#define MAX_EDGE_LAYERS (15) + +#define HEF_COUNT (2) +#define MAX_FACES (15) +#define MAX_BATCH (MAX_FACES) + +cv::Mat g_frame[1300]; +uint32_t i = 0; + +enum hse_output_index +{ + HSE_OUTPUT_AGE_VECTOR = 0, + HSE_OUTPUT_GENDER +}; + +typedef struct write_thread_args_t +{ + hailo_input_vstream input_vstream; + uint8_t *src_data[MAX_BATCH]; + size_t src_frame_size; + uint16_t frames_count; + hailo_status status; +} write_thread_args_t; + +typedef struct read_thread_args_t +{ + hailo_output_vstream output_vstream; + uint8_t *dst_data[MAX_BATCH]; + size_t dst_frame_size; + uint16_t frames_count; + hailo_status status; +} read_thread_args_t; + +/**********************************************************************************/ +/* WRITE_TO_DEVICE function */ +/* input: args - */ +/* * - the virtual stream to send the data */ +/* * - a ptr to the data to write */ +/* * - the data length and the */ +/* */ +/* output: status */ +/* * - Success or error code */ +/**********************************************************************************/ +void* write_to_device(void *args) +{ + hailo_status status = HAILO_UNINITIALIZED; + write_thread_args_t *write_args = (write_thread_args_t*)args; + + // looping over all frames that we sent to the device and calling the write api - hailo_vstream_write_raw_buffer + for (uint32_t frame = 0; frame < write_args->frames_count; frame++) + { + // Write data + status = hailo_vstream_write_raw_buffer(write_args->input_vstream, write_args->src_data[frame], write_args->src_frame_size); + REQUIRE_SUCCESS(status, l_exit, "Failed writing input frame to device"); + } + + status = HAILO_SUCCESS; +l_exit: + write_args->status = status; + return NULL; +} + +/**********************************************************************************/ +/* READ_FROM_DEVICE function */ +/* input: args - */ +/* * - the virtual stream to receive the data */ +/* * - a ptr to the data to read */ +/* * - the data length and the */ +/* */ +/* output: status */ +/* * - Success or error code */ +/**********************************************************************************/ +void* read_from_device(void *args) +{ + hailo_status status = HAILO_UNINITIALIZED; + read_thread_args_t *read_args = (read_thread_args_t*)args; + + for (uint32_t i = 0; i < read_args->frames_count; i++) + { + // Read data + status = hailo_vstream_read_raw_buffer(read_args->output_vstream, read_args->dst_data[i], read_args->dst_frame_size); + REQUIRE_SUCCESS(status, l_exit, "Failed reading output frame from device"); + + // Process data here + } + + status = HAILO_SUCCESS; +l_exit: + read_args->status = status; + return NULL; +} + +/**********************************************************************************/ +/* CREATE_INPUT_VSTREAM_THREAD function */ +/* input: args - */ +/* * - the input virtual stream */ +/* * - a ptr to the source data location */ +/* * - number of frames */ +/* */ +/* output: status */ +/* * - Success or error code */ +/* */ +/* the function initializes the values for the write arguments and creates the */ +/* input thread which writes(sends the inference data) to the device */ +/**********************************************************************************/ +hailo_status create_input_vstream_thread(hailo_input_vstream input_vstream, uint8_t **src_data, size_t src_frame_size, uint16_t frames_count, + pthread_t *input_thread, write_thread_args_t *write_args) +{ + + hailo_status status = HAILO_SUCCESS; // Success oriented + int pthread_create_res = 0; + + for (uint8_t i = 0; i < frames_count; i++) + { + write_args->src_data[i] = src_data[i]; + } + write_args->src_frame_size = src_frame_size; + write_args->input_vstream = input_vstream; + write_args->frames_count = frames_count; + write_args->status = HAILO_UNINITIALIZED; + + // Run write + pthread_create_res = pthread_create(input_thread, NULL, write_to_device, write_args); + REQUIRE_ACTION(0 == pthread_create_res, status = HAILO_INTERNAL_FAILURE, l_exit, "Failed creating thread"); +l_exit: + return status; +} + +/**********************************************************************************/ +/* CREATE_OUTPUT_VSTREAM_THREAD function */ +/* input: args - */ +/* * - the output virtual stream */ +/* * - a ptr to the dst data location */ +/* * - dst frame size */ +/* */ +/* output: status */ +/* * - Success or error code */ +/* */ +/* the function initializes the values for the read arguments and creates the */ +/* output thread which reads(receives the processed data) from the Hailo device */ +/**********************************************************************************/ +hailo_status create_output_vstream_thread(hailo_output_vstream output_vstream, uint8_t **dst_data, size_t dst_frame_size, + uint16_t frames_count, pthread_t *output_thread, read_thread_args_t *read_args) +{ + + hailo_status status = HAILO_SUCCESS; // Success oriented + int pthread_create_res = 0; + + for (uint8_t i = 0; i < frames_count; i++) + { + read_args->dst_data[i] = dst_data[i]; + } + read_args->dst_frame_size = dst_frame_size; + read_args->output_vstream = output_vstream; + read_args->frames_count = frames_count; + read_args->status = HAILO_UNINITIALIZED; + + // Run read + pthread_create_res = pthread_create(output_thread, NULL, read_from_device, read_args); + REQUIRE_ACTION(0 == pthread_create_res, status = HAILO_INTERNAL_FAILURE, l_exit, "Failed creating thread"); +l_exit: + return status; +} + +/**********************************************************************************/ +/* BUILD_STREAMS function */ +/* input: args - */ +/* * - network group - all hefs defined in a group */ +/* * - input vstreams and frame sizes */ +/* * - output vstreams and frame sizes */ +/* * - destination data */ +/* * - frame count */ +/* */ +/* output: status */ +/* * - Success or error code */ +/* */ +/* the function initializes the values for the read arguments and creates the */ +/* output thread which reads(receives the processed data) from the Hailo device */ +/**********************************************************************************/ +hailo_status build_streams(hailo_configured_network_group network_group, + hailo_input_vstream *input_vstreams, size_t *input_frame_sizes, + hailo_output_vstream *output_vstreams, size_t *output_frame_sizes, uint8_t *(*dst_data)[MAX_BATCH], + size_t *num_output_streams, uint8_t frames_count) +{ + hailo_status status = HAILO_UNINITIALIZED; + hailo_input_vstream_params_by_name_t input_vstream_params[MAX_EDGE_LAYERS]; + hailo_output_vstream_params_by_name_t output_vstream_params[MAX_EDGE_LAYERS]; + + size_t input_vstream_size = 1; + // Make sure it can hold amount of vstreams for hailo_make_input/output_vstream_params + size_t output_vstream_size = MAX_EDGE_LAYERS; + + // prepare all input vstreams param data in advance + status = hailo_make_input_vstream_params(network_group, true, HAILO_FORMAT_TYPE_AUTO, + input_vstream_params, &input_vstream_size); + REQUIRE_SUCCESS(status, l_exit, "Failed making input virtual stream params"); + + // prepare all output vstreams param data in advance + status = hailo_make_output_vstream_params(network_group, true, HAILO_FORMAT_TYPE_AUTO, + output_vstream_params, &output_vstream_size); + REQUIRE_SUCCESS(status, l_exit, "Failed making output virtual stream params"); + *num_output_streams = output_vstream_size; + + // create all input vstreams data in advance + status = hailo_create_input_vstreams(network_group, input_vstream_params, input_vstream_size, input_vstreams); + REQUIRE_SUCCESS(status, l_exit, "Failed creating virtual stream"); + + // create all output vstreams data in advance + status = hailo_create_output_vstreams(network_group, output_vstream_params, output_vstream_size, output_vstreams); + REQUIRE_SUCCESS(status, l_release_input_vstream, "Failed creating virtual stream"); + + for (size_t i = 0; i < input_vstream_size; i++) + { + status = hailo_get_input_vstream_frame_size(input_vstreams[i], &input_frame_sizes[i]); + REQUIRE_SUCCESS(status, l_clear_buffers, "Failed getting input virtual stream frame size"); + } + + for (size_t i = 0; i < output_vstream_size; i++) + { + status = hailo_get_output_vstream_frame_size(output_vstreams[i], &output_frame_sizes[i]); + REQUIRE_SUCCESS(status, l_clear_buffers, "Failed getting input virtual stream frame size"); + + for (uint8_t j = 0; j < frames_count; j++) + { + dst_data[i][j] = (uint8_t*)malloc(output_frame_sizes[i]); + REQUIRE_ACTION(NULL != dst_data[i], status = HAILO_OUT_OF_HOST_MEMORY, l_clear_buffers, "Out of memory"); + } + } + + status = HAILO_SUCCESS; + goto l_exit; + +l_clear_buffers: + for (size_t i = 0; i < output_vstream_size; i++) + { + for (uint8_t j = 0; j < frames_count; j++) + { + FREE(dst_data[i][j]); + } + } + (void)hailo_release_output_vstreams(output_vstreams, output_vstream_size); +l_release_input_vstream: + (void)hailo_release_input_vstreams(input_vstreams, input_vstream_size); +l_exit: + return status; +} + +/**********************************************************************************/ +/* RUN_NETWORK function */ +/* input: args - */ +/* * - network group - specific model to run */ +/* * - input vstreams and frame sizes */ +/* * - output vstreams and frame sizes */ +/* * - destination data */ +/* * - frame count */ +/* */ +/* output: status */ +/* * - Success or error code */ +/* */ +/* the function loads a specific model into the Hailo8 device and runs inference */ +/* it uses all the pre-prepared data in order to switch between models very fast */ +/**********************************************************************************/ +hailo_status run_network(hailo_configured_network_group network_group, hailo_input_vstream input_vstream, uint8_t **input_data, uint16_t frames_count, size_t input_frame_size, hailo_output_vstream *output_vstreams, size_t num_output_vstreams, uint8_t *(*dst_data)[MAX_BATCH], size_t *output_frame_size) +{ + hailo_status status = HAILO_UNINITIALIZED; + write_thread_args_t write_args; + read_thread_args_t read_args[MAX_EDGE_LAYERS]; + hailo_activated_network_group activated_network_group = NULL; + pthread_t input_vstream_thread; + pthread_t output_vstream_threads[MAX_EDGE_LAYERS]; + + printf("-I- Running network. Input frame size: %lu\n", input_frame_size); + + // Activating the specific model we would like to run within the network group + status = hailo_activate_network_group(network_group, NULL, &activated_network_group); + REQUIRE_SUCCESS(status, l_exit, "Failed activate network group"); + + // create a thread for the input vstream + status = create_input_vstream_thread(input_vstream, input_data, + input_frame_size, frames_count, &input_vstream_thread, &write_args); + + // create threads for the output vstreams + for (size_t i = 0; i < num_output_vstreams; i++) + { + status = create_output_vstream_thread(output_vstreams[i], dst_data[i], output_frame_size[i], frames_count, &output_vstream_threads[i], &read_args[i]); + } + + pthread_join(input_vstream_thread, NULL); + + for (size_t i = 0; i < num_output_vstreams; i++) + { + pthread_join(output_vstream_threads[i], NULL); + } + + status = hailo_deactivate_network_group(activated_network_group); + REQUIRE_SUCCESS(status, l_exit, "Failed activate network group"); + + status = HAILO_SUCCESS; + +l_exit: + return status; +} + + +/**********************************************************************************/ +/* GET_AGE function */ +/* input: args - */ +/* * - age probability vector */ +/* * - vecor length */ +/* */ +/* output: status */ +/* * - estimated age */ +/* */ +/* the function is a post processing function of the age gender model. */ +/* it gets the top two ages and calculate weighted average on their age based on */ +/* their probabilities. */ +/**********************************************************************************/ +uint8_t get_age(uint8_t *age_probability_vector, uint8_t vector_length) +{ + uint8_t max_index = 0; + uint8_t second_max_index = 0; + uint32_t max_index_probabilty = 0; + uint32_t second_max_index_probability = 0; + uint32_t sum_of_max_two = 0; + + for (uint8_t i = 1; i < vector_length; i++) + { + if (age_probability_vector[max_index] < age_probability_vector[i]) + { + second_max_index = max_index; + max_index = i; + } + } + + // keep the #1 probability - multiplied by the age + max_index_probabilty = max_index*age_probability_vector[max_index]; + + // keep the #2 probability - multiplied by the age + second_max_index_probability = second_max_index*age_probability_vector[second_max_index]; + + // keep the sum of #1+#2 probability + sum_of_max_two = age_probability_vector[max_index]+age_probability_vector[second_max_index]; + + // return the average of these two top age weighted by their probabilities + return ( (max_index_probabilty + second_max_index_probability) / sum_of_max_two) + 1; +} + + +int main(int argc, char **argv) +{ + hailo_status status = HAILO_UNINITIALIZED; + hailo_device device = NULL; + hailo_hef hef[HEF_COUNT] = {NULL}; + hailo_configure_params_t configure_params = {}; + hailo_configured_network_group network_groups[HEF_COUNT] = {NULL}; + size_t network_groups_size = 1; + hailo_input_vstream input_vstreams[HEF_COUNT][MAX_EDGE_LAYERS]; + hailo_output_vstream output_vstreams[HEF_COUNT][MAX_EDGE_LAYERS]; + size_t input_frame_size[HEF_COUNT][MAX_EDGE_LAYERS]; + size_t output_frame_size[HEF_COUNT][MAX_EDGE_LAYERS]; + // Initialize 2d array to all NULL + uint8_t *dst_data[HEF_COUNT][MAX_EDGE_LAYERS][MAX_BATCH] = {NULL}; + size_t num_output_vstreams[HEF_COUNT] = {0}; + uint8_t hef_index = 0; + cv::Mat input_image; + cv::Mat resized_input; + cv::Mat input_image_rgb; + uint8_t *faces_input[MAX_FACES] = {NULL}; + uint8_t faces_count = 0; + uint8_t file_counter = 0; + uint8_t anchor_index = 0; + hailo_vstream_info_t yolo_output_stream_info[ANCHORS_NUM] = {}; + hailo_vstream_info_t age_stream_info = {}; + float32_t bbox_array[MAX_BOXES][6] = {0.0f}; + uint32_t box_index = 0; + uint32_t dets_count = 0; + uint32_t dets_count_after_nms = 0; + cv::Mat cropped_image; + cv::Mat resized_image[MAX_FACES]; + char cropped_path[] = {'f','i','l','e','_','1','i','m','a','g','e','_','0','.','p','n','g','\0'}; + std::vector file_names; + char HEF_FILES[HEF_COUNT][MAX_HEF_PATH_LEN] = {"yolov5m_vehicles_bicycles_faces_acc.hef", "hse_age_gender_mobilenet_v2.hef"}; + + uint8_t HEF_MAX_BATCH_COUNT[HEF_COUNT] = {1, MAX_FACES}; + cv::Mat pp_frame(640, 640, CV_8UC3); + + // define the input folder as images + std::string source ="images/"; + printf("Usage: %s \n", source.c_str()); + cv::glob(source.c_str(), file_names, false); + + // Create the PCIE device - look for a device connected + status = hailo_create_pcie_device(NULL, &device); + REQUIRE_SUCCESS(status, l_release_image, "Failed to create pcie_device"); + + /*******************************/ + /* MODEL PREPARATION PART */ + /*******************************/ + + // For each one of the models we would like to run, init the config params and build the streams + for (hef_index = 0; hef_index < HEF_COUNT; hef_index++) + { + status = hailo_create_hef_file(&hef[hef_index], HEF_FILES[hef_index]); + REQUIRE_SUCCESS(status, l_release_hef, "Failed creating hef file %s", HEF_FILES[hef_index]); + + status = hailo_init_configure_params(hef[hef_index], HAILO_STREAM_INTERFACE_PCIE, &configure_params); + REQUIRE_SUCCESS(status, l_release_hef, "Failed init configure params"); + + status = hailo_configure_device(device, hef[hef_index], &configure_params, &network_groups[hef_index], &network_groups_size); + REQUIRE_SUCCESS(status, l_release_hef, "Failed configuring devcie"); + REQUIRE_ACTION(network_groups_size == 1, status = HAILO_INVALID_ARGUMENT, l_release_hef, + "Unexpected network group size"); + + status = build_streams(network_groups[hef_index], + input_vstreams[hef_index], input_frame_size[hef_index], + output_vstreams[hef_index], output_frame_size[hef_index], + dst_data[hef_index], &num_output_vstreams[hef_index], HEF_MAX_BATCH_COUNT[hef_index]); + REQUIRE_SUCCESS(status, l_release_vstreams, "Failed building streams"); + } + + // get the output streams info for all outputs for the detection model + for (anchor_index = 0; ANCHORS_NUM > anchor_index; anchor_index++) + { + status = hailo_get_output_vstream_info(output_vstreams[0][anchor_index], &(yolo_output_stream_info[anchor_index])); + REQUIRE_SUCCESS(status, l_release_vstreams, "Failed getting Yolo vstreams info"); + } + + // get the output streams info for the age and gender model + status = hailo_get_output_vstream_info(output_vstreams[1][HSE_OUTPUT_AGE_VECTOR], &age_stream_info); + REQUIRE_SUCCESS(status, l_release_vstreams, "Failed getting age vstream info"); + + /*******************************/ + /* INFERENCE PART */ + /*******************************/ + + // loop on all files in the file folder + for (std::string file : file_names) + { + + file_counter++; + printf("\nFILE NUMBER: %d\n\n",file_counter); + + // read the image from the file + input_image = cv::imread(file); + cv::resize(input_image, g_frame[i], cv::Size(640, 640), 1); + input_image_rgb = g_frame[i]; + + /******************************/ + /* Run YoloV5 detection */ + /******************************/ + status = run_network(network_groups[0], input_vstreams[0][0], &(input_image_rgb.data), 1, input_frame_size[0][0], output_vstreams[0], num_output_vstreams[0], dst_data[0], output_frame_size[0]); + REQUIRE_SUCCESS(status, l_release_vstreams, "Failed to run YoloV5m"); + printf("-I- YoloV5 ran successfully.\n"); + + // Process Yolo raw results into bounding boxes. + for (anchor_index = 0; ANCHORS_NUM > anchor_index ; anchor_index++) + { + extract_boxes(dst_data[0][anchor_index][0], yolo_output_stream_info[anchor_index].quant_info.qp_zp, + yolo_output_stream_info[anchor_index].quant_info.qp_scale, g_feature_map_size[anchor_index], + g_anchors[anchor_index], CONFIDENCE_THRESHOLD, &box_index, bbox_array); + } + dets_count = box_index; + dets_count_after_nms = dets_count; + + printf("-I- Detections before NMS: %u.\n", dets_count); + for (uint32_t i = 0; i < box_index; ++i) + { + if (bbox_array[i][4] > CONFIDENCE_THRESHOLD) + { + for (uint32_t j = i + 1; j < box_index; ++j) + { + if ((bbox_array[i][5] == bbox_array[j][5]) && (bbox_array[j][4] > CONFIDENCE_THRESHOLD)) + { + if (iou_calc_c(&bbox_array[i][0], &bbox_array[j][0]) >= IOU_THRESHOLD) + { + bbox_array[j][4] = 0; + dets_count_after_nms -= 1; + } + } + } + } + } + box_index = 0; + printf("-I- Detections after NMS: %u.\n", dets_count_after_nms); + + faces_count = 0; + for(uint32_t j = 0; (j < dets_count) && (faces_count < MAX_FACES); j++) + { + // Validate detection is a face and that the entire box is inside the image borders. + if (0 == bbox_array[j][4]) + { + continue; + } + + printf("Class ID: %f)\n", bbox_array[j][CLASS_ID]); + + if ((3 == bbox_array[j][CLASS_ID]) && (0 <= bbox_array[j][YMIN]) && (0 <= bbox_array[j][XMIN]) && + (IMAGE_SIZE > bbox_array[j][YMAX]) && (IMAGE_SIZE > bbox_array[j][XMAX])) + { + printf("Face %u at (%f, %f), (%f, %f)\n", faces_count, bbox_array[j][XMIN], bbox_array[j][YMIN], bbox_array[j][XMAX], bbox_array[j][YMAX]); + cropped_image = input_image_rgb(cv::Range(bbox_array[j][YMIN], bbox_array[j][YMAX]), cv::Range(bbox_array[j][XMIN], bbox_array[j][XMAX])); + + cropped_path[5] = '0' + file_counter; + cropped_path[12] = '0' + faces_count; + cv::imwrite(cropped_path, cropped_image); + + cv::resize(cropped_image, resized_image[faces_count], cv::Size(224,224), cv::INTER_LINEAR); + faces_input[faces_count] = resized_image[faces_count].data; + faces_count++; + } + } + + /******************************/ + /* Run Age/Gender recognition */ + /******************************/ + + // if any faces were found run the age/gender recognition otherwise go to the next file + if (0 < faces_count) + { + status = run_network(network_groups[1], input_vstreams[1][0], faces_input, faces_count, input_frame_size[1][0], output_vstreams[1], num_output_vstreams[1], dst_data[1], output_frame_size[1]); + REQUIRE_SUCCESS(status, l_release_vstreams, "Failed to run Age/Gender model"); + printf("-I- HSE ran successfully.\n"); + + + // for each face print the gender and age recognition + for (uint8_t face_index = 0; face_index < faces_count; face_index++) + { + printf("Face %u:\n", face_index); + if (0.6 <= fix_scale(dst_data[1][HSE_OUTPUT_GENDER][face_index][0], age_stream_info.quant_info.qp_scale, age_stream_info.quant_info.qp_zp)) + { + printf("\tMale - "); + } + else + { + printf("\tFemale - "); + } + printf("%u\n", get_age(dst_data[1][HSE_OUTPUT_AGE_VECTOR][face_index], output_frame_size[1][HSE_OUTPUT_AGE_VECTOR])); + } + } + } + status = HAILO_SUCCESS; + goto l_release_vstreams; + +l_release_vstreams: + for (hef_index = 0; hef_index < HEF_COUNT; hef_index++) + { + (void)hailo_release_output_vstreams(output_vstreams[hef_index], num_output_vstreams[hef_index]); + (void)hailo_release_input_vstreams(input_vstreams[hef_index], 1); + + for (hef_index = 0; hef_index < HEF_COUNT; hef_index++) + { + // TODO: Add proper release of Mat inputs? + for (size_t i = 0; i < num_output_vstreams[hef_index]; i++) + { + if (NULL != dst_data[hef_index] && NULL != dst_data[hef_index][i]) + { + for (uint8_t j; j < MAX_BATCH; j++) + { + FREE(dst_data[hef_index][i][j]); + } + } + } + } + } + +l_release_hef: + for (hef_index = 0; hef_index < HEF_COUNT; hef_index++) + { + if (NULL != hef[hef_index]) + { + (void)hailo_release_hef(hef[hef_index]); + } + } + (void)hailo_release_device(device); +l_release_image: + input_image.release(); +//l_exit: + return status; +} diff --git a/cpp_inference/yolov5.cpp b/cpp_inference/yolov5.cpp new file mode 100644 index 0000000..1dd2bfa --- /dev/null +++ b/cpp_inference/yolov5.cpp @@ -0,0 +1,201 @@ +#include "yolov5.hpp" +#include +#include +#include + + +float fix_scale(float32_t input, float32_t qp_scale, float32_t qp_zp) +{ + return (input - qp_zp) * qp_scale; +} + +inline float minm(float n1, float n2) +{ + if (n1n2) { + return n1; + } + return n2; +} + +float32_t iou_calc_c(float32_t *box_1, float32_t *box_2) +{ + float width_of_overlap_area = minm(box_1[XMAX], box_2[XMAX]) - maxm(box_1[XMIN], box_2[XMIN]); + float height_of_overlap_area = minm(box_1[YMAX], box_2[YMAX]) - maxm(box_1[YMIN], box_2[YMIN]); + float positive_width_of_overlap_area = maxm(width_of_overlap_area + 1, 0.0f); + float positive_height_of_overlap_area = maxm(height_of_overlap_area + 1, 0.0f); + float area_of_overlap = positive_width_of_overlap_area * positive_height_of_overlap_area; + float box_1_area = (box_1[YMAX] - box_1[YMIN] + 1) * (box_1[XMAX] - box_1[XMIN] + 1); + float box_2_area = (box_2[YMAX] - box_2[YMIN] + 1) * (box_2[XMAX] - box_2[XMIN] + 1); + return area_of_overlap / (box_1_area + box_2_area - area_of_overlap); + +} + +#if 1 +static void get_class(uint8_t *fm, uint32_t row, uint32_t col, uint32_t anchor, uint32_t feature_map_size, +uint32_t *class_id, uint32_t *class_prob) +{ + uint32_t cls_prob, prob_max = 0; + uint32_t selected_class_id = 1; + for (uint32_t cls_id = 1; cls_id <= CLASSES_COUNT; ++cls_id) + { + uint32_t cls_prob_tensor_index = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * col + FEATURE_MAP_CHANNELS * anchor + CLASS_CHANNEL_OFFSET + cls_id - 1; + cls_prob = fm[cls_prob_tensor_index]; + if (cls_prob > prob_max) + { + selected_class_id = cls_id; + prob_max = cls_prob; + } + } + *class_prob = prob_max; + *class_id = selected_class_id; + return; +} + +void extract_boxes(uint8_t *fm, float32_t qp_zp, float32_t qp_scale, uint32_t feature_map_size, + int* anchors, float32_t thr, uint32_t *box_index, float32_t (*box_array)[6]) +{ + float32_t confidence, x, y, h, w, xmin, ymin, xmax, ymax = 0.0f; + uint32_t confidence_tensor_index = 0; + uint32_t class_id = 0; + uint32_t class_prob_int = 0; + float32_t class_prob = 0.0f; + uint32_t x_tensor_index = 0; + for (uint32_t row = 0; row < feature_map_size; ++row) { + for (uint32_t col = 0; col < feature_map_size; ++col) { + for (uint32_t anchor = 0; anchor < ANCHORS_NUM; ++anchor) { + confidence_tensor_index = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * col + FEATURE_MAP_CHANNELS * anchor + CONF_CHANNEL_OFFSET; + if (feature_map_size==20 && confidence_tensor_index >= 9600) { + printf("row: %u col %u anchor %u\n", row, col, anchor); + } + confidence = fix_scale(fm[confidence_tensor_index], qp_scale, qp_zp); + if (confidence < thr) { + continue; + } + get_class(fm, row, col, anchor, feature_map_size, &class_id, &class_prob_int); + class_prob = fix_scale(class_prob_int, qp_scale, qp_zp); + confidence = class_prob * confidence; + if (confidence > thr) + { + //printf("class_prob: %f, confidence: %f\n", class_prob, confidence); + x_tensor_index = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * col + FEATURE_MAP_CHANNELS * anchor; + if (row == 1 && col == 1 && anchor == 1) { + printf("x index: %u\n", x_tensor_index); + } + x = (fix_scale(fm[x_tensor_index], qp_scale, qp_zp) * 2.0f - 0.5f + col) / feature_map_size; + y = (fix_scale(fm[x_tensor_index + 1], qp_scale, qp_zp) * 2.0f - 0.5f + row) / feature_map_size; + w = pow(2.0f * (fix_scale(fm[x_tensor_index + 2], qp_scale, qp_zp)), 2.0f) * anchors[anchor * 2] / IMAGE_SIZE; + h = pow(2.0f * (fix_scale(fm[x_tensor_index + 3], qp_scale, qp_zp)), 2.0f) * anchors[anchor * 2 + 1] / IMAGE_SIZE; + + xmin = (x - (w / 2.0f)) * IMAGE_SIZE; + ymin = (y - (h / 2.0f)) * IMAGE_SIZE; + xmax = (x + (w / 2.0f)) * IMAGE_SIZE; + ymax = (y + (h / 2.0f)) * IMAGE_SIZE; + if (*box_index < MAX_BOXES) + { + //printf("class type %d\n", chosen_cls); + box_array[*box_index][0] = ymin; + box_array[*box_index][1] = xmin; + box_array[*box_index][2] = ymax; + box_array[*box_index][3] = xmax; + box_array[*box_index][4] = confidence; + box_array[*box_index][5] = class_id; + *box_index = *box_index + 1; + } + } + } + } + } +} +#endif + +#if 0 +void extract_boxes(uint8_t *fm, float32_t qp_zp, float32_t qp_scale, int feature_map_size, + int* anchors, float32_t thr, uint32_t *box_index, float32_t (*box_array)[6]) +{ + float32_t confidence, x, y, h, w, xmin, ymin, xmax, ymax, conf_max = 0.0f; + int add = 0, anchor = 0, chosen_row = 0, chosen_col = 0, chosen_cls = -1; + float32_t cls_prob, prob_max = 0.0f; + + // channels 0-3 are box coordinates, channel 4 is the confidence, and channels 5-84 are classes + for (int row = 0; row < feature_map_size; ++row) { + for (int col = 0; col < feature_map_size; ++col) { + prob_max = 0; + for (int a = 0; a < ANCHORS_NUM; ++a) { + add = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * col + FEATURE_MAP_CHANNELS * a + CONF_CHANNEL_OFFSET; + //confidence = fix_scale(fm[add], qp_scale, qp_zp); + confidence = (fm[add])*qp_scale; + if (confidence > thr) + //printf("no way we are here %f\n",qp_scale); + for (int c = CLASS_CHANNEL_OFFSET; c < FEATURE_MAP_CHANNELS; ++c) { + add = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * col + FEATURE_MAP_CHANNELS * a + c; + // final confidence: box confidence * class probability + cls_prob = fm[add]; + if (cls_prob > prob_max) + { + conf_max = fix_scale(cls_prob, qp_scale, qp_zp) * confidence; + chosen_cls = c - CLASS_CHANNEL_OFFSET + 1; + prob_max = cls_prob; + anchor = a; + chosen_row = row; + chosen_col = col; + } + } + } + float basetemp;// = 2.0f * (fix_scale(fm[add + 2], qp_scale, qp_zp)); + float tempresult = 1.0; + //float exptemp = 2.0; + + if (conf_max > thr) { + add = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * chosen_row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * chosen_col + FEATURE_MAP_CHANNELS * anchor; + x = (fix_scale(fm[add], qp_scale, qp_zp) * 2.0f - 0.5f + chosen_col) / feature_map_size; + y = (fix_scale(fm[add + 1], qp_scale, qp_zp) * 2.0f - 0.5f + chosen_row) / feature_map_size; + basetemp = 2.0f * (fix_scale(fm[add + 2], qp_scale, qp_zp)); + //tempresult = 1.0f; + //exptemp = 2.0f; + + //while (exptemp != 0) { + // tempresult *= basetemp; + // --exptemp; + //} + tempresult = basetemp*basetemp; + w = tempresult * anchors[anchor * 2] / IMAGE_SIZE; + + basetemp = 2.0f * (fix_scale(fm[add + 3], qp_scale, qp_zp)); + //tempresult = 1.0f; + //exptemp = 2.0f; + + //while (exptemp != 0) { + // tempresult *= basetemp; + // --exptemp; + //} + tempresult = basetemp*basetemp; + h = tempresult * anchors[anchor * 2 + 1] / IMAGE_SIZE; + + xmin = (x - (w / 2.0f)) * IMAGE_SIZE; + ymin = (y - (h / 2.0f)) * IMAGE_SIZE; + xmax = (x + (w / 2.0f)) * IMAGE_SIZE; + ymax = (y + (h / 2.0f)) * IMAGE_SIZE; + if (*box_index < MAX_BOXES) + { + //printf("class type %d\n", chosen_cls); + box_array[*box_index][0] = ymin; + box_array[*box_index][1] = xmin; + box_array[*box_index][2] = ymax; + box_array[*box_index][3] = xmax; + box_array[*box_index][4] = conf_max; + box_array[*box_index][5] = chosen_cls; + *box_index = *box_index + 1; + } + } + } + } +} +#endif \ No newline at end of file diff --git a/cpp_inference/yolov5.hpp b/cpp_inference/yolov5.hpp new file mode 100644 index 0000000..d51c215 --- /dev/null +++ b/cpp_inference/yolov5.hpp @@ -0,0 +1,57 @@ +#ifndef _YOLOV5M_ +#define _YOLOV5M_ + +#include + +#if 0 +#define FEATURE_MAP_SIZE1 20 +#define FEATURE_MAP_SIZE2 40 +#define FEATURE_MAP_SIZE3 80 +#endif +#if 1 +#define FEATURE_MAP_SIZE1 80 +#define FEATURE_MAP_SIZE2 40 +#define FEATURE_MAP_SIZE3 20 +#endif +//#define FEATURE_MAP_CHANNELS 85 +#define FEATURE_MAP_CHANNELS 8 +#define IMAGE_SIZE 640 +#define ANCHORS_NUM 3 +#define IOU_THRESHOLD 0.45f +#define CONFIDENCE_THRESHOLD 0.7f +#define MAX_BOXES 100 +#define CLASSES_COUNT 3 +//#define CLASSES_COUNT 90 +#define CONF_CHANNEL_OFFSET 4 +#define CLASS_CHANNEL_OFFSET 5 +#define YMIN 0 +#define XMIN 1 +#define YMAX 2 +#define XMAX 3 +#define CONFIDENCE 4 +#define CLASS_ID 5 + +typedef float float32_t; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +static uint8_t g_feature_map_size[ANCHORS_NUM] = {FEATURE_MAP_SIZE1, FEATURE_MAP_SIZE2, FEATURE_MAP_SIZE3}; +#if 1 +static int32_t g_anchors[ANCHORS_NUM][6] = {{10, 13, 16, 30, 33, 23}, + {30, 61, 62, 45, 59, 119}, + {116, 90, 156, 198, 373, 326}}; +#endif +#if 0 +static int32_t g_anchors[ANCHORS_NUM][6] = {{116, 90, 156, 198, 373, 326}, + {30, 61, 62, 45, 59, 119}, + {10, 13, 16, 30, 33, 23}}; +#endif +#pragma GCC diagnostic pop + + +void extract_boxes(uint8_t *fm, float32_t qp_zp, float32_t qp_scale, uint32_t feature_map_size, + int* anchors, float32_t thr, uint32_t *box_index, float32_t (*box_array)[6]); + +float32_t iou_calc_c(float32_t *box_1, float32_t *box_2); +float fix_scale(float32_t input, float32_t qp_scale, float32_t qp_zp); +#endif /* _YOLOV5M_ */ \ No newline at end of file diff --git a/cpp_inference/yolov5m_vehicles_bicycles_faces_acc.hef b/cpp_inference/yolov5m_vehicles_bicycles_faces_acc.hef new file mode 100644 index 0000000..69068f9 Binary files /dev/null and b/cpp_inference/yolov5m_vehicles_bicycles_faces_acc.hef differ diff --git a/inference.py b/python_inference/inference.py similarity index 100% rename from inference.py rename to python_inference/inference.py diff --git a/ros_inference.py b/python_inference/ros_inference.py similarity index 100% rename from ros_inference.py rename to python_inference/ros_inference.py