raphael/hailo-inference
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

adds cpp inference example

Browse Source
This commit is contained in:
raphael 2022-03-31 11:10:10 +02:00
parent dd99e3570f
commit 34242b5dbd
17 changed files with 1102 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
cpp_inference/CMakeLists.txt Normal file
View File

@ -0,0 +1,36 @@
cmake_minimum_required(VERSION 3.1)
project(hailo_demo)
set(HAILORT_ROOT $ENV{HAILORT_ROOT})
set(ARCH $ENV{ARCH})
set(HAILORT_LIB $ENV{HAILORT_ROOT}/lib/${ARCH}/libhailort.so.$ENV{HAILORT_LIB_VER})
set(HAILORT_INCLUDE_DIR "$ENV{HAILORT_ROOT}/include")
#set(COMPILE_OPTIONS_CPP -Werror -g -O0 -std=c++2a)
#set(COMPILE_OPTIONS_CPP -Wall -Werror -O3 -DNDEBUG -std=c++2a)
set(COMPILE_OPTIONS_CPP -Wall -Werror -pedantic -g -O3)
message(STATUS "HAILORT_ROOT: $ENV{HAILORT_ROOT}")
message(STATUS "HAILORT_LIB: ${HAILORT_LIB}")
message(STATUS "HAILORT_INCLUDE_DIR: ${HAILORT_INCLUDE_DIR}")
include_directories(${HAILORT_INCLUDE_DIR} ./)
find_package(Threads)
find_package( OpenCV REQUIRED )
message(STATUS "opencv libraries: ${OpenCV_LIBS}")
message(STATUS "HAILORT_LIB_VER: $ENV{HAILORT_LIB_VER}")
message(STATUS "ARCH: ${ARCH}")
file(GLOB SOURCES
./*.hpp
./*.cpp
)
add_executable(${PROJECT_NAME} ${SOURCES})
target_compile_options(${PROJECT_NAME} PRIVATE ${COMPILE_OPTIONS_CPP})
target_link_libraries(${PROJECT_NAME} ${CMAKE_THREAD_LIBS_INIT})
target_link_libraries(${PROJECT_NAME} ${HAILORT_LIB})
target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS})

15
cpp_inference/build.sh Normal file
View File

@ -0,0 +1,15 @@
#!/bin/bash
HAILORT_ROOT=~/HailoRT_v4.4.0/Hailort/Linux/Installer/platform/hailort
#export CXX=g++-9
HAILORT_LIB_VER=4.4.0 HAILORT_ROOT=${HAILORT_ROOT} ARCH=x86_64 cmake -S. -Bbuild
cmake --build build
if [[ -f "hailort.log" ]]; then
rm hailort.log
fi
cp *.hef build
cp -r images build

50
cpp_inference/c_common.h Normal file
View File

@ -0,0 +1,50 @@
/**
* Copyright 2020 (C) Hailo Technologies Ltd.
* All rights reserved.
*
* Hailo Technologies Ltd. ("Hailo") disclaims any warranties, including, but not limited to,
* the implied warranties of merchantability and fitness for a particular purpose.
* This software is provided on an "AS IS" basis, and Hailo has no obligation to provide maintenance,
* support, updates, enhancements, or modifications.
*
* You may use this software in the development of any project.
* You shall not reproduce, modify or distribute this software without prior written permission.
**/
/**
* @ file example_common.h
* Common macros and defines used by Hailort Examples
**/
#ifndef _EXAMPLE_COMMON_H_
#define _EXAMPLE_COMMON_H_
#include <stdio.h>
#include <stdlib.h>
#define FREE(var) \
do { \
if (NULL != (var)) { \
free(var); \
var = NULL; \
} \
} while(0)
#define REQUIRE_ACTION(cond, action, label, ...) \
do { \
if (!(cond)) { \
printf(__VA_ARGS__); \
printf("\n"); \
action; \
goto label; \
} \
} while(0)
#define REQUIRE_SUCCESS(status, label, ...) REQUIRE_ACTION((HAILO_SUCCESS == (status)), , label, __VA_ARGS__)
#define ARRAY_LENGTH(__array) (sizeof((__array)) / sizeof((__array)[0]))
#define NSEC_IN_SEC (1e+9)
#endif /* _EXAMPLE_COMMON_H_ */

BIN
cpp_inference/hse_age_gender_mobilenet_v2.hef Normal file
View File

Binary file not shown.

BIN
cpp_inference/images/faces.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 270 KiB

BIN
cpp_inference/images/nothing.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.9 KiB

BIN
cpp_inference/images/pexels-jopwell-2422290_640_640.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 354 KiB

BIN
cpp_inference/images/pexels-rodnae-productions-7502593_640_640.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 370 KiB

BIN
cpp_inference/images/pexels-yan-krukov-7691691_640_640.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 292 KiB

BIN
cpp_inference/images/z_old_couple.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 323 KiB

128
cpp_inference/readme.md Normal file
View File

@ -0,0 +1,128 @@
<style>
#term {color: white; background: black}
</style>
# YOLOV5 Age Gender Model Demo
This example demonstrates basic usage of HailoRT streaming API over multiple networks, using vstreams.
It loads a folder of images and tries to detect faces in them. Once it found a face it will switch to a different model that will do age and gender recognition.
## Setup on Ubuntu 20.04
### OpenCV
To install OpenCV run:
```bash
sudo apt install libopencv-dev python3-opencv
```
To verify the installation run:
```bash
python3 -c "import cv2; print(cv2.__version__)"
```
### Hailo-8
Confirm the Hailo-8 PCIe Module has been detected
```bash
sudo update-pciids
lspci
```
<pre id=term>04:00.0 Co-processor: Hailo Technologies Ltd. Hailo-8 AI Processor (rev 01)</pre>
### HailoRT
Install HailoRT available from https://hailo.ai/developer-zone/ and confirm the driver is working.
Enter virtual environment
```bash
source hailo_platform_venv/bin/activate
```
Check Hailo firmware
```bash
hailo fw-control identify
```
<pre id=term>(hailo) Running command &apos;fw-control&apos; with &apos;hailortcli&apos;
Identifying board
Control Protocol Version: 2
Firmware Version: 4.4.0 (release,app)
Logger Version: 0
Board Name: Hailo-8
Device Architecture: HAILO8_B0
Serial Number: HAILO00000000000
Part Number: HM218B1C2FA
Product Name: HAILO-8 AI ACCELERATOR M.2 M KEY MODULE
</pre>
### Building the demo
Modify the following line in build.sh to fit your HailoRT installation.
```bash
HAILORT_ROOT=~/HailoRT_v4.4.0/Hailort/Linux/Installer/platform/hailort
```
Build the demo
```bash
./build.sh
```
After building the hailo_demo, the script will copy the two HEF files and the images directory into the build folder. Run the demo.
```bash
cd build
./hailo_demo.sh
```
<pre id=term>-I- Running network. Input frame size: 1228800
-I- YoloV5 ran successfully.
-I- Detections before NMS: 100.
-I- Detections after NMS: 9.
Class ID: 3.000000)
Face 0 at (68.490112, 61.253448), (180.168640, 208.362183)
Class ID: 3.000000)
Face 1 at (268.007507, 64.514343), (375.192413, 202.842468)
Class ID: 3.000000)
Face 2 at (449.910217, 62.940426), (556.207397, 204.165405)
Class ID: 3.000000)
Face 3 at (75.827576, 257.073730), (180.360580, 398.298706)
Class ID: 3.000000)
Face 4 at (258.935303, 256.697327), (369.707764, 397.922302)
Class ID: 3.000000)
Face 5 at (456.254761, 257.156647), (567.933289, 408.756989)
Class ID: 3.000000)
Face 6 at (74.192513, 450.853729), (180.489777, 593.538330)
Class ID: 3.000000)
Face 7 at (256.249298, 451.983093), (366.119324, 594.667725)
Class ID: 3.000000)
Face 8 at (455.748230, 451.876953), (561.161499, 596.028809)
-I- Running network. Input frame size: 150528
-I- HSE ran successfully.
Face 0:
Male - 26
Face 1:
Female - 23
Face 2:
Female - 23
Face 3:
Male - 27
Face 4:
Female - 29
Face 5:
Male - 29
Face 6:
Female - 23
Face 7:
Female - 29
Face 8:
Female - 27
</pre>

615
cpp_inference/switch_hefs_example.cpp Normal file
View File

@ -0,0 +1,615 @@
/**
* Copyright 2020 (C) Hailo Technologies Ltd.
* All rights reserved.
*
* Hailo Technologies Ltd. ("Hailo") disclaims any warranties, including, but not limited to,
* the implied warranties of merchantability and fitness for a particular purpose.
* This software is provided on an "AS IS" basis, and Hailo has no obligation to provide maintenance,
* support, updates, enhancements, or modifications.
*
* You may use this software in the development of any project.
* You shall not reproduce, modify or distribute this software without prior written permission.
**/
/**
* @ file switch_hefs_example.cpp
* This example demonstrates basic usage of HailoRT streaming api over multiple networks, using vstreams.
* It loads a folder of images and tries to detect faces in them. Once it found a face it will switch to a different
* model that will do age and gender recognition.
**/
#include "c_common.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
#include "hailo/hailort.h"
#pragma GCC diagnostic pop
#include "yolov5.hpp"
#include <pthread.h>
#include <time.h>
#include <vector>
#include <opencv2/opencv.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
#include <iostream>
#define MAX_HEF_PATH_LEN (255)
#define MAX_EDGE_LAYERS (15)
#define HEF_COUNT (2)
#define MAX_FACES (15)
#define MAX_BATCH (MAX_FACES)
cv::Mat g_frame[1300];
uint32_t i = 0;
enum hse_output_index
{
HSE_OUTPUT_AGE_VECTOR = 0,
HSE_OUTPUT_GENDER
};
typedef struct write_thread_args_t
{
hailo_input_vstream input_vstream;
uint8_t *src_data[MAX_BATCH];
size_t src_frame_size;
uint16_t frames_count;
hailo_status status;
} write_thread_args_t;
typedef struct read_thread_args_t
{
hailo_output_vstream output_vstream;
uint8_t *dst_data[MAX_BATCH];
size_t dst_frame_size;
uint16_t frames_count;
hailo_status status;
} read_thread_args_t;
/**********************************************************************************/
/* WRITE_TO_DEVICE function */
/* input: args - */
/* * - the virtual stream to send the data */
/* * - a ptr to the data to write */
/* * - the data length and the */
/* */
/* output: status */
/* * - Success or error code */
/**********************************************************************************/
void* write_to_device(void *args)
{
hailo_status status = HAILO_UNINITIALIZED;
write_thread_args_t *write_args = (write_thread_args_t*)args;
// looping over all frames that we sent to the device and calling the write api - hailo_vstream_write_raw_buffer
for (uint32_t frame = 0; frame < write_args->frames_count; frame++)
{
// Write data
status = hailo_vstream_write_raw_buffer(write_args->input_vstream, write_args->src_data[frame], write_args->src_frame_size);
REQUIRE_SUCCESS(status, l_exit, "Failed writing input frame to device");
}
status = HAILO_SUCCESS;
l_exit:
write_args->status = status;
return NULL;
}
/**********************************************************************************/
/* READ_FROM_DEVICE function */
/* input: args - */
/* * - the virtual stream to receive the data */
/* * - a ptr to the data to read */
/* * - the data length and the */
/* */
/* output: status */
/* * - Success or error code */
/**********************************************************************************/
void* read_from_device(void *args)
{
hailo_status status = HAILO_UNINITIALIZED;
read_thread_args_t *read_args = (read_thread_args_t*)args;
for (uint32_t i = 0; i < read_args->frames_count; i++)
{
// Read data
status = hailo_vstream_read_raw_buffer(read_args->output_vstream, read_args->dst_data[i], read_args->dst_frame_size);
REQUIRE_SUCCESS(status, l_exit, "Failed reading output frame from device");
// Process data here
}
status = HAILO_SUCCESS;
l_exit:
read_args->status = status;
return NULL;
}
/**********************************************************************************/
/* CREATE_INPUT_VSTREAM_THREAD function */
/* input: args - */
/* * - the input virtual stream */
/* * - a ptr to the source data location */
/* * - number of frames */
/* */
/* output: status */
/* * - Success or error code */
/* */
/* the function initializes the values for the write arguments and creates the */
/* input thread which writes(sends the inference data) to the device */
/**********************************************************************************/
hailo_status create_input_vstream_thread(hailo_input_vstream input_vstream, uint8_t **src_data, size_t src_frame_size, uint16_t frames_count,
pthread_t *input_thread, write_thread_args_t *write_args)
{
hailo_status status = HAILO_SUCCESS; // Success oriented
int pthread_create_res = 0;
for (uint8_t i = 0; i < frames_count; i++)
{
write_args->src_data[i] = src_data[i];
}
write_args->src_frame_size = src_frame_size;
write_args->input_vstream = input_vstream;
write_args->frames_count = frames_count;
write_args->status = HAILO_UNINITIALIZED;
// Run write
pthread_create_res = pthread_create(input_thread, NULL, write_to_device, write_args);
REQUIRE_ACTION(0 == pthread_create_res, status = HAILO_INTERNAL_FAILURE, l_exit, "Failed creating thread");
l_exit:
return status;
}
/**********************************************************************************/
/* CREATE_OUTPUT_VSTREAM_THREAD function */
/* input: args - */
/* * - the output virtual stream */
/* * - a ptr to the dst data location */
/* * - dst frame size */
/* */
/* output: status */
/* * - Success or error code */
/* */
/* the function initializes the values for the read arguments and creates the */
/* output thread which reads(receives the processed data) from the Hailo device */
/**********************************************************************************/
hailo_status create_output_vstream_thread(hailo_output_vstream output_vstream, uint8_t **dst_data, size_t dst_frame_size,
uint16_t frames_count, pthread_t *output_thread, read_thread_args_t *read_args)
{
hailo_status status = HAILO_SUCCESS; // Success oriented
int pthread_create_res = 0;
for (uint8_t i = 0; i < frames_count; i++)
{
read_args->dst_data[i] = dst_data[i];
}
read_args->dst_frame_size = dst_frame_size;
read_args->output_vstream = output_vstream;
read_args->frames_count = frames_count;
read_args->status = HAILO_UNINITIALIZED;
// Run read
pthread_create_res = pthread_create(output_thread, NULL, read_from_device, read_args);
REQUIRE_ACTION(0 == pthread_create_res, status = HAILO_INTERNAL_FAILURE, l_exit, "Failed creating thread");
l_exit:
return status;
}
/**********************************************************************************/
/* BUILD_STREAMS function */
/* input: args - */
/* * - network group - all hefs defined in a group */
/* * - input vstreams and frame sizes */
/* * - output vstreams and frame sizes */
/* * - destination data */
/* * - frame count */
/* */
/* output: status */
/* * - Success or error code */
/* */
/* the function initializes the values for the read arguments and creates the */
/* output thread which reads(receives the processed data) from the Hailo device */
/**********************************************************************************/
hailo_status build_streams(hailo_configured_network_group network_group,
hailo_input_vstream *input_vstreams, size_t *input_frame_sizes,
hailo_output_vstream *output_vstreams, size_t *output_frame_sizes, uint8_t *(*dst_data)[MAX_BATCH],
size_t *num_output_streams, uint8_t frames_count)
{
hailo_status status = HAILO_UNINITIALIZED;
hailo_input_vstream_params_by_name_t input_vstream_params[MAX_EDGE_LAYERS];
hailo_output_vstream_params_by_name_t output_vstream_params[MAX_EDGE_LAYERS];
size_t input_vstream_size = 1;
// Make sure it can hold amount of vstreams for hailo_make_input/output_vstream_params
size_t output_vstream_size = MAX_EDGE_LAYERS;
// prepare all input vstreams param data in advance
status = hailo_make_input_vstream_params(network_group, true, HAILO_FORMAT_TYPE_AUTO,
input_vstream_params, &input_vstream_size);
REQUIRE_SUCCESS(status, l_exit, "Failed making input virtual stream params");
// prepare all output vstreams param data in advance
status = hailo_make_output_vstream_params(network_group, true, HAILO_FORMAT_TYPE_AUTO,
output_vstream_params, &output_vstream_size);
REQUIRE_SUCCESS(status, l_exit, "Failed making output virtual stream params");
*num_output_streams = output_vstream_size;
// create all input vstreams data in advance
status = hailo_create_input_vstreams(network_group, input_vstream_params, input_vstream_size, input_vstreams);
REQUIRE_SUCCESS(status, l_exit, "Failed creating virtual stream");
// create all output vstreams data in advance
status = hailo_create_output_vstreams(network_group, output_vstream_params, output_vstream_size, output_vstreams);
REQUIRE_SUCCESS(status, l_release_input_vstream, "Failed creating virtual stream");
for (size_t i = 0; i < input_vstream_size; i++)
{
status = hailo_get_input_vstream_frame_size(input_vstreams[i], &input_frame_sizes[i]);
REQUIRE_SUCCESS(status, l_clear_buffers, "Failed getting input virtual stream frame size");
}
for (size_t i = 0; i < output_vstream_size; i++)
{
status = hailo_get_output_vstream_frame_size(output_vstreams[i], &output_frame_sizes[i]);
REQUIRE_SUCCESS(status, l_clear_buffers, "Failed getting input virtual stream frame size");
for (uint8_t j = 0; j < frames_count; j++)
{
dst_data[i][j] = (uint8_t*)malloc(output_frame_sizes[i]);
REQUIRE_ACTION(NULL != dst_data[i], status = HAILO_OUT_OF_HOST_MEMORY, l_clear_buffers, "Out of memory");
}
}
status = HAILO_SUCCESS;
goto l_exit;
l_clear_buffers:
for (size_t i = 0; i < output_vstream_size; i++)
{
for (uint8_t j = 0; j < frames_count; j++)
{
FREE(dst_data[i][j]);
}
}
(void)hailo_release_output_vstreams(output_vstreams, output_vstream_size);
l_release_input_vstream:
(void)hailo_release_input_vstreams(input_vstreams, input_vstream_size);
l_exit:
return status;
}
/**********************************************************************************/
/* RUN_NETWORK function */
/* input: args - */
/* * - network group - specific model to run */
/* * - input vstreams and frame sizes */
/* * - output vstreams and frame sizes */
/* * - destination data */
/* * - frame count */
/* */
/* output: status */
/* * - Success or error code */
/* */
/* the function loads a specific model into the Hailo8 device and runs inference */
/* it uses all the pre-prepared data in order to switch between models very fast */
/**********************************************************************************/
hailo_status run_network(hailo_configured_network_group network_group, hailo_input_vstream input_vstream, uint8_t **input_data, uint16_t frames_count, size_t input_frame_size, hailo_output_vstream *output_vstreams, size_t num_output_vstreams, uint8_t *(*dst_data)[MAX_BATCH], size_t *output_frame_size)
{
hailo_status status = HAILO_UNINITIALIZED;
write_thread_args_t write_args;
read_thread_args_t read_args[MAX_EDGE_LAYERS];
hailo_activated_network_group activated_network_group = NULL;
pthread_t input_vstream_thread;
pthread_t output_vstream_threads[MAX_EDGE_LAYERS];
printf("-I- Running network. Input frame size: %lu\n", input_frame_size);
// Activating the specific model we would like to run within the network group
status = hailo_activate_network_group(network_group, NULL, &activated_network_group);
REQUIRE_SUCCESS(status, l_exit, "Failed activate network group");
// create a thread for the input vstream
status = create_input_vstream_thread(input_vstream, input_data,
input_frame_size, frames_count, &input_vstream_thread, &write_args);
// create threads for the output vstreams
for (size_t i = 0; i < num_output_vstreams; i++)
{
status = create_output_vstream_thread(output_vstreams[i], dst_data[i], output_frame_size[i], frames_count, &output_vstream_threads[i], &read_args[i]);
}
pthread_join(input_vstream_thread, NULL);
for (size_t i = 0; i < num_output_vstreams; i++)
{
pthread_join(output_vstream_threads[i], NULL);
}
status = hailo_deactivate_network_group(activated_network_group);
REQUIRE_SUCCESS(status, l_exit, "Failed activate network group");
status = HAILO_SUCCESS;
l_exit:
return status;
}
/**********************************************************************************/
/* GET_AGE function */
/* input: args - */
/* * - age probability vector */
/* * - vecor length */
/* */
/* output: status */
/* * - estimated age */
/* */
/* the function is a post processing function of the age gender model. */
/* it gets the top two ages and calculate weighted average on their age based on */
/* their probabilities. */
/**********************************************************************************/
uint8_t get_age(uint8_t *age_probability_vector, uint8_t vector_length)
{
uint8_t max_index = 0;
uint8_t second_max_index = 0;
uint32_t max_index_probabilty = 0;
uint32_t second_max_index_probability = 0;
uint32_t sum_of_max_two = 0;
for (uint8_t i = 1; i < vector_length; i++)
{
if (age_probability_vector[max_index] < age_probability_vector[i])
{
second_max_index = max_index;
max_index = i;
}
}
// keep the #1 probability - multiplied by the age
max_index_probabilty = max_index*age_probability_vector[max_index];
// keep the #2 probability - multiplied by the age
second_max_index_probability = second_max_index*age_probability_vector[second_max_index];
// keep the sum of #1+#2 probability
sum_of_max_two = age_probability_vector[max_index]+age_probability_vector[second_max_index];
// return the average of these two top age weighted by their probabilities
return ( (max_index_probabilty + second_max_index_probability) / sum_of_max_two) + 1;
}
int main(int argc, char **argv)
{
hailo_status status = HAILO_UNINITIALIZED;
hailo_device device = NULL;
hailo_hef hef[HEF_COUNT] = {NULL};
hailo_configure_params_t configure_params = {};
hailo_configured_network_group network_groups[HEF_COUNT] = {NULL};
size_t network_groups_size = 1;
hailo_input_vstream input_vstreams[HEF_COUNT][MAX_EDGE_LAYERS];
hailo_output_vstream output_vstreams[HEF_COUNT][MAX_EDGE_LAYERS];
size_t input_frame_size[HEF_COUNT][MAX_EDGE_LAYERS];
size_t output_frame_size[HEF_COUNT][MAX_EDGE_LAYERS];
// Initialize 2d array to all NULL
uint8_t *dst_data[HEF_COUNT][MAX_EDGE_LAYERS][MAX_BATCH] = {NULL};
size_t num_output_vstreams[HEF_COUNT] = {0};
uint8_t hef_index = 0;
cv::Mat input_image;
cv::Mat resized_input;
cv::Mat input_image_rgb;
uint8_t *faces_input[MAX_FACES] = {NULL};
uint8_t faces_count = 0;
uint8_t file_counter = 0;
uint8_t anchor_index = 0;
hailo_vstream_info_t yolo_output_stream_info[ANCHORS_NUM] = {};
hailo_vstream_info_t age_stream_info = {};
float32_t bbox_array[MAX_BOXES][6] = {0.0f};
uint32_t box_index = 0;
uint32_t dets_count = 0;
uint32_t dets_count_after_nms = 0;
cv::Mat cropped_image;
cv::Mat resized_image[MAX_FACES];
char cropped_path[] = {'f','i','l','e','_','1','i','m','a','g','e','_','0','.','p','n','g','\0'};
std::vector <cv::String > file_names;
char HEF_FILES[HEF_COUNT][MAX_HEF_PATH_LEN] = {"yolov5m_vehicles_bicycles_faces_acc.hef", "hse_age_gender_mobilenet_v2.hef"};
uint8_t HEF_MAX_BATCH_COUNT[HEF_COUNT] = {1, MAX_FACES};
cv::Mat pp_frame(640, 640, CV_8UC3);
// define the input folder as images
std::string source ="images/";
printf("Usage: %s \n", source.c_str());
cv::glob(source.c_str(), file_names, false);
// Create the PCIE device - look for a device connected
status = hailo_create_pcie_device(NULL, &device);
REQUIRE_SUCCESS(status, l_release_image, "Failed to create pcie_device");
/*******************************/
/* MODEL PREPARATION PART */
/*******************************/
// For each one of the models we would like to run, init the config params and build the streams
for (hef_index = 0; hef_index < HEF_COUNT; hef_index++)
{
status = hailo_create_hef_file(&hef[hef_index], HEF_FILES[hef_index]);
REQUIRE_SUCCESS(status, l_release_hef, "Failed creating hef file %s", HEF_FILES[hef_index]);
status = hailo_init_configure_params(hef[hef_index], HAILO_STREAM_INTERFACE_PCIE, &configure_params);
REQUIRE_SUCCESS(status, l_release_hef, "Failed init configure params");
status = hailo_configure_device(device, hef[hef_index], &configure_params, &network_groups[hef_index], &network_groups_size);
REQUIRE_SUCCESS(status, l_release_hef, "Failed configuring devcie");
REQUIRE_ACTION(network_groups_size == 1, status = HAILO_INVALID_ARGUMENT, l_release_hef,
"Unexpected network group size");
status = build_streams(network_groups[hef_index],
input_vstreams[hef_index], input_frame_size[hef_index],
output_vstreams[hef_index], output_frame_size[hef_index],
dst_data[hef_index], &num_output_vstreams[hef_index], HEF_MAX_BATCH_COUNT[hef_index]);
REQUIRE_SUCCESS(status, l_release_vstreams, "Failed building streams");
}
// get the output streams info for all outputs for the detection model
for (anchor_index = 0; ANCHORS_NUM > anchor_index; anchor_index++)
{
status = hailo_get_output_vstream_info(output_vstreams[0][anchor_index], &(yolo_output_stream_info[anchor_index]));
REQUIRE_SUCCESS(status, l_release_vstreams, "Failed getting Yolo vstreams info");
}
// get the output streams info for the age and gender model
status = hailo_get_output_vstream_info(output_vstreams[1][HSE_OUTPUT_AGE_VECTOR], &age_stream_info);
REQUIRE_SUCCESS(status, l_release_vstreams, "Failed getting age vstream info");
/*******************************/
/* INFERENCE PART */
/*******************************/
// loop on all files in the file folder
for (std::string file : file_names)
{
file_counter++;
printf("\nFILE NUMBER: %d\n\n",file_counter);
// read the image from the file
input_image = cv::imread(file);
cv::resize(input_image, g_frame[i], cv::Size(640, 640), 1);
input_image_rgb = g_frame[i];
/******************************/
/* Run YoloV5 detection */
/******************************/
status = run_network(network_groups[0], input_vstreams[0][0], &(input_image_rgb.data), 1, input_frame_size[0][0], output_vstreams[0], num_output_vstreams[0], dst_data[0], output_frame_size[0]);
REQUIRE_SUCCESS(status, l_release_vstreams, "Failed to run YoloV5m");
printf("-I- YoloV5 ran successfully.\n");
// Process Yolo raw results into bounding boxes.
for (anchor_index = 0; ANCHORS_NUM > anchor_index ; anchor_index++)
{
extract_boxes(dst_data[0][anchor_index][0], yolo_output_stream_info[anchor_index].quant_info.qp_zp,
yolo_output_stream_info[anchor_index].quant_info.qp_scale, g_feature_map_size[anchor_index],
g_anchors[anchor_index], CONFIDENCE_THRESHOLD, &box_index, bbox_array);
}
dets_count = box_index;
dets_count_after_nms = dets_count;
printf("-I- Detections before NMS: %u.\n", dets_count);
for (uint32_t i = 0; i < box_index; ++i)
{
if (bbox_array[i][4] > CONFIDENCE_THRESHOLD)
{
for (uint32_t j = i + 1; j < box_index; ++j)
{
if ((bbox_array[i][5] == bbox_array[j][5]) && (bbox_array[j][4] > CONFIDENCE_THRESHOLD))
{
if (iou_calc_c(&bbox_array[i][0], &bbox_array[j][0]) >= IOU_THRESHOLD)
{
bbox_array[j][4] = 0;
dets_count_after_nms -= 1;
}
}
}
}
}
box_index = 0;
printf("-I- Detections after NMS: %u.\n", dets_count_after_nms);
faces_count = 0;
for(uint32_t j = 0; (j < dets_count) && (faces_count < MAX_FACES); j++)
{
// Validate detection is a face and that the entire box is inside the image borders.
if (0 == bbox_array[j][4])
{
continue;
}
printf("Class ID: %f)\n", bbox_array[j][CLASS_ID]);
if ((3 == bbox_array[j][CLASS_ID]) && (0 <= bbox_array[j][YMIN]) && (0 <= bbox_array[j][XMIN]) &&
(IMAGE_SIZE > bbox_array[j][YMAX]) && (IMAGE_SIZE > bbox_array[j][XMAX]))
{
printf("Face %u at (%f, %f), (%f, %f)\n", faces_count, bbox_array[j][XMIN], bbox_array[j][YMIN], bbox_array[j][XMAX], bbox_array[j][YMAX]);
cropped_image = input_image_rgb(cv::Range(bbox_array[j][YMIN], bbox_array[j][YMAX]), cv::Range(bbox_array[j][XMIN], bbox_array[j][XMAX]));
cropped_path[5] = '0' + file_counter;
cropped_path[12] = '0' + faces_count;
cv::imwrite(cropped_path, cropped_image);
cv::resize(cropped_image, resized_image[faces_count], cv::Size(224,224), cv::INTER_LINEAR);
faces_input[faces_count] = resized_image[faces_count].data;
faces_count++;
}
}
/******************************/
/* Run Age/Gender recognition */
/******************************/
// if any faces were found run the age/gender recognition otherwise go to the next file
if (0 < faces_count)
{
status = run_network(network_groups[1], input_vstreams[1][0], faces_input, faces_count, input_frame_size[1][0], output_vstreams[1], num_output_vstreams[1], dst_data[1], output_frame_size[1]);
REQUIRE_SUCCESS(status, l_release_vstreams, "Failed to run Age/Gender model");
printf("-I- HSE ran successfully.\n");
// for each face print the gender and age recognition
for (uint8_t face_index = 0; face_index < faces_count; face_index++)
{
printf("Face %u:\n", face_index);
if (0.6 <= fix_scale(dst_data[1][HSE_OUTPUT_GENDER][face_index][0], age_stream_info.quant_info.qp_scale, age_stream_info.quant_info.qp_zp))
{
printf("\tMale - ");
}
else
{
printf("\tFemale - ");
}
printf("%u\n", get_age(dst_data[1][HSE_OUTPUT_AGE_VECTOR][face_index], output_frame_size[1][HSE_OUTPUT_AGE_VECTOR]));
}
}
}
status = HAILO_SUCCESS;
goto l_release_vstreams;
l_release_vstreams:
for (hef_index = 0; hef_index < HEF_COUNT; hef_index++)
{
(void)hailo_release_output_vstreams(output_vstreams[hef_index], num_output_vstreams[hef_index]);
(void)hailo_release_input_vstreams(input_vstreams[hef_index], 1);
for (hef_index = 0; hef_index < HEF_COUNT; hef_index++)
{
// TODO: Add proper release of Mat inputs?
for (size_t i = 0; i < num_output_vstreams[hef_index]; i++)
{
if (NULL != dst_data[hef_index] && NULL != dst_data[hef_index][i])
{
for (uint8_t j; j < MAX_BATCH; j++)
{
FREE(dst_data[hef_index][i][j]);
}
}
}
}
}
l_release_hef:
for (hef_index = 0; hef_index < HEF_COUNT; hef_index++)
{
if (NULL != hef[hef_index])
{
(void)hailo_release_hef(hef[hef_index]);
}
}
(void)hailo_release_device(device);
l_release_image:
input_image.release();
//l_exit:
return status;
}

201
cpp_inference/yolov5.cpp Normal file
View File

@ -0,0 +1,201 @@
#include "yolov5.hpp"
#include <cmath>
#include <stdint.h>
#include <stdio.h>
float fix_scale(float32_t input, float32_t qp_scale, float32_t qp_zp)
{
return (input - qp_zp) * qp_scale;
}
inline float minm(float n1, float n2)
{
if (n1<n2) {
return n1;
}
return n2;
}
inline float maxm(float n1, float n2)
{
if (n1>n2) {
return n1;
}
return n2;
}
float32_t iou_calc_c(float32_t *box_1, float32_t *box_2)
{
float width_of_overlap_area = minm(box_1[XMAX], box_2[XMAX]) - maxm(box_1[XMIN], box_2[XMIN]);
float height_of_overlap_area = minm(box_1[YMAX], box_2[YMAX]) - maxm(box_1[YMIN], box_2[YMIN]);
float positive_width_of_overlap_area = maxm(width_of_overlap_area + 1, 0.0f);
float positive_height_of_overlap_area = maxm(height_of_overlap_area + 1, 0.0f);
float area_of_overlap = positive_width_of_overlap_area * positive_height_of_overlap_area;
float box_1_area = (box_1[YMAX] - box_1[YMIN] + 1) * (box_1[XMAX] - box_1[XMIN] + 1);
float box_2_area = (box_2[YMAX] - box_2[YMIN] + 1) * (box_2[XMAX] - box_2[XMIN] + 1);
return area_of_overlap / (box_1_area + box_2_area - area_of_overlap);
}
#if 1
static void get_class(uint8_t *fm, uint32_t row, uint32_t col, uint32_t anchor, uint32_t feature_map_size,
uint32_t *class_id, uint32_t *class_prob)
{
uint32_t cls_prob, prob_max = 0;
uint32_t selected_class_id = 1;
for (uint32_t cls_id = 1; cls_id <= CLASSES_COUNT; ++cls_id)
{
uint32_t cls_prob_tensor_index = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * col + FEATURE_MAP_CHANNELS * anchor + CLASS_CHANNEL_OFFSET + cls_id - 1;
cls_prob = fm[cls_prob_tensor_index];
if (cls_prob > prob_max)
{
selected_class_id = cls_id;
prob_max = cls_prob;
}
}
*class_prob = prob_max;
*class_id = selected_class_id;
return;
}
void extract_boxes(uint8_t *fm, float32_t qp_zp, float32_t qp_scale, uint32_t feature_map_size,
int* anchors, float32_t thr, uint32_t *box_index, float32_t (*box_array)[6])
{
float32_t confidence, x, y, h, w, xmin, ymin, xmax, ymax = 0.0f;
uint32_t confidence_tensor_index = 0;
uint32_t class_id = 0;
uint32_t class_prob_int = 0;
float32_t class_prob = 0.0f;
uint32_t x_tensor_index = 0;
for (uint32_t row = 0; row < feature_map_size; ++row) {
for (uint32_t col = 0; col < feature_map_size; ++col) {
for (uint32_t anchor = 0; anchor < ANCHORS_NUM; ++anchor) {
confidence_tensor_index = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * col + FEATURE_MAP_CHANNELS * anchor + CONF_CHANNEL_OFFSET;
if (feature_map_size==20 && confidence_tensor_index >= 9600) {
printf("row: %u col %u anchor %u\n", row, col, anchor);
}
confidence = fix_scale(fm[confidence_tensor_index], qp_scale, qp_zp);
if (confidence < thr) {
continue;
}
get_class(fm, row, col, anchor, feature_map_size, &class_id, &class_prob_int);
class_prob = fix_scale(class_prob_int, qp_scale, qp_zp);
confidence = class_prob * confidence;
if (confidence > thr)
{
//printf("class_prob: %f, confidence: %f\n", class_prob, confidence);
x_tensor_index = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * col + FEATURE_MAP_CHANNELS * anchor;
if (row == 1 && col == 1 && anchor == 1) {
printf("x index: %u\n", x_tensor_index);
}
x = (fix_scale(fm[x_tensor_index], qp_scale, qp_zp) * 2.0f - 0.5f + col) / feature_map_size;
y = (fix_scale(fm[x_tensor_index + 1], qp_scale, qp_zp) * 2.0f - 0.5f + row) / feature_map_size;
w = pow(2.0f * (fix_scale(fm[x_tensor_index + 2], qp_scale, qp_zp)), 2.0f) * anchors[anchor * 2] / IMAGE_SIZE;
h = pow(2.0f * (fix_scale(fm[x_tensor_index + 3], qp_scale, qp_zp)), 2.0f) * anchors[anchor * 2 + 1] / IMAGE_SIZE;
xmin = (x - (w / 2.0f)) * IMAGE_SIZE;
ymin = (y - (h / 2.0f)) * IMAGE_SIZE;
xmax = (x + (w / 2.0f)) * IMAGE_SIZE;
ymax = (y + (h / 2.0f)) * IMAGE_SIZE;
if (*box_index < MAX_BOXES)
{
//printf("class type %d\n", chosen_cls);
box_array[*box_index][0] = ymin;
box_array[*box_index][1] = xmin;
box_array[*box_index][2] = ymax;
box_array[*box_index][3] = xmax;
box_array[*box_index][4] = confidence;
box_array[*box_index][5] = class_id;
*box_index = *box_index + 1;
}
}
}
}
}
}
#endif
#if 0
void extract_boxes(uint8_t *fm, float32_t qp_zp, float32_t qp_scale, int feature_map_size,
int* anchors, float32_t thr, uint32_t *box_index, float32_t (*box_array)[6])
{
float32_t confidence, x, y, h, w, xmin, ymin, xmax, ymax, conf_max = 0.0f;
int add = 0, anchor = 0, chosen_row = 0, chosen_col = 0, chosen_cls = -1;
float32_t cls_prob, prob_max = 0.0f;
// channels 0-3 are box coordinates, channel 4 is the confidence, and channels 5-84 are classes
for (int row = 0; row < feature_map_size; ++row) {
for (int col = 0; col < feature_map_size; ++col) {
prob_max = 0;
for (int a = 0; a < ANCHORS_NUM; ++a) {
add = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * col + FEATURE_MAP_CHANNELS * a + CONF_CHANNEL_OFFSET;
//confidence = fix_scale(fm[add], qp_scale, qp_zp);
confidence = (fm[add])*qp_scale;
if (confidence > thr)
//printf("no way we are here %f\n",qp_scale);
for (int c = CLASS_CHANNEL_OFFSET; c < FEATURE_MAP_CHANNELS; ++c) {
add = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * col + FEATURE_MAP_CHANNELS * a + c;
// final confidence: box confidence * class probability
cls_prob = fm[add];
if (cls_prob > prob_max)
{
conf_max = fix_scale(cls_prob, qp_scale, qp_zp) * confidence;
chosen_cls = c - CLASS_CHANNEL_OFFSET + 1;
prob_max = cls_prob;
anchor = a;
chosen_row = row;
chosen_col = col;
}
}
}
float basetemp;// = 2.0f * (fix_scale(fm[add + 2], qp_scale, qp_zp));
float tempresult = 1.0;
//float exptemp = 2.0;
if (conf_max > thr) {
add = FEATURE_MAP_CHANNELS * ANCHORS_NUM * feature_map_size * chosen_row + FEATURE_MAP_CHANNELS * ANCHORS_NUM * chosen_col + FEATURE_MAP_CHANNELS * anchor;
x = (fix_scale(fm[add], qp_scale, qp_zp) * 2.0f - 0.5f + chosen_col) / feature_map_size;
y = (fix_scale(fm[add + 1], qp_scale, qp_zp) * 2.0f - 0.5f + chosen_row) / feature_map_size;
basetemp = 2.0f * (fix_scale(fm[add + 2], qp_scale, qp_zp));
//tempresult = 1.0f;
//exptemp = 2.0f;
//while (exptemp != 0) {
// tempresult *= basetemp;
// --exptemp;
//}
tempresult = basetemp*basetemp;
w = tempresult * anchors[anchor * 2] / IMAGE_SIZE;
basetemp = 2.0f * (fix_scale(fm[add + 3], qp_scale, qp_zp));
//tempresult = 1.0f;
//exptemp = 2.0f;
//while (exptemp != 0) {
// tempresult *= basetemp;
// --exptemp;
//}
tempresult = basetemp*basetemp;
h = tempresult * anchors[anchor * 2 + 1] / IMAGE_SIZE;
xmin = (x - (w / 2.0f)) * IMAGE_SIZE;
ymin = (y - (h / 2.0f)) * IMAGE_SIZE;
xmax = (x + (w / 2.0f)) * IMAGE_SIZE;
ymax = (y + (h / 2.0f)) * IMAGE_SIZE;
if (*box_index < MAX_BOXES)
{
//printf("class type %d\n", chosen_cls);
box_array[*box_index][0] = ymin;
box_array[*box_index][1] = xmin;
box_array[*box_index][2] = ymax;
box_array[*box_index][3] = xmax;
box_array[*box_index][4] = conf_max;
box_array[*box_index][5] = chosen_cls;
*box_index = *box_index + 1;
}
}
}
}
}
#endif

57
cpp_inference/yolov5.hpp Normal file
View File

@ -0,0 +1,57 @@
#ifndef _YOLOV5M_
#define _YOLOV5M_
#include <stdint.h>
#if 0
#define FEATURE_MAP_SIZE1 20
#define FEATURE_MAP_SIZE2 40
#define FEATURE_MAP_SIZE3 80
#endif
#if 1
#define FEATURE_MAP_SIZE1 80
#define FEATURE_MAP_SIZE2 40
#define FEATURE_MAP_SIZE3 20
#endif
//#define FEATURE_MAP_CHANNELS 85
#define FEATURE_MAP_CHANNELS 8
#define IMAGE_SIZE 640
#define ANCHORS_NUM 3
#define IOU_THRESHOLD 0.45f
#define CONFIDENCE_THRESHOLD 0.7f
#define MAX_BOXES 100
#define CLASSES_COUNT 3
//#define CLASSES_COUNT 90
#define CONF_CHANNEL_OFFSET 4
#define CLASS_CHANNEL_OFFSET 5
#define YMIN 0
#define XMIN 1
#define YMAX 2
#define XMAX 3
#define CONFIDENCE 4
#define CLASS_ID 5
typedef float float32_t;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
static uint8_t g_feature_map_size[ANCHORS_NUM] = {FEATURE_MAP_SIZE1, FEATURE_MAP_SIZE2, FEATURE_MAP_SIZE3};
#if 1
static int32_t g_anchors[ANCHORS_NUM][6] = {{10, 13, 16, 30, 33, 23},
{30, 61, 62, 45, 59, 119},
{116, 90, 156, 198, 373, 326}};
#endif
#if 0
static int32_t g_anchors[ANCHORS_NUM][6] = {{116, 90, 156, 198, 373, 326},
{30, 61, 62, 45, 59, 119},
{10, 13, 16, 30, 33, 23}};
#endif
#pragma GCC diagnostic pop
void extract_boxes(uint8_t *fm, float32_t qp_zp, float32_t qp_scale, uint32_t feature_map_size,
int* anchors, float32_t thr, uint32_t *box_index, float32_t (*box_array)[6]);
float32_t iou_calc_c(float32_t *box_1, float32_t *box_2);
float fix_scale(float32_t input, float32_t qp_scale, float32_t qp_zp);
#endif /* _YOLOV5M_ */

BIN
cpp_inference/yolov5m_vehicles_bicycles_faces_acc.hef Normal file
View File

Binary file not shown.

0
inference.py → python_inference/inference.py
View File

0
ros_inference.py → python_inference/ros_inference.py
View File