21#include "objdetectdata.pb.h"
31ObjectDetection::ObjectDetection(std::string clipObDetectDataPath)
34 init_effect_details();
47 init_effect_details();
54void ObjectDetection::init_effect_details()
73 cv::Mat cv_image = frame->GetImageCV();
81 std::vector<QRectF> boxRects;
83 std::vector<std::shared_ptr<QImage>> childClipImages;
86 if (detectionsData.find(frame_number) != detectionsData.end()) {
87 float fw = cv_image.size().width;
88 float fh = cv_image.size().height;
91 for(
int i = 0; i<detections.
boxes.size(); i++){
94 if(detections.
confidences.at(i) < confidence_threshold){
98 if( display_classes.size() > 0 &&
99 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
104 int objectId = detections.
objectIds.at(i);
110 std::shared_ptr<TrackedObjectBBox> trackedObject = std::static_pointer_cast<TrackedObjectBBox>(trackedObject_it->second);
113 if (trackedObject->Contains(frame_number) &&
114 trackedObject->visible.GetValue(frame_number) == 1)
117 BBox trackedBox = trackedObject->GetBox(frame_number);
118 bool draw_text = !display_box_text.
GetValue(frame_number);
119 std::vector<int> stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number);
120 int stroke_width = trackedObject->stroke_width.GetValue(frame_number);
121 float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number);
122 std::vector<int> bg_rgba = trackedObject->background.GetColorRGBA(frame_number);
123 float bg_alpha = trackedObject->background_alpha.GetValue(frame_number);
126 (
int)( (trackedBox.
cx-trackedBox.
width/2)*fw),
127 (
int)( (trackedBox.
cy-trackedBox.
height/2)*fh),
128 (
int)( trackedBox.
width*fw),
129 (
int)( trackedBox.
height*fh)
133 if (trackedObject->draw_box.GetValue(frame_number) == 0)
140 box, cv_image, detections.
objectIds.at(i), bg_rgba, bg_alpha, 1,
true, draw_text);
142 box, cv_image, detections.
objectIds.at(i), stroke_rgba, stroke_alpha, stroke_width,
false, draw_text);
146 if (trackedObject->ChildClipId() !=
""){
151 Clip* childClip = parentTimeline->
GetClip(trackedObject->ChildClipId());
155 std::shared_ptr<Frame> childClipFrame = childClip->
GetFrame(frame_number);
156 childClipImages.push_back(childClipFrame->GetImage());
160 boxRect.setRect((
int)((trackedBox.
cx-trackedBox.
width/2)*fw),
161 (int)((trackedBox.
cy - trackedBox.
height/2)*fh),
162 (
int)(trackedBox.
width*fw),
163 (
int)(trackedBox.
height*fh));
164 boxRects.push_back(boxRect);
173 frame->SetImageCV(cv_image);
176 if(boxRects.size() > 0){
178 QImage frameImage = *(frame->GetImage());
179 for(
int i; i < boxRects.size();i++){
181 QPainter painter(&frameImage);
183 painter.drawImage(boxRects[i], *childClipImages[i]);
186 frame->AddImage(std::make_shared<QImage>(frameImage));
192void ObjectDetection::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color,
float alpha,
193 int thickness,
bool is_background){
195 cv::Point2f vertices2f[4];
196 box.points(vertices2f);
204 cv::Mat overlayFrame;
205 frame_image.copyTo(overlayFrame);
208 cv::Point vertices[4];
209 for(
int i = 0; i < 4; ++i){
210 vertices[i] = vertices2f[i];}
212 cv::Rect rect = box.boundingRect();
213 cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
215 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
218 cv::Mat overlayFrame;
219 frame_image.copyTo(overlayFrame);
222 for (
int i = 0; i < 4; i++)
224 cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
225 thickness, cv::LINE_AA);
229 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
233void ObjectDetection::drawPred(
int classId,
float conf, cv::Rect2d box, cv::Mat& frame,
int objectNumber, std::vector<int> color,
234 float alpha,
int thickness,
bool is_background,
bool display_text)
238 cv::Mat overlayFrame;
239 frame.copyTo(overlayFrame);
242 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
245 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
248 cv::Mat overlayFrame;
249 frame.copyTo(overlayFrame);
252 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), thickness);
256 std::string label = cv::format(
"%.2f", conf);
257 if (!classNames.empty())
259 CV_Assert(classId < (
int)classNames.size());
260 label = classNames[classId] +
":" + label;
265 cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
268 double top = std::max((
int)box.y, labelSize.height);
270 cv::rectangle(overlayFrame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine),
271 cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
272 putText(overlayFrame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1);
275 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
282 pb_objdetect::ObjDetect objMessage;
285 std::fstream input(inputFilePath, std::ios::in | std::ios::binary);
286 if (!objMessage.ParseFromIstream(&input)) {
287 std::cerr <<
"Failed to parse protobuf message." << std::endl;
293 detectionsData.clear();
299 for(
int i = 0; i < objMessage.classnames_size(); i++)
301 classNames.push_back(objMessage.classnames(i));
302 classesColor.push_back(cv::Scalar(std::rand()%205 + 50, std::rand()%205 + 50, std::rand()%205 + 50));
306 for (
size_t i = 0; i < objMessage.frame_size(); i++)
309 const pb_objdetect::Frame& pbFrameData = objMessage.frame(i);
312 size_t id = pbFrameData.id();
315 const google::protobuf::RepeatedPtrField<pb_objdetect::Frame_Box > &pBox = pbFrameData.bounding_box();
318 std::vector<int> classIds;
319 std::vector<float> confidences;
320 std::vector<cv::Rect_<float>> boxes;
321 std::vector<int> objectIds;
324 for(
int i = 0; i < pbFrameData.bounding_box_size(); i++)
327 float x = pBox.Get(i).x();
328 float y = pBox.Get(i).y();
329 float w = pBox.Get(i).w();
330 float h = pBox.Get(i).h();
332 int classId = pBox.Get(i).classid();
334 float confidence = pBox.Get(i).confidence();
337 int objectId = pBox.Get(i).objectid();
345 trackedObject->second->AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
350 TrackedObjectBBox trackedObj((
int)classesColor[classId](0), (
int)classesColor[classId](1), (
int)classesColor[classId](2), (
int)0);
351 trackedObj.
AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
353 std::shared_ptr<TrackedObjectBBox> trackedObjPtr = std::make_shared<TrackedObjectBBox>(trackedObj);
355 trackedObjPtr->ParentClip(parentClip);
359 trackedObjPtr->Id(std::to_string(objectId));
364 cv::Rect_<float> box(x, y, w, h);
367 boxes.push_back(box);
368 classIds.push_back(classId);
369 confidences.push_back(confidence);
370 objectIds.push_back(objectId);
374 detectionsData[
id] =
DetectionData(classIds, confidences, boxes,
id, objectIds);
378 google::protobuf::ShutdownProtobufLibrary();
388 root[
"visible_objects_index"] = Json::Value(Json::arrayValue);
389 root[
"visible_objects_id"] = Json::Value(Json::arrayValue);
392 if (detectionsData.find(frame_number) == detectionsData.end()){
393 return root.toStyledString();
398 for(
int i = 0; i<detections.
boxes.size(); i++){
400 if(detections.
confidences.at(i) < confidence_threshold){
405 if( display_classes.size() > 0 &&
406 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
410 int objectId = detections.
objectIds.at(i);
415 Json::Value trackedObjectJSON = trackedObject->second->PropertiesJSON(frame_number);
417 if (trackedObjectJSON[
"visible"][
"value"].asBool() &&
418 trackedObject->second->ExactlyContains(frame_number)){
420 root[
"visible_objects_index"].append(trackedObject->first);
421 root[
"visible_objects_id"].append(trackedObject->second->Id());
425 return root.toStyledString();
441 root[
"protobuf_data_path"] = protobuf_data_path;
443 root[
"confidence_threshold"] = confidence_threshold;
444 root[
"display_box_text"] = display_box_text.
JsonValue();
449 Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
451 objects[trackedObject.second->Id()] = trackedObjectJSON;
453 root[
"objects"] = objects;
469 catch (
const std::exception& e)
472 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
482 if (!root[
"protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1){
483 protobuf_data_path = root[
"protobuf_data_path"].asString();
486 throw InvalidFile(
"Invalid protobuf data path",
"");
487 protobuf_data_path =
"";
492 if (!root[
"selected_object_index"].isNull())
495 if (!root[
"confidence_threshold"].isNull())
496 confidence_threshold = root[
"confidence_threshold"].asFloat();
498 if (!root[
"display_box_text"].isNull())
499 display_box_text.
SetJsonValue(root[
"display_box_text"]);
501 if (!root[
"class_filter"].isNull()){
502 class_filter = root[
"class_filter"].asString();
503 std::stringstream ss(class_filter);
504 display_classes.clear();
509 std::getline( ss, substr,
',' );
510 display_classes.push_back( substr );
514 if (!root[
"objects"].isNull()){
516 std::string obj_id = std::to_string(trackedObject.first);
517 if(!root[
"objects"][obj_id].isNull()){
518 trackedObject.second->SetJsonValue(root[
"objects"][obj_id]);
524 if (!root[
"objects_id"].isNull()){
526 Json::Value trackedObjectJSON;
527 trackedObjectJSON[
"box_id"] = root[
"objects_id"][trackedObject.first].asString();
528 trackedObject.second->SetJsonValue(trackedObjectJSON);
543 Json::Value trackedObjectJSON = selectedObject->PropertiesJSON(requested_frame);
545 objects[selectedObject->Id()] = trackedObjectJSON;
548 root[
"objects"] = objects;
551 root[
"id"] =
add_property_json(
"ID", 0.0,
"string",
Id(), NULL, -1, -1,
true, requested_frame);
552 root[
"position"] =
add_property_json(
"Position",
Position(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
554 root[
"start"] =
add_property_json(
"Start",
Start(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
555 root[
"end"] =
add_property_json(
"End",
End(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
556 root[
"duration"] =
add_property_json(
"Duration",
Duration(),
"float",
"", NULL, 0, 1000 * 60 * 30,
true, requested_frame);
557 root[
"confidence_threshold"] =
add_property_json(
"Confidence Theshold", confidence_threshold,
"float",
"", NULL, 0, 1,
false, requested_frame);
558 root[
"class_filter"] =
add_property_json(
"Class Filter", 0.0,
"string", class_filter, NULL, -1, -1,
false, requested_frame);
560 root[
"display_box_text"] =
add_property_json(
"Draw Box Text", display_box_text.
GetValue(requested_frame),
"int",
"", &display_box_text, 0, 1.0,
false, requested_frame);
565 return root.toStyledString();
Header file for all Exception classes.
Header file for Object Detection effect class.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
float Start() const
Get start position (in seconds) of clip (trim start of video)
float Duration() const
Get the length of this clip (in seconds)
virtual float End() const
Get end position (in seconds) of clip (trim end of video)
std::string Id() const
Get the Id of this clip object.
Json::Value add_property_choice_json(std::string name, int value, int selected_value) const
Generate JSON choice for a property (dropdown properties)
int Layer() const
Get layer of clip on timeline (lower number is covered by higher numbers)
float Position() const
Get position on timeline (in seconds)
virtual openshot::TimelineBase * ParentTimeline()
Get the associated Timeline pointer (if any)
std::string id
ID Property for all derived Clip and Effect classes.
Json::Value add_property_json(std::string name, float value, std::string type, std::string memo, const Keyframe *keyframe, float min_value, float max_value, bool readonly, int64_t requested_frame) const
Generate JSON for a property.
This class represents a clip (used to arrange readers on the timeline)
std::shared_ptr< openshot::Frame > GetFrame(int64_t clip_frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
virtual Json::Value JsonValue() const
Generate Json::Value for this object.
openshot::ClipBase * ParentClip()
Parent clip object of this effect (which can be unparented and NULL)
virtual void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
EffectInfoStruct info
Information about the current effect.
std::map< int, std::shared_ptr< openshot::TrackedObjectBase > > trackedObjects
Map of Tracked Object's by their indices (used by Effects that track objects on clips)
Exception for files that can not be found or opened.
Exception for invalid JSON.
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
double GetValue(int64_t index) const
Get the value at a specific index.
Json::Value JsonValue() const
Generate Json::Value for this object.
Json::Value JsonValue() const override
Generate Json::Value for this object.
int selectedObjectIndex
Index of the Tracked Object that was selected to modify it's properties.
std::shared_ptr< Frame > GetFrame(std::shared_ptr< Frame > frame, int64_t frame_number) override
This method is required for all derived classes of EffectBase, and returns a modified openshot::Frame...
ObjectDetection()
Default constructor.
bool LoadObjDetectdData(std::string inputFilePath)
Load protobuf data file.
std::string GetVisibleObjects(int64_t frame_number) const override
Get the indexes and IDs of all visible objects in the given frame.
std::string Json() const override
Generate JSON string of this object.
std::string PropertiesJSON(int64_t requested_frame) const override
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
void SetJson(const std::string value) override
Load JSON string into this object.
This class represents a timeline.
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
This class contains the properties of a tracked object and functions to manipulate it.
void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle) override
Add a BBox to the BoxVec map.
This namespace is the default namespace for all code in the openshot library.
const Json::Value stringToJson(const std::string value)
std::vector< cv::Rect_< float > > boxes
std::vector< float > confidences
std::vector< int > classIds
std::vector< int > objectIds
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width
bool has_video
Determines if this effect manipulates the image of a frame.
bool has_audio
Determines if this effect manipulates the audio of a frame.
std::string class_name
The class name of the effect.
std::string name
The name of the effect.
std::string description
The description of this effect and what it does.
bool has_tracked_object
Determines if this effect track objects through the clip.