OpenShot Library | libopenshot 0.3.2
Tracker.cpp
Go to the documentation of this file.
1
10// Copyright (c) 2008-2019 OpenShot Studios, LLC
11//
12// SPDX-License-Identifier: LGPL-3.0-or-later
13
14#include <string>
15#include <memory>
16#include <fstream>
17#include <iostream>
18
19#include "effects/Tracker.h"
20#include "Exceptions.h"
21#include "Timeline.h"
22#include "trackerdata.pb.h"
23
24#include <google/protobuf/util/time_util.h>
25
26#include <QImage>
27#include <QPainter>
28#include <QRectF>
29
30using namespace std;
31using namespace openshot;
32using google::protobuf::util::TimeUtil;
33
35Tracker::Tracker(std::string clipTrackerDataPath)
36{
37 // Init effect properties
38 init_effect_details();
39 // Instantiate a TrackedObjectBBox object and point to it
40 TrackedObjectBBox trackedDataObject;
41 trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
42 // Tries to load the tracked object's data from protobuf file
43 trackedData->LoadBoxData(clipTrackerDataPath);
44 ClipBase* parentClip = this->ParentClip();
45 trackedData->ParentClip(parentClip);
46 trackedData->Id(std::to_string(0));
47 // Insert TrackedObject with index 0 to the trackedObjects map
48 trackedObjects.insert({0, trackedData});
49}
50
51// Default constructor
53{
54 // Init effect properties
55 init_effect_details();
56 // Instantiate a TrackedObjectBBox object and point to it
57 TrackedObjectBBox trackedDataObject;
58 trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
59 ClipBase* parentClip = this->ParentClip();
60 trackedData->ParentClip(parentClip);
61 trackedData->Id(std::to_string(0));
62 // Insert TrackedObject with index 0 to the trackedObjects map
63 trackedObjects.insert({0, trackedData});
64}
65
66
67// Init effect settings
68void Tracker::init_effect_details()
69{
72
74 info.class_name = "Tracker";
75 info.name = "Tracker";
76 info.description = "Track the selected bounding box through the video.";
77 info.has_audio = false;
78 info.has_video = true;
80
81 this->TimeScale = 1.0;
82}
83
84// This method is required for all derived classes of EffectBase, and returns a
85// modified openshot::Frame object
86std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
87{
88 // Get the frame's image
89 cv::Mat frame_image = frame->GetImageCV();
90
91 // Initialize the Qt rectangle that will hold the positions of the bounding-box
92 QRectF boxRect;
93 // Initialize the image of the TrackedObject child clip
94 std::shared_ptr<QImage> childClipImage = nullptr;
95
96 // Check if frame isn't NULL
97 if(!frame_image.empty() &&
98 trackedData->Contains(frame_number) &&
99 trackedData->visible.GetValue(frame_number) == 1)
100 {
101 // Get the width and height of the image
102 float fw = frame_image.size().width;
103 float fh = frame_image.size().height;
104
105 // Get the bounding-box of given frame
106 BBox fd = trackedData->GetBox(frame_number);
107
108 // Check if track data exists for the requested frame
109 if (trackedData->draw_box.GetValue(frame_number) == 1)
110 {
111 std::vector<int> stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number);
112 int stroke_width = trackedData->stroke_width.GetValue(frame_number);
113 float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number);
114 std::vector<int> bg_rgba = trackedData->background.GetColorRGBA(frame_number);
115 float bg_alpha = trackedData->background_alpha.GetValue(frame_number);
116
117 // Create a rotated rectangle object that holds the bounding box
118 cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ),
119 cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ),
120 (int) (fd.angle) );
121
122 DrawRectangleRGBA(frame_image, box, bg_rgba, bg_alpha, 1, true);
123 DrawRectangleRGBA(frame_image, box, stroke_rgba, stroke_alpha, stroke_width, false);
124 }
125
126 // Get the image of the Tracked Object' child clip
127 if (trackedData->ChildClipId() != ""){
128 // Cast the parent timeline of this effect
129 Timeline* parentTimeline = static_cast<Timeline *>(ParentTimeline());
130 if (parentTimeline){
131 // Get the Tracked Object's child clip
132 Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId());
133 if (childClip){
134 // Get the image of the child clip for this frame
135 std::shared_ptr<Frame> childClipFrame = childClip->GetFrame(frame_number);
136 childClipImage = childClipFrame->GetImage();
137
138 // Set the Qt rectangle with the bounding-box properties
139 boxRect.setRect((int)((fd.cx-fd.width/2)*fw),
140 (int)((fd.cy - fd.height/2)*fh),
141 (int)(fd.width*fw),
142 (int)(fd.height*fh) );
143 }
144 }
145 }
146
147 }
148
149 // Set image with drawn box to frame
150 // If the input image is NULL or doesn't have tracking data, it's returned as it came
151 frame->SetImageCV(frame_image);
152
153 // Set the bounding-box image with the Tracked Object's child clip image
154 if (childClipImage){
155 // Get the frame image
156 QImage frameImage = *(frame->GetImage());
157
158 // Set a Qt painter to the frame image
159 QPainter painter(&frameImage);
160
161 // Draw the child clip image inside the bounding-box
162 painter.drawImage(boxRect, *childClipImage);
163
164 // Set the frame image as the composed image
165 frame->AddImage(std::make_shared<QImage>(frameImage));
166 }
167
168 return frame;
169}
170
171void Tracker::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color, float alpha, int thickness, bool is_background){
172 // Get the bouding box vertices
173 cv::Point2f vertices2f[4];
174 box.points(vertices2f);
175
176 // TODO: take a rectangle of frame_image by refencence and draw on top of that to improve speed
177 // select min enclosing rectangle to draw on a small portion of the image
178 // cv::Rect rect = box.boundingRect();
179 // cv::Mat image = frame_image(rect)
180
181 if(is_background){
182 cv::Mat overlayFrame;
183 frame_image.copyTo(overlayFrame);
184
185 // draw bounding box background
186 cv::Point vertices[4];
187 for(int i = 0; i < 4; ++i){
188 vertices[i] = vertices2f[i];}
189
190 cv::Rect rect = box.boundingRect();
191 cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
192 // add opacity
193 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
194 }
195 else{
196 cv::Mat overlayFrame;
197 frame_image.copyTo(overlayFrame);
198
199 // Draw bounding box
200 for (int i = 0; i < 4; i++)
201 {
202 cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
203 thickness, cv::LINE_AA);
204 }
205
206 // add opacity
207 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
208 }
209}
210
211// Get the indexes and IDs of all visible objects in the given frame
212std::string Tracker::GetVisibleObjects(int64_t frame_number) const{
213
214 // Initialize the JSON objects
215 Json::Value root;
216 root["visible_objects_index"] = Json::Value(Json::arrayValue);
217 root["visible_objects_id"] = Json::Value(Json::arrayValue);
218
219 // Iterate through the tracked objects
220 for (const auto& trackedObject : trackedObjects){
221 // Get the tracked object JSON properties for this frame
222 Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number);
223 if (trackedObjectJSON["visible"]["value"].asBool()){
224 // Save the object's index and ID if it's visible in this frame
225 root["visible_objects_index"].append(trackedObject.first);
226 root["visible_objects_id"].append(trackedObject.second->Id());
227 }
228 }
229
230 return root.toStyledString();
231}
232
233// Generate JSON string of this object
234std::string Tracker::Json() const {
235
236 // Return formatted string
237 return JsonValue().toStyledString();
238}
239
240// Generate Json::Value for this object
241Json::Value Tracker::JsonValue() const {
242
243 // Create root json object
244 Json::Value root = EffectBase::JsonValue(); // get parent properties
245
246 // Save the effect's properties on root
247 root["type"] = info.class_name;
248 root["protobuf_data_path"] = protobuf_data_path;
249 root["BaseFPS"]["num"] = BaseFPS.num;
250 root["BaseFPS"]["den"] = BaseFPS.den;
251 root["TimeScale"] = this->TimeScale;
252
253 // Add trackedObjects IDs to JSON
254 Json::Value objects;
255 for (auto const& trackedObject : trackedObjects){
256 Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
257 // add object json
258 objects[trackedObject.second->Id()] = trackedObjectJSON;
259 }
260 root["objects"] = objects;
261
262 // return JsonValue
263 return root;
264}
265
266// Load JSON string into this object
267void Tracker::SetJson(const std::string value) {
268
269 // Parse JSON string into JSON objects
270 try
271 {
272 const Json::Value root = openshot::stringToJson(value);
273 // Set all values that match
274 SetJsonValue(root);
275 }
276 catch (const std::exception& e)
277 {
278 // Error parsing JSON (or missing keys)
279 throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
280 }
281 return;
282}
283
284// Load Json::Value into this object
285void Tracker::SetJsonValue(const Json::Value root) {
286
287 // Set parent data
289
290 if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject())
291 {
292 if (!root["BaseFPS"]["num"].isNull())
293 {
294 BaseFPS.num = (int) root["BaseFPS"]["num"].asInt();
295 }
296 if (!root["BaseFPS"]["den"].isNull())
297 {
298 BaseFPS.den = (int) root["BaseFPS"]["den"].asInt();
299 }
300 }
301
302 if (!root["TimeScale"].isNull())
303 TimeScale = (double) root["TimeScale"].asDouble();
304
305 // Set data from Json (if key is found)
306 if (!root["protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1)
307 {
308 protobuf_data_path = root["protobuf_data_path"].asString();
309 if(!trackedData->LoadBoxData(protobuf_data_path))
310 {
311 std::clog << "Invalid protobuf data path " << protobuf_data_path << '\n';
313 }
314 }
315
316 if (!root["objects"].isNull()){
317 for (auto const& trackedObject : trackedObjects){
318 std::string obj_id = std::to_string(trackedObject.first);
319 if(!root["objects"][obj_id].isNull()){
320 trackedObject.second->SetJsonValue(root["objects"][obj_id]);
321 }
322 }
323 }
324
325 // Set the tracked object's ids
326 if (!root["objects_id"].isNull()){
327 for (auto const& trackedObject : trackedObjects){
328 Json::Value trackedObjectJSON;
329 trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString();
330 trackedObject.second->SetJsonValue(trackedObjectJSON);
331 }
332 }
333
334 return;
335}
336
337// Get all properties for a specific frame
338std::string Tracker::PropertiesJSON(int64_t requested_frame) const {
339
340 // Generate JSON properties list
341 Json::Value root;
342
343 // Add trackedObject properties to JSON
344 Json::Value objects;
345 for (auto const& trackedObject : trackedObjects){
346 Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
347 // add object json
348 objects[trackedObject.second->Id()] = trackedObjectJSON;
349 }
350 root["objects"] = objects;
351
352 // Append effect's properties
353 root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame);
354 root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
355 root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame);
356 root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
357 root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
358 root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame);
359
360 // Return formatted string
361 return root.toStyledString();
362}
Header file for all Exception classes.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
Definition: ClipBase.h:33
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition: ClipBase.h:88
float Duration() const
Get the length of this clip (in seconds)
Definition: ClipBase.h:90
virtual float End() const
Get end position (in seconds) of clip (trim end of video)
Definition: ClipBase.h:89
std::string Id() const
Get the Id of this clip object.
Definition: ClipBase.h:85
int Layer() const
Get layer of clip on timeline (lower number is covered by higher numbers)
Definition: ClipBase.h:87
float Position() const
Get position on timeline (in seconds)
Definition: ClipBase.h:86
virtual openshot::TimelineBase * ParentTimeline()
Get the associated Timeline pointer (if any)
Definition: ClipBase.h:91
Json::Value add_property_json(std::string name, float value, std::string type, std::string memo, const Keyframe *keyframe, float min_value, float max_value, bool readonly, int64_t requested_frame) const
Generate JSON for a property.
Definition: ClipBase.cpp:96
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:89
std::shared_ptr< openshot::Frame > GetFrame(int64_t clip_frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:389
virtual Json::Value JsonValue() const
Generate Json::Value for this object.
Definition: EffectBase.cpp:77
openshot::ClipBase * ParentClip()
Parent clip object of this effect (which can be unparented and NULL)
Definition: EffectBase.cpp:173
virtual void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: EffectBase.cpp:112
EffectInfoStruct info
Information about the current effect.
Definition: EffectBase.h:69
std::map< int, std::shared_ptr< openshot::TrackedObjectBase > > trackedObjects
Map of Tracked Object's by their indices (used by Effects that track objects on clips)
Definition: EffectBase.h:66
int num
Numerator for the fraction.
Definition: Fraction.h:32
int den
Denominator for the fraction.
Definition: Fraction.h:33
Exception for invalid JSON.
Definition: Exceptions.h:218
This class represents a timeline.
Definition: Timeline.h:150
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
Definition: Timeline.cpp:408
This class contains the properties of a tracked object and functions to manipulate it.
std::string Json() const override
Generate JSON string of this object.
Definition: Tracker.cpp:234
std::string GetVisibleObjects(int64_t frame_number) const override
Get the indexes and IDs of all visible objects in the given frame.
Definition: Tracker.cpp:212
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: Tracker.cpp:241
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: Tracker.cpp:267
std::shared_ptr< Frame > GetFrame(std::shared_ptr< Frame > frame, int64_t frame_number) override
Apply this effect to an openshot::Frame.
Definition: Tracker.cpp:86
std::string PropertiesJSON(int64_t requested_frame) const override
Definition: Tracker.cpp:338
std::shared_ptr< TrackedObjectBBox > trackedData
Pointer to an object that holds the bounding-box data and it's Keyframes.
Definition: Tracker.h:52
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: Tracker.cpp:285
Tracker()
Default constructor.
Definition: Tracker.cpp:52
void DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector< int > color, float alpha, int thickness, bool is_background)
Definition: Tracker.cpp:171
std::string protobuf_data_path
Path to the protobuf file that holds the bounding-box data.
Definition: Tracker.h:51
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:29
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width
float angle
bounding box rotation angle [degrees]
bool has_video
Determines if this effect manipulates the image of a frame.
Definition: EffectBase.h:40
bool has_audio
Determines if this effect manipulates the audio of a frame.
Definition: EffectBase.h:41
std::string class_name
The class name of the effect.
Definition: EffectBase.h:36
std::string name
The name of the effect.
Definition: EffectBase.h:37
std::string description
The description of this effect and what it does.
Definition: EffectBase.h:38
bool has_tracked_object
Determines if this effect track objects through the clip.
Definition: EffectBase.h:42