Scroll Top

Raft (default)

C++ (Windows, Linux, MacOS / CUDA and Metal accelerated) port of RAFT.

Example Input & Outputs

Inputs Outputs
Input
Frame 0
Input
Frame 1
Input
Normalized Optical Flow

Demo Code

 1#include "blace_ai.h"
 2#include <opencv2/opencv.hpp>
 3
 4// include the models you want to use
 5#include "raft_v1_default_v2_ALL_export_version_v17.h"
 6
 7/**
 8x
 9*/
10
11std::shared_ptr<blace::RawMemoryObject> memory_from_file(std::string file) {
12  // read image into memory
13  cv::Mat image = cv::imread(file, cv::IMREAD_COLOR);
14
15  // construct a random hash from the filename
16  std::hash<std::string> seeder;
17  int seed = seeder(file);
18  blace::ml_core::BlaceHash random_hash(seed);
19
20  // construct the memory object. We set copy_memory to true, since image will
21  // be out-of-scope upon method return and therefore we need to take ownership
22  // of the data
23  blace::RawMemoryObject raw_mem(
24      (void *)image.data, blace::ml_core::DataTypeEnum::BLACE_BYTE,
25      blace::ml_core::ColorFormatEnum::BGR,
26      std::vector<int64_t>{1, image.rows, image.cols, 3}, blace::ml_core::BHWC,
27      blace::ml_core::ZERO_TO_255, random_hash, true);
28
29  return std::make_shared<blace::RawMemoryObject>(raw_mem);
30}
31
32int main() {
33
34  // register model at server
35  blace::util::registerModel(raft_v1_default_v2_ALL_export_version_v17,
36                             blace::util::getPathToExe());
37
38  // load image into op
39  auto exe_path = blace::util::getPathToExe();
40  std::filesystem::path frame_0 = exe_path / "raft_frame_0.png";
41  std::filesystem::path frame_1 = exe_path / "raft_frame_1.png";
42
43  auto frame_0_mem = memory_from_file(frame_0.string());
44  auto frame_1_mem = memory_from_file(frame_1.string());
45
46  auto frame_0_op = CONSTRUCT_OP_GET(blace::ops::FromRawMemoryOp(frame_0_mem));
47  auto frame_1_op = CONSTRUCT_OP_GET(blace::ops::FromRawMemoryOp(frame_1_mem));
48
49  // construct model inference arguments
50  blace::ml_core::InferenceArgsCollection infer_args;
51  infer_args.inference_args.device = blace::util::get_accelerator().value();
52
53  // construct inference operation
54  auto infer_op = CONSTRUCT_OP_GET(
55      blace::ops::InferenceOp(raft_v1_default_v2_ALL_export_version_v17_IDENT,
56                              {frame_0_op, frame_1_op}, infer_args, 0));
57
58  // normalize optical flow to zero-one range for plotting. The model returns
59  // relative offsets in -1 to 1 pixel space, so the raw values are to small to
60  // plot
61  infer_op = CONSTRUCT_OP_GET(blace::ops::NormalizeToZeroOneOP(infer_op));
62
63  // convert uv color to rgb (by U->R, V->G, 1->B)
64  infer_op =
65      CONSTRUCT_OP_GET(blace::ops::ToColorOp(infer_op, blace::ml_core::RGB));
66
67  // we prepare the result for later copy to cv::Mat. The values set here are
68  // based on implicit knowledge of cv::Mat internal data storage.
69  auto normalized_matte = CONSTRUCT_OP_GET(blace::ops::PrepareForHostCopyOP(
70      infer_op, blace::ml_core::BLACE_BYTE, blace::ml_core::RGB,
71      blace::ml_core::HWC, blace::ml_core::ZERO_TO_255));
72
73  // construct evaluator and evaluate to raw memory object
74  blace::computation_graph::GraphEvaluator evaluator(normalized_matte);
75  auto raw_mem = evaluator.evaluateToRawMemory().value();
76
77  // get the sizes
78  int w = raw_mem.get_memory_sizes()[1];
79  int h = raw_mem.get_memory_sizes()[0];
80
81  // initialize an empty cv::Mat
82  cv::Mat cv_mat(h, w, CV_8UC3);
83  cv_mat.setTo(cv::Scalar(0, 0, 0));
84
85  // and copy the memory
86  std::memcpy(cv_mat.data, raw_mem.get_data_ptr(), raw_mem.get_memory_size());
87
88  // save to disk and return
89  auto out_file = exe_path / "optical_flow.png";
90  cv::imwrite(out_file.string(), cv_mat);
91
92  // unload all models before program exits
93  blace::util::unloadModels();
94
95  return 0;
96}

Follow the 5 minute instructions to build and run the demo.

Tested on version v0.9.28 of blace.ai sdk.

Artifacts

Payload Demo Project Header

License