Skip to content

Commit 9232328

Browse files
alibuildalibuild
committed
Please consider the following formatting changes
1 parent 74cf0e7 commit 9232328

File tree

2 files changed

+89
-75
lines changed

2 files changed

+89
-75
lines changed

‎Common/ML/include/ML/ort_interface.h‎

Lines changed: 37 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -35,60 +35,58 @@ namespace ml
3535
class OrtModel
3636
{
3737

38-
public:
39-
// Constructor
40-
OrtModel() = default;
41-
OrtModel(std::unordered_map<std::string, std::string> optionsMap){ reset(optionsMap); }
42-
void init(std::unordered_map<std::string, std::string> optionsMap){ reset(optionsMap); }
43-
void reset(std::unordered_map<std::string, std::string>);
38+
public:
39+
// Constructor
40+
OrtModel() = default;
41+
OrtModel(std::unordered_map<std::string, std::string> optionsMap) { reset(optionsMap); }
42+
void init(std::unordered_map<std::string, std::string> optionsMap) { reset(optionsMap); }
43+
void reset(std::unordered_map<std::string, std::string>);
4444

45-
virtual ~OrtModel() = default;
45+
virtual ~OrtModel() = default;
4646

47-
// Conversion
48-
template<class I, class O>
49-
std::vector<O> v2v(std::vector<I>&, bool = true);
47+
// Conversion
48+
template <class I, class O>
49+
std::vector<O> v2v(std::vector<I>&, bool = true);
5050

51-
// Inferencing
52-
template<class I, class O> // class I is the input data type, e.g. float, class O is the output data type, e.g. OrtDataType::Float16_t from O2/Common/ML/include/ML/GPUORTFloat16.h
53-
std::vector<O> inference(std::vector<I>&);
51+
// Inferencing
52+
template <class I, class O> // class I is the input data type, e.g. float, class O is the output data type, e.g. OrtDataType::Float16_t from O2/Common/ML/include/ML/GPUORTFloat16.h
53+
std::vector<O> inference(std::vector<I>&);
5454

55-
template<class I, class O> // class I is the input data type, e.g. float, class O is the output data type, e.g. O2::gpu::OrtDataType::Float16_t from O2/GPU/GPUTracking/ML/convert_float16.h
56-
std::vector<O> inference(std::vector<std::vector<I>>&);
55+
template <class I, class O> // class I is the input data type, e.g. float, class O is the output data type, e.g. O2::gpu::OrtDataType::Float16_t from O2/GPU/GPUTracking/ML/convert_float16.h
56+
std::vector<O> inference(std::vector<std::vector<I>>&);
5757

58-
// template<class I, class T, class O> // class I is the input data type, e.g. float, class T the throughput data type and class O is the output data type
59-
// std::vector<O> inference(std::vector<I>&);
58+
// template<class I, class T, class O> // class I is the input data type, e.g. float, class T the throughput data type and class O is the output data type
59+
// std::vector<O> inference(std::vector<I>&);
6060

61-
// Reset session
62-
void resetSession();
61+
// Reset session
62+
void resetSession();
6363

64-
std::vector<std::vector<int64_t>> getNumInputNodes() const { return mInputShapes; }
65-
std::vector<std::vector<int64_t>> getNumOutputNodes() const { return mOutputShapes; }
66-
std::vector<std::string> getInputNames() const { return mInputNames; }
67-
std::vector<std::string> getOutputNames() const { return mOutputNames; }
64+
std::vector<std::vector<int64_t>> getNumInputNodes() const { return mInputShapes; }
65+
std::vector<std::vector<int64_t>> getNumOutputNodes() const { return mOutputShapes; }
66+
std::vector<std::string> getInputNames() const { return mInputNames; }
67+
std::vector<std::string> getOutputNames() const { return mOutputNames; }
6868

69-
void setActiveThreads(int threads) { intraOpNumThreads = threads; }
69+
void setActiveThreads(int threads) { intraOpNumThreads = threads; }
7070

71-
private:
71+
private:
72+
// ORT variables -> need to be hidden as Pimpl
73+
struct OrtVariables;
74+
OrtVariables* pImplOrt;
7275

73-
// ORT variables -> need to be hidden as Pimpl
74-
struct OrtVariables;
75-
OrtVariables* pImplOrt;
76+
// Input & Output specifications of the loaded network
77+
std::vector<const char*> inputNamesChar, outputNamesChar;
78+
std::vector<std::string> mInputNames, mOutputNames;
79+
std::vector<std::vector<int64_t>> mInputShapes, mOutputShapes;
7680

77-
// Input & Output specifications of the loaded network
78-
std::vector<const char*> inputNamesChar, outputNamesChar;
79-
std::vector<std::string> mInputNames, mOutputNames;
80-
std::vector<std::vector<int64_t>> mInputShapes, mOutputShapes;
81-
82-
// Environment settings
83-
std::string modelPath, device = "cpu", dtype = "float"; // device options should be cpu, rocm, migraphx, cuda
84-
int intraOpNumThreads = 0, deviceId = 0, enableProfiling = 0, loggingLevel = 0, allocateDeviceMemory = 0, enableOptimizations = 0;
85-
86-
std::string printShape(const std::vector<int64_t>&);
81+
// Environment settings
82+
std::string modelPath, device = "cpu", dtype = "float"; // device options should be cpu, rocm, migraphx, cuda
83+
int intraOpNumThreads = 0, deviceId = 0, enableProfiling = 0, loggingLevel = 0, allocateDeviceMemory = 0, enableOptimizations = 0;
8784

85+
std::string printShape(const std::vector<int64_t>&);
8886
};
8987

9088
} // namespace ml
9189

92-
} // namespace ml
90+
} // namespace o2
9391

9492
#endif // O2_ML_ORT_INTERFACE_H

‎Common/ML/src/ort_interface.cxx‎

Lines changed: 52 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ namespace o2
2525
namespace ml
2626
{
2727

28-
struct OrtModel::OrtVariables { // The actual implementation is hidden in the .cxx file
28+
struct OrtModel::OrtVariables { // The actual implementation is hidden in the .cxx file
2929
// ORT runtime objects
3030
Ort::RunOptions runOptions;
3131
std::shared_ptr<Ort::Env> env = nullptr;
@@ -35,55 +35,56 @@ struct OrtModel::OrtVariables { // The actual implementation is hidden in the .
3535
Ort::MemoryInfo memoryInfo = Ort::MemoryInfo("Cpu", OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemType::OrtMemTypeDefault);
3636
};
3737

38-
void OrtModel::reset(std::unordered_map<std::string, std::string> optionsMap){
38+
void OrtModel::reset(std::unordered_map<std::string, std::string> optionsMap)
39+
{
3940

4041
pImplOrt = new OrtVariables();
4142

4243
// Load from options map
43-
if(!optionsMap.contains("model-path")){
44+
if (!optionsMap.contains("model-path")) {
4445
LOG(fatal) << "(ORT) Model path cannot be empty!";
4546
}
4647
modelPath = optionsMap["model-path"];
4748
device = (optionsMap.contains("device") ? optionsMap["device"] : "CPU");
4849
dtype = (optionsMap.contains("dtype") ? optionsMap["dtype"] : "float");
4950
deviceId = (optionsMap.contains("device-id") ? std::stoi(optionsMap["device-id"]) : 0);
5051
allocateDeviceMemory = (optionsMap.contains("allocate-device-memory") ? std::stoi(optionsMap["allocate-device-memory"]) : 0);
51-
intraOpNumThreads = (optionsMap.contains("intra-op-num-threads") ? std::stoi(optionsMap["intra-op-num-threads"]) : 0);
52+
intraOpNumThreads = (optionsMap.contains("intra-op-num-threads") ? std::stoi(optionsMap["intra-op-num-threads"]) : 0);
5253
loggingLevel = (optionsMap.contains("logging-level") ? std::stoi(optionsMap["logging-level"]) : 0);
5354
enableProfiling = (optionsMap.contains("enable-profiling") ? std::stoi(optionsMap["enable-profiling"]) : 0);
5455
enableOptimizations = (optionsMap.contains("enable-optimizations") ? std::stoi(optionsMap["enable-optimizations"]) : 0);
5556

5657
std::string dev_mem_str = "Hip";
5758
#ifdef ORT_ROCM_BUILD
58-
if(device == "ROCM") {
59+
if (device == "ROCM") {
5960
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_ROCM(pImplOrt->sessionOptions, deviceId));
6061
LOG(info) << "(ORT) ROCM execution provider set";
6162
}
6263
#endif
6364
#ifdef ORT_MIGRAPHX_BUILD
64-
if(device == "MIGRAPHX") {
65+
if (device == "MIGRAPHX") {
6566
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_MIGraphX(pImplOrt->sessionOptions, deviceId));
6667
LOG(info) << "(ORT) MIGraphX execution provider set";
6768
}
6869
#endif
6970
#ifdef ORT_CUDA_BUILD
70-
if(device == "CUDA") {
71+
if (device == "CUDA") {
7172
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_CUDA(pImplOrt->sessionOptions, deviceId));
7273
LOG(info) << "(ORT) CUDA execution provider set";
7374
dev_mem_str = "Cuda";
7475
}
7576
#endif
7677

77-
if(allocateDeviceMemory){
78+
if (allocateDeviceMemory) {
7879
pImplOrt->memoryInfo = Ort::MemoryInfo(dev_mem_str.c_str(), OrtAllocatorType::OrtDeviceAllocator, deviceId, OrtMemType::OrtMemTypeDefault);
7980
LOG(info) << "(ORT) Memory info set to on-device memory";
8081
}
8182

82-
if(device == "CPU") {
83+
if (device == "CPU") {
8384
(pImplOrt->sessionOptions).SetIntraOpNumThreads(intraOpNumThreads);
84-
if(intraOpNumThreads > 1){
85+
if (intraOpNumThreads > 1) {
8586
(pImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_PARALLEL);
86-
} else if(intraOpNumThreads == 1){
87+
} else if (intraOpNumThreads == 1) {
8788
(pImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL);
8889
}
8990
LOG(info) << "(ORT) CPU execution provider set with " << intraOpNumThreads << " threads";
@@ -92,8 +93,8 @@ void OrtModel::reset(std::unordered_map<std::string, std::string> optionsMap){
9293
(pImplOrt->sessionOptions).DisableMemPattern();
9394
(pImplOrt->sessionOptions).DisableCpuMemArena();
9495

95-
if(enableProfiling){
96-
if(optionsMap.contains("profiling-output-path")){
96+
if (enableProfiling) {
97+
if (optionsMap.contains("profiling-output-path")) {
9798
(pImplOrt->sessionOptions).EnableProfiling((optionsMap["profiling-output-path"] + "/ORT_LOG_").c_str());
9899
} else {
99100
LOG(warning) << "(ORT) If profiling is enabled, optionsMap[\"profiling-output-path\"] should be set. Disabling profiling for now.";
@@ -109,27 +110,27 @@ void OrtModel::reset(std::unordered_map<std::string, std::string> optionsMap){
109110
(pImplOrt->session).reset(new Ort::Session{*(pImplOrt->env), modelPath.c_str(), pImplOrt->sessionOptions});
110111

111112
for (size_t i = 0; i < (pImplOrt->session)->GetInputCount(); ++i) {
112-
mInputNames.push_back((pImplOrt->session)->GetInputNameAllocated(i, pImplOrt->allocator).get());
113+
mInputNames.push_back((pImplOrt->session)->GetInputNameAllocated(i, pImplOrt->allocator).get());
113114
}
114115
for (size_t i = 0; i < (pImplOrt->session)->GetInputCount(); ++i) {
115-
mInputShapes.emplace_back((pImplOrt->session)->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape());
116+
mInputShapes.emplace_back((pImplOrt->session)->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape());
116117
}
117118
for (size_t i = 0; i < (pImplOrt->session)->GetOutputCount(); ++i) {
118-
mOutputNames.push_back((pImplOrt->session)->GetOutputNameAllocated(i, pImplOrt->allocator).get());
119+
mOutputNames.push_back((pImplOrt->session)->GetOutputNameAllocated(i, pImplOrt->allocator).get());
119120
}
120121
for (size_t i = 0; i < (pImplOrt->session)->GetOutputCount(); ++i) {
121-
mOutputShapes.emplace_back((pImplOrt->session)->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape());
122+
mOutputShapes.emplace_back((pImplOrt->session)->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape());
122123
}
123124

124125
inputNamesChar.resize(mInputNames.size(), nullptr);
125126
std::transform(std::begin(mInputNames), std::end(mInputNames), std::begin(inputNamesChar),
126-
[&](const std::string& str) { return str.c_str(); });
127+
[&](const std::string& str) { return str.c_str(); });
127128
outputNamesChar.resize(mOutputNames.size(), nullptr);
128129
std::transform(std::begin(mOutputNames), std::end(mOutputNames), std::begin(outputNamesChar),
129-
[&](const std::string& str) { return str.c_str(); });
130+
[&](const std::string& str) { return str.c_str(); });
130131

131132
// Print names
132-
if(loggingLevel > 1) {
133+
if (loggingLevel > 1) {
133134
LOG(info) << "Input Nodes:";
134135
for (size_t i = 0; i < mInputNames.size(); i++) {
135136
LOG(info) << "\t" << mInputNames[i] << " : " << printShape(mInputShapes[i]);
@@ -142,24 +143,28 @@ void OrtModel::reset(std::unordered_map<std::string, std::string> optionsMap){
142143
}
143144
}
144145

145-
void OrtModel::resetSession() {
146+
void OrtModel::resetSession()
147+
{
146148
(pImplOrt->session).reset(new Ort::Session{*(pImplOrt->env), modelPath.c_str(), pImplOrt->sessionOptions});
147149
}
148150

149-
template<class I, class O>
150-
std::vector<O> OrtModel::v2v(std::vector<I>& input, bool clearInput) {
151-
if constexpr (std::is_same_v<I,O>){
151+
template <class I, class O>
152+
std::vector<O> OrtModel::v2v(std::vector<I>& input, bool clearInput)
153+
{
154+
if constexpr (std::is_same_v<I, O>) {
152155
return input;
153156
} else {
154157
std::vector<O> output(input.size());
155158
std::transform(std::begin(input), std::end(input), std::begin(output), [](I f) { return O(f); });
156-
if(clearInput) input.clear();
159+
if (clearInput)
160+
input.clear();
157161
return output;
158162
}
159163
}
160164

161-
template<class I, class O> // class I is the input data type, e.g. float, class O is the output data type, e.g. O2::gpu::OrtDataType::Float16_t from O2/GPU/GPUTracking/ML/convert_float16.h
162-
std::vector<O> OrtModel::inference(std::vector<I>& input){
165+
template <class I, class O> // class I is the input data type, e.g. float, class O is the output data type, e.g. O2::gpu::OrtDataType::Float16_t from O2/GPU/GPUTracking/ML/convert_float16.h
166+
std::vector<O> OrtModel::inference(std::vector<I>& input)
167+
{
163168
std::vector<int64_t> inputShape{(int64_t)(input.size() / mInputShapes[0][1]), (int64_t)mInputShapes[0][1]};
164169
std::vector<Ort::Value> inputTensor;
165170
inputTensor.emplace_back(Ort::Value::CreateTensor<O>(pImplOrt->memoryInfo, (reinterpret_cast<O*>(input)).data(), input.size(), inputShape.data(), inputShape.size()));
@@ -171,10 +176,11 @@ std::vector<O> OrtModel::inference(std::vector<I>& input){
171176
return outputValuesVec;
172177
}
173178

174-
template<class I, class O> // class I is the input data type, e.g. float, class O is the output data type, e.g. O2::gpu::OrtDataType::Float16_t from O2/GPU/GPUTracking/ML/convert_float16.h
175-
std::vector<O> OrtModel::inference(std::vector<std::vector<I>>& input){
179+
template <class I, class O> // class I is the input data type, e.g. float, class O is the output data type, e.g. O2::gpu::OrtDataType::Float16_t from O2/GPU/GPUTracking/ML/convert_float16.h
180+
std::vector<O> OrtModel::inference(std::vector<std::vector<I>>& input)
181+
{
176182
std::vector<Ort::Value> inputTensor;
177-
for(auto i : input){
183+
for (auto i : input) {
178184
std::vector<int64_t> inputShape{(int64_t)(i.size() / mInputShapes[0][1]), (int64_t)mInputShapes[0][1]};
179185
inputTensor.emplace_back(Ort::Value::CreateTensor<O>(pImplOrt->memoryInfo, (reinterpret_cast<O*>(i)).data(), i.size(), inputShape.data(), inputShape.size()));
180186
}
@@ -195,7 +201,9 @@ std::string OrtModel::printShape(const std::vector<int64_t>& v)
195201
return ss.str();
196202
}
197203

198-
template <> std::vector<float> OrtModel::inference<float, float>(std::vector<float>& input) {
204+
template <>
205+
std::vector<float> OrtModel::inference<float, float>(std::vector<float>& input)
206+
{
199207
std::vector<int64_t> inputShape{(int64_t)(input.size() / mInputShapes[0][1]), (int64_t)mInputShapes[0][1]};
200208
std::vector<Ort::Value> inputTensor;
201209
inputTensor.emplace_back(Ort::Value::CreateTensor<float>(pImplOrt->memoryInfo, input.data(), input.size(), inputShape.data(), inputShape.size()));
@@ -207,7 +215,9 @@ template <> std::vector<float> OrtModel::inference<float, float>(std::vector<flo
207215
return outputValuesVec;
208216
}
209217

210-
template <> std::vector<float> OrtModel::inference<OrtDataType::Float16_t, float>(std::vector<OrtDataType::Float16_t>& input) {
218+
template <>
219+
std::vector<float> OrtModel::inference<OrtDataType::Float16_t, float>(std::vector<OrtDataType::Float16_t>& input)
220+
{
211221
std::vector<int64_t> inputShape{(int64_t)(input.size() / mInputShapes[0][1]), (int64_t)mInputShapes[0][1]};
212222
std::vector<Ort::Value> inputTensor;
213223
inputTensor.emplace_back(Ort::Value::CreateTensor<Ort::Float16_t>(pImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(input.data()), input.size(), inputShape.data(), inputShape.size()));
@@ -219,7 +229,9 @@ template <> std::vector<float> OrtModel::inference<OrtDataType::Float16_t, float
219229
return outputValuesVec;
220230
}
221231

222-
template <> std::vector<OrtDataType::Float16_t> OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(std::vector<OrtDataType::Float16_t>& input) {
232+
template <>
233+
std::vector<OrtDataType::Float16_t> OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(std::vector<OrtDataType::Float16_t>& input)
234+
{
223235
std::vector<int64_t> inputShape{(int64_t)(input.size() / mInputShapes[0][1]), (int64_t)mInputShapes[0][1]};
224236
std::vector<Ort::Value> inputTensor;
225237
inputTensor.emplace_back(Ort::Value::CreateTensor<Ort::Float16_t>(pImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(input.data()), input.size(), inputShape.data(), inputShape.size()));
@@ -231,7 +243,9 @@ template <> std::vector<OrtDataType::Float16_t> OrtModel::inference<OrtDataType:
231243
return outputValuesVec;
232244
}
233245

234-
template <> std::vector<OrtDataType::Float16_t> OrtModel::inference<float, OrtDataType::Float16_t>(std::vector<float>& input) {
246+
template <>
247+
std::vector<OrtDataType::Float16_t> OrtModel::inference<float, OrtDataType::Float16_t>(std::vector<float>& input)
248+
{
235249
std::vector<int64_t> inputShape{(int64_t)(input.size() / mInputShapes[0][1]), (int64_t)mInputShapes[0][1]};
236250
std::vector<Ort::Value> inputTensor;
237251
inputTensor.emplace_back(Ort::Value::CreateTensor<Ort::Float16_t>(pImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(input.data()), input.size(), inputShape.data(), inputShape.size()));
@@ -243,9 +257,11 @@ template <> std::vector<OrtDataType::Float16_t> OrtModel::inference<float, OrtDa
243257
return outputValuesVec;
244258
}
245259

246-
template <> std::vector<OrtDataType::Float16_t> OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(std::vector<std::vector<OrtDataType::Float16_t>>& input) {
260+
template <>
261+
std::vector<OrtDataType::Float16_t> OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(std::vector<std::vector<OrtDataType::Float16_t>>& input)
262+
{
247263
std::vector<Ort::Value> inputTensor;
248-
for(auto i : input){
264+
for (auto i : input) {
249265
std::vector<int64_t> inputShape{(int64_t)(i.size() / mInputShapes[0][1]), (int64_t)mInputShapes[0][1]};
250266
inputTensor.emplace_back(Ort::Value::CreateTensor<Ort::Float16_t>(pImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(i.data()), i.size(), inputShape.data(), inputShape.size()));
251267
}

0 commit comments

Comments
 (0)