Skip to content

Commit 6d0ccb2

Browse files
ChSonnabendChSonnabend
committed
Changing to mPImplOrt
1 parent df00152 commit 6d0ccb2

File tree

2 files changed

+60
-60
lines changed

2 files changed

+60
-60
lines changed

‎Common/ML/include/ML/OrtInterface.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ class OrtModel
113113
private:
114114
// ORT variables -> need to be hidden as pImpl
115115
struct OrtVariables;
116-
OrtVariables* pImplOrt;
116+
OrtVariables* mPImplOrt;
117117

118118
// Input & Output specifications of the loaded network
119119
std::vector<const char*> mInputNamesChar, mOutputNamesChar;

‎Common/ML/src/OrtInterface.cxx‎

Lines changed: 59 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ struct OrtModel::OrtVariables { // The actual implementation is hidden in the .c
4141
// General purpose
4242
void OrtModel::initOptions(std::unordered_map<std::string, std::string> optionsMap)
4343
{
44-
pImplOrt = new OrtVariables();
44+
mPImplOrt = new OrtVariables();
4545

4646
// Load from options map
4747
if (!optionsMap.contains("model-path")) {
@@ -61,37 +61,37 @@ void OrtModel::initOptions(std::unordered_map<std::string, std::string> optionsM
6161
mEnvName = (optionsMap.contains("onnx-environment-name") ? optionsMap["onnx-environment-name"] : "onnx_model_inference");
6262

6363
if (mDeviceType == "CPU") {
64-
(pImplOrt->sessionOptions).SetIntraOpNumThreads(mIntraOpNumThreads);
65-
(pImplOrt->sessionOptions).SetInterOpNumThreads(mInterOpNumThreads);
64+
(mPImplOrt->sessionOptions).SetIntraOpNumThreads(mIntraOpNumThreads);
65+
(mPImplOrt->sessionOptions).SetInterOpNumThreads(mInterOpNumThreads);
6666
if (mIntraOpNumThreads > 1 || mInterOpNumThreads > 1) {
67-
(pImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_PARALLEL);
67+
(mPImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_PARALLEL);
6868
} else if (mIntraOpNumThreads == 1) {
69-
(pImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL);
69+
(mPImplOrt->sessionOptions).SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL);
7070
}
7171
if (mLoggingLevel < 2) {
7272
LOG(info) << "(ORT) CPU execution provider set with " << mIntraOpNumThreads << " (mIntraOpNumThreads) and " << mInterOpNumThreads << " (mInterOpNumThreads) threads";
7373
}
7474
}
7575

7676
// OrtROCMProviderOptions rocm_options{};
77-
// (pImplOrt->sessionOptions).AppendExecutionProvider_ROCM(rocm_options);
77+
// (mPImplOrt->sessionOptions).AppendExecutionProvider_ROCM(rocm_options);
7878

79-
(pImplOrt->sessionOptions).DisableMemPattern();
80-
(pImplOrt->sessionOptions).DisableCpuMemArena();
79+
(mPImplOrt->sessionOptions).DisableMemPattern();
80+
(mPImplOrt->sessionOptions).DisableCpuMemArena();
8181

8282
if (mEnableProfiling) {
8383
if (optionsMap.contains("profiling-output-path")) {
84-
(pImplOrt->sessionOptions).EnableProfiling((optionsMap["profiling-output-path"] + "/ORT_LOG_").c_str());
84+
(mPImplOrt->sessionOptions).EnableProfiling((optionsMap["profiling-output-path"] + "/ORT_LOG_").c_str());
8585
} else {
8686
LOG(warning) << "(ORT) If profiling is enabled, optionsMap[\"profiling-output-path\"] should be set. Disabling profiling for now.";
87-
(pImplOrt->sessionOptions).DisableProfiling();
87+
(mPImplOrt->sessionOptions).DisableProfiling();
8888
}
8989
} else {
90-
(pImplOrt->sessionOptions).DisableProfiling();
90+
(mPImplOrt->sessionOptions).DisableProfiling();
9191
}
9292

93-
(pImplOrt->sessionOptions).SetGraphOptimizationLevel(GraphOptimizationLevel(mEnableOptimizations));
94-
(pImplOrt->sessionOptions).SetLogSeverityLevel(OrtLoggingLevel(mLoggingLevel));
93+
(mPImplOrt->sessionOptions).SetGraphOptimizationLevel(GraphOptimizationLevel(mEnableOptimizations));
94+
(mPImplOrt->sessionOptions).SetLogSeverityLevel(OrtLoggingLevel(mLoggingLevel));
9595

9696
mInitialized = true;
9797
} else {
@@ -101,7 +101,7 @@ void OrtModel::initOptions(std::unordered_map<std::string, std::string> optionsM
101101

102102
void OrtModel::initEnvironment()
103103
{
104-
pImplOrt->env = std::make_shared<Ort::Env>(
104+
mPImplOrt->env = std::make_shared<Ort::Env>(
105105
OrtLoggingLevel(mLoggingLevel),
106106
(mEnvName.empty() ? "ORT" : mEnvName.c_str()),
107107
// Integrate ORT logging into Fairlogger
@@ -121,16 +121,16 @@ void OrtModel::initEnvironment()
121121
}
122122
},
123123
(void*)3);
124-
(pImplOrt->env)->DisableTelemetryEvents(); // Disable telemetry events
124+
(mPImplOrt->env)->DisableTelemetryEvents(); // Disable telemetry events
125125
}
126126

127127
void OrtModel::initSession()
128128
{
129129
if (mAllocateDeviceMemory) {
130130
memoryOnDevice(mDeviceId);
131131
}
132-
pImplOrt->session = std::make_shared<Ort::Session>(*pImplOrt->env, mModelPath.c_str(), pImplOrt->sessionOptions);
133-
pImplOrt->ioBinding = std::make_unique<Ort::IoBinding>(*pImplOrt->session);
132+
mPImplOrt->session = std::make_shared<Ort::Session>(*mPImplOrt->env, mModelPath.c_str(), mPImplOrt->sessionOptions);
133+
mPImplOrt->ioBinding = std::make_unique<Ort::IoBinding>(*mPImplOrt->session);
134134

135135
setIO();
136136

@@ -142,13 +142,13 @@ void OrtModel::initSession()
142142
void OrtModel::memoryOnDevice(int32_t deviceIndex)
143143
{
144144
if (deviceIndex >= 0) {
145-
(pImplOrt->runOptions).AddConfigEntry("disable_synchronize_execution_providers", "1");
146-
(pImplOrt->sessionOptions).AddConfigEntry("session.use_device_allocator_for_initializers", "1"); // See kOrtSessionOptionsUseDeviceAllocatorForInitializers, https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
147-
(pImplOrt->sessionOptions).AddConfigEntry("session.use_env_allocators", "1"); // This should enable to use the volatile memory allocation defined in O2/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx; not working yet: ONNX still assigns new memory at init time
148-
(pImplOrt->sessionOptions).AddConfigEntry("session_options.enable_cpu_mem_arena", "0"); // This should enable to use the volatile memory allocation defined in O2/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx; not working yet: ONNX still assigns new memory at init time
145+
(mPImplOrt->runOptions).AddConfigEntry("disable_synchronize_execution_providers", "1");
146+
(mPImplOrt->sessionOptions).AddConfigEntry("session.use_device_allocator_for_initializers", "1"); // See kOrtSessionOptionsUseDeviceAllocatorForInitializers, https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
147+
(mPImplOrt->sessionOptions).AddConfigEntry("session.use_env_allocators", "1"); // This should enable to use the volatile memory allocation defined in O2/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx; not working yet: ONNX still assigns new memory at init time
148+
(mPImplOrt->sessionOptions).AddConfigEntry("session_options.enable_cpu_mem_arena", "0"); // This should enable to use the volatile memory allocation defined in O2/GPU/GPUTracking/TPCClusterFinder/GPUTPCNNClusterizerHost.cxx; not working yet: ONNX still assigns new memory at init time
149149
// Arena memory shrinkage comes at performance cost
150150
/// For now prefer to use single allocation, enabled by O2/GPU/GPUTracking/Base/cuda/GPUReconstructionCUDA.cu -> SetONNXGPUStream -> rocm_options.arena_extend_strategy = 0;
151-
// (pImplOrt->runOptions).AddConfigEntry("memory.enable_memory_arena_shrinkage", ("gpu:" + std::to_string(deviceIndex)).c_str()); // See kOrtRunOptionsConfigEnableMemoryArenaShrinkage, https://github.com/microsoft/onnxruntime/blob/90c263f471bbce724e77d8e62831d3a9fa838b2f/include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h#L27
151+
// (mPImplOrt->runOptions).AddConfigEntry("memory.enable_memory_arena_shrinkage", ("gpu:" + std::to_string(deviceIndex)).c_str()); // See kOrtRunOptionsConfigEnableMemoryArenaShrinkage, https://github.com/microsoft/onnxruntime/blob/90c263f471bbce724e77d8e62831d3a9fa838b2f/include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h#L27
152152

153153
std::string dev_mem_str = "";
154154
if (mDeviceType == "ROCM") {
@@ -157,32 +157,32 @@ void OrtModel::memoryOnDevice(int32_t deviceIndex)
157157
if (mDeviceType == "CUDA") {
158158
dev_mem_str = "Cuda";
159159
}
160-
pImplOrt->memoryInfo = Ort::MemoryInfo(dev_mem_str.c_str(), OrtAllocatorType::OrtDeviceAllocator, deviceIndex, OrtMemType::OrtMemTypeDefault);
160+
mPImplOrt->memoryInfo = Ort::MemoryInfo(dev_mem_str.c_str(), OrtAllocatorType::OrtDeviceAllocator, deviceIndex, OrtMemType::OrtMemTypeDefault);
161161
if (mLoggingLevel < 2) {
162-
LOG(info) << "(ORT) Memory info set to on-device memory for device type " << mDeviceType << " with ID " << deviceIndex << " and pImplOrt pointer " << pImplOrt;
162+
LOG(info) << "(ORT) Memory info set to on-device memory for device type " << mDeviceType << " with ID " << deviceIndex << " and mPImplOrt pointer " << mPImplOrt;
163163
}
164164
}
165165
}
166166

167167
void OrtModel::resetSession()
168168
{
169-
pImplOrt->session = std::make_shared<Ort::Session>(*(pImplOrt->env), mModelPath.c_str(), pImplOrt->sessionOptions);
169+
mPImplOrt->session = std::make_shared<Ort::Session>(*(mPImplOrt->env), mModelPath.c_str(), mPImplOrt->sessionOptions);
170170
}
171171

172172
// Getters
173173
Ort::SessionOptions* OrtModel::getSessionOptions()
174174
{
175-
return &pImplOrt->sessionOptions;
175+
return &mPImplOrt->sessionOptions;
176176
}
177177

178178
Ort::MemoryInfo* OrtModel::getMemoryInfo()
179179
{
180-
return &pImplOrt->memoryInfo;
180+
return &mPImplOrt->memoryInfo;
181181
}
182182

183183
Ort::Env* OrtModel::getEnv()
184184
{
185-
return (pImplOrt->env).get();
185+
return (mPImplOrt->env).get();
186186
}
187187

188188
template <class I, class O>
@@ -202,17 +202,17 @@ std::vector<O> OrtModel::v2v(std::vector<I>& input, bool clearInput)
202202

203203
void OrtModel::setIO()
204204
{
205-
for (size_t i = 0; i < (pImplOrt->session)->GetInputCount(); ++i) {
206-
mInputNames.push_back((pImplOrt->session)->GetInputNameAllocated(i, pImplOrt->allocator).get());
205+
for (size_t i = 0; i < (mPImplOrt->session)->GetInputCount(); ++i) {
206+
mInputNames.push_back((mPImplOrt->session)->GetInputNameAllocated(i, mPImplOrt->allocator).get());
207207
}
208-
for (size_t i = 0; i < (pImplOrt->session)->GetInputCount(); ++i) {
209-
mInputShapes.emplace_back((pImplOrt->session)->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape());
208+
for (size_t i = 0; i < (mPImplOrt->session)->GetInputCount(); ++i) {
209+
mInputShapes.emplace_back((mPImplOrt->session)->GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape());
210210
}
211-
for (size_t i = 0; i < (pImplOrt->session)->GetOutputCount(); ++i) {
212-
mOutputNames.push_back((pImplOrt->session)->GetOutputNameAllocated(i, pImplOrt->allocator).get());
211+
for (size_t i = 0; i < (mPImplOrt->session)->GetOutputCount(); ++i) {
212+
mOutputNames.push_back((mPImplOrt->session)->GetOutputNameAllocated(i, mPImplOrt->allocator).get());
213213
}
214-
for (size_t i = 0; i < (pImplOrt->session)->GetOutputCount(); ++i) {
215-
mOutputShapes.emplace_back((pImplOrt->session)->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape());
214+
for (size_t i = 0; i < (mPImplOrt->session)->GetOutputCount(); ++i) {
215+
mOutputShapes.emplace_back((mPImplOrt->session)->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape());
216216
}
217217

218218
mInputNamesChar.resize(mInputNames.size(), nullptr);
@@ -252,7 +252,7 @@ void OrtModel::setIO()
252252

253253
void OrtModel::setEnv(Ort::Env* env)
254254
{
255-
pImplOrt->env = std::shared_ptr<Ort::Env>(env);
255+
mPImplOrt->env = std::shared_ptr<Ort::Env>(env);
256256
}
257257

258258
// Inference
@@ -266,12 +266,12 @@ std::vector<O> OrtModel::inference(std::vector<I>& input)
266266
}
267267
std::vector<Ort::Value> inputTensor;
268268
if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
269-
inputTensor.emplace_back(Ort::Value::CreateTensor<Ort::Float16_t>(pImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(input.data()), input.size(), inputShape.data(), inputShape.size()));
269+
inputTensor.emplace_back(Ort::Value::CreateTensor<Ort::Float16_t>(mPImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(input.data()), input.size(), inputShape.data(), inputShape.size()));
270270
} else {
271-
inputTensor.emplace_back(Ort::Value::CreateTensor<I>(pImplOrt->memoryInfo, input.data(), input.size(), inputShape.data(), inputShape.size()));
271+
inputTensor.emplace_back(Ort::Value::CreateTensor<I>(mPImplOrt->memoryInfo, input.data(), input.size(), inputShape.data(), inputShape.size()));
272272
}
273273
// input.clear();
274-
auto outputTensors = (pImplOrt->session)->Run(pImplOrt->runOptions, mInputNamesChar.data(), inputTensor.data(), inputTensor.size(), mOutputNamesChar.data(), mOutputNamesChar.size());
274+
auto outputTensors = (mPImplOrt->session)->Run(mPImplOrt->runOptions, mInputNamesChar.data(), inputTensor.data(), inputTensor.size(), mOutputNamesChar.data(), mOutputNamesChar.size());
275275
O* outputValues = outputTensors[0].template GetTensorMutableData<O>();
276276
std::vector<O> outputValuesVec{outputValues, outputValues + inputShape[0] * mOutputShapes[0][1]};
277277
outputTensors.clear();
@@ -292,22 +292,22 @@ void OrtModel::inference(I* input, int64_t input_size, O* output)
292292
std::vector<int64_t> inputShape{input_size, (int64_t)mInputShapes[0][1]};
293293
Ort::Value inputTensor = Ort::Value(nullptr);
294294
if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
295-
inputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(pImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(input), input_size * mInputShapes[0][1], inputShape.data(), inputShape.size());
295+
inputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(mPImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(input), input_size * mInputShapes[0][1], inputShape.data(), inputShape.size());
296296
} else {
297-
inputTensor = Ort::Value::CreateTensor<I>(pImplOrt->memoryInfo, input, input_size * mInputShapes[0][1], inputShape.data(), inputShape.size());
297+
inputTensor = Ort::Value::CreateTensor<I>(mPImplOrt->memoryInfo, input, input_size * mInputShapes[0][1], inputShape.data(), inputShape.size());
298298
}
299-
(pImplOrt->ioBinding)->BindInput(mInputNames[0].c_str(), inputTensor);
299+
(mPImplOrt->ioBinding)->BindInput(mInputNames[0].c_str(), inputTensor);
300300

301301
std::vector<int64_t> outputShape{input_size, mOutputShapes[0][1]};
302302
Ort::Value outputTensor = Ort::Value(nullptr);
303303
if constexpr (std::is_same_v<O, OrtDataType::Float16_t>) {
304-
outputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(pImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(output), input_size * mOutputShapes[0][1], outputShape.data(), outputShape.size());
304+
outputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(mPImplOrt->memoryInfo, reinterpret_cast<Ort::Float16_t*>(output), input_size * mOutputShapes[0][1], outputShape.data(), outputShape.size());
305305
} else {
306-
outputTensor = Ort::Value::CreateTensor<O>(pImplOrt->memoryInfo, output, input_size * mOutputShapes[0][1], outputShape.data(), outputShape.size());
306+
outputTensor = Ort::Value::CreateTensor<O>(mPImplOrt->memoryInfo, output, input_size * mOutputShapes[0][1], outputShape.data(), outputShape.size());
307307
}
308-
(pImplOrt->ioBinding)->BindOutput(mOutputNames[0].c_str(), outputTensor);
308+
(mPImplOrt->ioBinding)->BindOutput(mOutputNames[0].c_str(), outputTensor);
309309

310-
(pImplOrt->session)->Run(pImplOrt->runOptions, *pImplOrt->ioBinding);
310+
(mPImplOrt->session)->Run(mPImplOrt->runOptions, *mPImplOrt->ioBinding);
311311
}
312312

313313
template void OrtModel::inference<OrtDataType::Float16_t, OrtDataType::Float16_t>(OrtDataType::Float16_t*, int64_t, OrtDataType::Float16_t*);
@@ -327,14 +327,14 @@ void OrtModel::inference(I** input, int64_t input_size, O* output)
327327

328328
if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
329329
inputTensors[i] = Ort::Value::CreateTensor<Ort::Float16_t>(
330-
pImplOrt->memoryInfo,
330+
mPImplOrt->memoryInfo,
331331
reinterpret_cast<Ort::Float16_t*>(input[i]),
332332
mInputSizePerNode[i] * input_size,
333333
mInputShapesCopy[i].data(),
334334
mInputShapesCopy[i].size());
335335
} else {
336336
inputTensors[i] = Ort::Value::CreateTensor<I>(
337-
pImplOrt->memoryInfo,
337+
mPImplOrt->memoryInfo,
338338
input[i],
339339
mInputSizePerNode[i] * input_size,
340340
mInputShapesCopy[i].data(),
@@ -345,23 +345,23 @@ void OrtModel::inference(I** input, int64_t input_size, O* output)
345345
Ort::Value outputTensor = Ort::Value(nullptr);
346346
if constexpr (std::is_same_v<O, OrtDataType::Float16_t>) {
347347
outputTensor = Ort::Value::CreateTensor<Ort::Float16_t>(
348-
pImplOrt->memoryInfo,
348+
mPImplOrt->memoryInfo,
349349
reinterpret_cast<Ort::Float16_t*>(output),
350350
mOutputSizePerNode[0] * input_size, // assumes that there is only one output node
351351
mOutputShapesCopy[0].data(),
352352
mOutputShapesCopy[0].size());
353353
} else {
354354
outputTensor = Ort::Value::CreateTensor<O>(
355-
pImplOrt->memoryInfo,
355+
mPImplOrt->memoryInfo,
356356
output,
357357
mOutputSizePerNode[0] * input_size, // assumes that there is only one output node
358358
mOutputShapesCopy[0].data(),
359359
mOutputShapesCopy[0].size());
360360
}
361361

362362
// === Run inference ===
363-
pImplOrt->session->Run(
364-
pImplOrt->runOptions,
363+
mPImplOrt->session->Run(
364+
mPImplOrt->runOptions,
365365
mInputNamesChar.data(),
366366
inputTensors.data(),
367367
mInputNamesChar.size(),
@@ -387,15 +387,15 @@ std::vector<O> OrtModel::inference(std::vector<std::vector<I>>& inputs)
387387
if constexpr (std::is_same_v<I, OrtDataType::Float16_t>) {
388388
input_tensors.emplace_back(
389389
Ort::Value::CreateTensor<Ort::Float16_t>(
390-
pImplOrt->memoryInfo,
390+
mPImplOrt->memoryInfo,
391391
reinterpret_cast<Ort::Float16_t*>(inputs[i].data()),
392392
mInputSizePerNode[i] * mInputShapesCopy[i][0],
393393
mInputShapesCopy[i].data(),
394394
mInputShapesCopy[i].size()));
395395
} else {
396396
input_tensors.emplace_back(
397397
Ort::Value::CreateTensor<I>(
398-
pImplOrt->memoryInfo,
398+
mPImplOrt->memoryInfo,
399399
inputs[i].data(),
400400
mInputSizePerNode[i] * mInputShapesCopy[i][0],
401401
mInputShapesCopy[i].data(),
@@ -406,8 +406,8 @@ std::vector<O> OrtModel::inference(std::vector<std::vector<I>>& inputs)
406406
int32_t totalOutputSize = mOutputsTotal * mInputShapesCopy[0][0];
407407

408408
// === Run inference ===
409-
auto output_tensors = pImplOrt->session->Run(
410-
pImplOrt->runOptions,
409+
auto output_tensors = mPImplOrt->session->Run(
410+
mPImplOrt->runOptions,
411411
mInputNamesChar.data(),
412412
input_tensors.data(),
413413
input_tensors.size(),
@@ -428,9 +428,9 @@ template std::vector<OrtDataType::Float16_t> OrtModel::inference<OrtDataType::Fl
428428
void OrtModel::release(bool profilingEnabled)
429429
{
430430
// if (profilingEnabled) {
431-
// pImplOrt->session->EndProfiling();
431+
// mPImplOrt->session->EndProfiling();
432432
// }
433-
LOG(info) << "(ORT) Size of pImplOrt: " << sizeof(*pImplOrt) << " bytes";
433+
LOG(info) << "(ORT) Size of mPImplOrt: " << sizeof(*mPImplOrt) << " bytes";
434434
}
435435

436436
// private

0 commit comments

Comments
 (0)