Skip to content

Commit b309589

Browse files
matthiasrichterktf
matthiasrichter
authored and
ktf
committed
Merge pull request #8733 from matthiasrichter/dev-datadescriptormatcher-serializer
DPL: full serialization of DataDescriptorMatcher in worklfow json dump Fully qualified specs which are expressed as `ConcreteDataMatcher` are dumped in the same format as before, for all other cases, the `DataDescriptorMatcher` hierarchy is dumped in detail.
1 parent 4339414 commit b309589

File tree

2 files changed

+183
-19
lines changed

2 files changed

+183
-19
lines changed

‎Framework/Core/include/Framework/DataDescriptorMatcher.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,8 @@ class DataDescriptorMatcher
297297

298298
Node const& getLeft() const { return mLeft; };
299299
Node const& getRight() const { return mRight; };
300+
Node& getLeft() { return mLeft; };
301+
Node& getRight() { return mRight; };
300302
Op getOp() const { return mOp; };
301303

302304
private:

‎Framework/Core/src/WorkflowSerializationHelpers.cxx‎

Lines changed: 181 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@
1313
#include "Framework/WorkflowSpec.h"
1414
#include "Framework/DataDescriptorQueryBuilder.h"
1515
#include "Framework/DataSpecUtils.h"
16+
#include "Framework/VariantHelpers.h"
1617
#include "Framework/VariantJSONHelpers.h"
1718
#include "Framework/DataDescriptorMatcher.h"
19+
#include "Framework/DataMatcherWalker.h"
1820
#include "Framework/Logger.h"
1921

2022
#include <rapidjson/reader.h>
@@ -56,6 +58,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
5658
IN_INPUT_DESCRIPTION,
5759
IN_INPUT_SUBSPEC,
5860
IN_INPUT_LIFETIME,
61+
IN_INPUT_STARTTIME,
5962
IN_INPUT_MATCHER,
6063
IN_INPUT_MATCHER_OPERATION,
6164
IN_INPUT_LEFT_MATCHER,
@@ -166,6 +169,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
166169
case State::IN_INPUT_LIFETIME:
167170
s << "IN_INPUT_LIFETIME";
168171
break;
172+
case State::IN_INPUT_STARTTIME:
173+
s << "IN_INPUT_STARTTIME";
174+
break;
169175
case State::IN_INPUT_OPTIONS:
170176
s << "IN_INPUT_OPTIONS";
171177
break;
@@ -267,11 +273,13 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
267273
push(State::IN_INPUT);
268274
inputMatcherNodes.clear();
269275
} else if (in(State::IN_INPUT_MATCHER)) {
270-
assert(0); // to be implemented
276+
// start a new embedded matcher
271277
} else if (in(State::IN_INPUT_LEFT_MATCHER)) {
272-
assert(0); // to be implemented
278+
// this is a matcher leaf, i.e. last matcher of a branch
279+
// will be merged into the parent matcher
273280
} else if (in(State::IN_INPUT_RIGHT_MATCHER)) {
274-
assert(0); // to be implemented
281+
// this is a matcher leaf, i.e. last matcher of a branch
282+
// will be merged into the parent matcher
275283
} else if (in(State::IN_OUTPUTS)) {
276284
push(State::IN_OUTPUT);
277285
outputHasSubSpec = false;
@@ -313,7 +321,13 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
313321
return lastMatcher;
314322
};
315323

316-
auto matcher = buildMatcher(inputMatcherNodes);
324+
std::unique_ptr<DataDescriptorMatcher> matcher;
325+
if (auto* pval = std::get_if<std::unique_ptr<DataDescriptorMatcher>>(&inputMatcherNodes[0])) {
326+
assert(inputMatcherNodes.size() == 1);
327+
matcher = std::move(*pval);
328+
} else {
329+
matcher = buildMatcher(inputMatcherNodes);
330+
}
317331
auto concrete = DataSpecUtils::optionalConcreteDataMatcherFrom(*matcher);
318332
if (concrete.has_value()) {
319333
// the matcher is fully qualified with unique parameters so we add ConcreteDataMatcher
@@ -324,6 +338,60 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
324338
inputMatcherNodes.clear();
325339
inputOptions.clear();
326340

341+
} else if (in(State::IN_INPUT_MATCHER) && inputMatcherNodes.size() > 1) {
342+
data_matcher::Node child = std::move(inputMatcherNodes.back());
343+
inputMatcherNodes.pop_back();
344+
auto* matcher = std::get_if<std::unique_ptr<DataDescriptorMatcher>>(&child);
345+
assert(matcher != nullptr);
346+
auto* parent = std::get_if<std::unique_ptr<DataDescriptorMatcher>>(&inputMatcherNodes.back());
347+
assert(parent != nullptr);
348+
std::unique_ptr<DataDescriptorMatcher> node;
349+
auto mergeDown = [&node, &parent, &child]() -> bool {
350+
// FIXME: do we need a dedicated default state, or can we simply use ConstantValueMatcher
351+
if (auto* pval1 = std::get_if<ConstantValueMatcher>(&((*parent)->getLeft()))) {
352+
if (*pval1 == ConstantValueMatcher{false}) {
353+
node = std::make_unique<DataDescriptorMatcher>((*parent)->getOp(),
354+
std::move(child),
355+
std::move((*parent)->getRight()));
356+
return true;
357+
}
358+
}
359+
if (auto* pval2 = std::get_if<ConstantValueMatcher>(&((*parent)->getRight()))) {
360+
if (*pval2 == ConstantValueMatcher{false}) {
361+
node = std::make_unique<DataDescriptorMatcher>((*parent)->getOp(),
362+
std::move((*parent)->getLeft()),
363+
std::move(child));
364+
return true;
365+
}
366+
}
367+
return false;
368+
};
369+
if (!mergeDown()) {
370+
states.push_back(State::IN_ERROR);
371+
}
372+
inputMatcherNodes.pop_back();
373+
inputMatcherNodes.push_back(std::move(node));
374+
} else if (in(State::IN_INPUT_LEFT_MATCHER)) {
375+
assert(inputMatcherNodes.size() >= 2);
376+
size_t nMatchers = inputMatcherNodes.size();
377+
auto* parent = std::get_if<std::unique_ptr<DataDescriptorMatcher>>(&inputMatcherNodes[nMatchers - 2]);
378+
assert(parent != nullptr);
379+
auto node = std::make_unique<DataDescriptorMatcher>((*parent)->getOp(),
380+
std::move(inputMatcherNodes[nMatchers - 1]),
381+
std::move((*parent)->getRight()));
382+
inputMatcherNodes.pop_back();
383+
inputMatcherNodes.pop_back();
384+
inputMatcherNodes.push_back(std::move(node));
385+
} else if (in(State::IN_INPUT_RIGHT_MATCHER)) {
386+
data_matcher::Node child = std::move(inputMatcherNodes.back());
387+
inputMatcherNodes.pop_back();
388+
auto* parent = std::get_if<std::unique_ptr<DataDescriptorMatcher>>(&inputMatcherNodes.back());
389+
assert(parent != nullptr);
390+
auto node = std::make_unique<DataDescriptorMatcher>((*parent)->getOp(),
391+
std::move((*parent)->getLeft()),
392+
std::move(child));
393+
inputMatcherNodes.pop_back();
394+
inputMatcherNodes.push_back(std::move(node));
327395
} else if (in(State::IN_OUTPUT)) {
328396
if (outputHasSubSpec) {
329397
dataProcessors.back().outputs.push_back(OutputSpec({binding}, origin, description, subspec, lifetime));
@@ -479,13 +547,13 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
479547
} else if (in(State::IN_INPUT) && strncmp(str, "subspec", length) == 0) {
480548
push(State::IN_INPUT_SUBSPEC);
481549
} else if (in(State::IN_INPUT) && strncmp(str, "matcher", length) == 0) {
482-
assert(0);
483550
// the outermost matcher is starting here
484551
// we create a placeholder which is being updated later
485552
inputMatcherNodes.push_back(std::make_unique<DataDescriptorMatcher>(DataDescriptorMatcher::Op::And, ConstantValueMatcher{false}));
486553
push(State::IN_INPUT_MATCHER);
487554
} else if (in(State::IN_INPUT_MATCHER) && strncmp(str, "matcher", length) == 0) {
488-
// recursive matchers, can maybe combine with above
555+
// recursive matchers
556+
inputMatcherNodes.push_back(std::make_unique<DataDescriptorMatcher>(DataDescriptorMatcher::Op::And, ConstantValueMatcher{false}));
489557
push(State::IN_INPUT_MATCHER);
490558
} else if (in(State::IN_INPUT_MATCHER) && strncmp(str, "operation", length) == 0) {
491559
push(State::IN_INPUT_MATCHER_OPERATION);
@@ -505,8 +573,14 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
505573
push(State::IN_INPUT_SUBSPEC);
506574
} else if (in(State::IN_INPUT_RIGHT_MATCHER) && strncmp(str, "subspec", length) == 0) {
507575
push(State::IN_INPUT_SUBSPEC);
576+
} else if (in(State::IN_INPUT_LEFT_MATCHER) && strncmp(str, "starttime", length) == 0) {
577+
push(State::IN_INPUT_STARTTIME);
578+
} else if (in(State::IN_INPUT_RIGHT_MATCHER) && strncmp(str, "starttime", length) == 0) {
579+
push(State::IN_INPUT_STARTTIME);
508580
} else if (in(State::IN_INPUT) && strncmp(str, "lifetime", length) == 0) {
509581
push(State::IN_INPUT_LIFETIME);
582+
} else if (in(State::IN_INPUT) && strncmp(str, "starttime", length) == 0) {
583+
push(State::IN_INPUT_STARTTIME);
510584
} else if (in(State::IN_INPUT) && strncmp(str, "metadata", length) == 0) {
511585
push(State::IN_INPUT_OPTIONS);
512586
} else if (in(State::IN_OUTPUT) && strncmp(str, "binding", length) == 0) {
@@ -594,8 +668,30 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
594668
description.runtimeInit(s.c_str(), std::min(s.size(), 16UL));
595669
std::string v(s.c_str(), std::min(s.size(), 16UL));
596670
inputMatcherNodes.push_back(DescriptionValueMatcher{v});
671+
} else if (in(State::IN_INPUT_STARTTIME)) {
672+
// we add StartTimeValueMatcher with ContextRef for starttime, no matter what
673+
// has been in the configuration.
674+
inputMatcherNodes.push_back(StartTimeValueMatcher(ContextRef{ContextPos::STARTTIME_POS}));
597675
} else if (in(State::IN_INPUT_MATCHER_OPERATION)) {
598-
// FIXME: read operation
676+
// FIXME: need to implement operator>> to read the op parameter
677+
DataDescriptorMatcher::Op op = DataDescriptorMatcher::Op::And;
678+
if (s == "and") {
679+
op = DataDescriptorMatcher::Op::And;
680+
} else if (s == "or") {
681+
op = DataDescriptorMatcher::Op::Or;
682+
} else if (s == "xor") {
683+
op = DataDescriptorMatcher::Op::Xor;
684+
} else if (s == "just") {
685+
op = DataDescriptorMatcher::Op::Just;
686+
} else if (s == "not") {
687+
op = DataDescriptorMatcher::Op::Not;
688+
}
689+
// FIXME: we could drop the placeholder which has been added when entering
690+
// the states which can read key 'operation', but then we need to make sure
691+
// that this key is always present
692+
auto node = std::make_unique<DataDescriptorMatcher>(op, ConstantValueMatcher{false});
693+
inputMatcherNodes.pop_back();
694+
inputMatcherNodes.push_back(std::move(node));
599695
} else if (in(State::IN_OUTPUT_BINDING)) {
600696
binding = s;
601697
} else if (in(State::IN_OUTPUT_ORIGIN)) {
@@ -629,6 +725,10 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
629725
push(State::IN_METADATUM_CHANNEL);
630726
} else if (in(State::IN_COMMAND)) {
631727
command.merge({s});
728+
} else {
729+
std::stringstream errstr;
730+
errstr << "No string handling for argument '" << std::string(str, length) << "' in state " << states.back() << std::endl;
731+
throw std::runtime_error(errstr.str());
632732
}
633733
pop();
634734
return true;
@@ -788,6 +888,67 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
788888
rapidjson::OStreamWrapper osw(out);
789889
rapidjson::PrettyWriter<rapidjson::OStreamWrapper> w(osw);
790890

891+
// handlers for serialization of InputSpec matchers
892+
auto edgeWalker = overloaded{
893+
[&w](EdgeActions::EnterNode action) {
894+
w.Key("matcher");
895+
w.StartObject();
896+
w.Key("operation");
897+
std::stringstream ss;
898+
ss << action.node->getOp();
899+
w.String(ss.str().c_str());
900+
if (action.node->getOp() == DataDescriptorMatcher::Op::Just ||
901+
action.node->getOp() == DataDescriptorMatcher::Op::Not) {
902+
return ChildAction::VisitLeft;
903+
}
904+
return ChildAction::VisitBoth;
905+
},
906+
[&w](EdgeActions::EnterLeft) {
907+
w.Key("left");
908+
w.StartObject();
909+
},
910+
[&w](EdgeActions::ExitLeft) {
911+
w.EndObject();
912+
},
913+
[&w](EdgeActions::EnterRight) {
914+
w.Key("right");
915+
w.StartObject();
916+
},
917+
[&w](EdgeActions::ExitRight) {
918+
w.EndObject();
919+
},
920+
[&w](EdgeActions::ExitNode) {
921+
w.EndObject();
922+
},
923+
[&w](auto) {}};
924+
auto leafWalker = overloaded{
925+
[&w](OriginValueMatcher const& origin) {
926+
w.Key("origin");
927+
std::stringstream ss;
928+
ss << origin;
929+
w.String(ss.str().c_str());
930+
},
931+
[&w](DescriptionValueMatcher const& description) {
932+
w.Key("description");
933+
std::stringstream ss;
934+
ss << description;
935+
w.String(ss.str().c_str());
936+
},
937+
[&w](SubSpecificationTypeValueMatcher const& subspec) {
938+
w.Key("subspec");
939+
std::stringstream ss;
940+
ss << subspec;
941+
w.Uint64(std::stoul(ss.str()));
942+
},
943+
[&w](StartTimeValueMatcher const& startTime) {
944+
w.Key("starttime");
945+
std::stringstream ss;
946+
ss << startTime;
947+
w.String(ss.str().c_str());
948+
},
949+
[&w](ConstantValueMatcher const& constant) {},
950+
[&w](auto t) {}};
951+
791952
w.StartObject();
792953
w.Key("workflow");
793954
w.StartArray();
@@ -802,27 +963,28 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
802963

803964
w.Key("inputs");
804965
w.StartArray();
805-
for (auto& input : processor.inputs) {
966+
for (auto const& input : processor.inputs) {
806967
/// FIXME: this only works for a selected set of InputSpecs...
807968
/// a proper way to fully serialize an InputSpec with
808969
/// a DataDescriptorMatcher is needed.
809970
w.StartObject();
810971
w.Key("binding");
811972
w.String(input.binding.c_str());
812-
auto origin = DataSpecUtils::getOptionalOrigin(input);
813-
if (origin.has_value()) {
973+
if (auto const* concrete = std::get_if<ConcreteDataMatcher>(&input.matcher)) {
814974
w.Key("origin");
815-
w.String(origin->str, strnlen(origin->str, 4));
816-
}
817-
auto description = DataSpecUtils::getOptionalDescription(input);
818-
if (description.has_value()) {
975+
w.String(concrete->origin.str, strnlen(concrete->origin.str, 4));
819976
w.Key("description");
820-
w.String(description->str, strnlen(description->str, 16));
821-
}
822-
auto subSpec = DataSpecUtils::getOptionalSubSpec(input);
823-
if (subSpec.has_value()) {
977+
w.String(concrete->description.str, strnlen(concrete->description.str, 16));
824978
w.Key("subspec");
825-
w.Uint64(*subSpec);
979+
w.Uint64(concrete->subSpec);
980+
//auto tmp = DataSpecUtils::dataDescriptorMatcherFrom(*concrete);
981+
//DataMatcherWalker::walk(tmp,
982+
// edgeWalker,
983+
// leafWalker);
984+
} else if (auto const* matcher = std::get_if<DataDescriptorMatcher>(&input.matcher)) {
985+
DataMatcherWalker::walk(*matcher,
986+
edgeWalker,
987+
leafWalker);
826988
}
827989
w.Key("lifetime");
828990
w.Uint((int)input.lifetime);

0 commit comments

Comments
 (0)