Skip to content

Commit d6f8686

Browse files
mavroudomavroudo
authored
Merge pull request #12 from mavroudo/v2
V2
2 parents 8e9a73d + e5cc406 commit d6f8686

37 files changed

Lines changed: 1456 additions & 1063 deletions

‎src/main/java/com/datalab/siesta/queryprocessor/SaseConnection/SaseConnector.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ public List<Occurrences> evaluate(SIESTAPattern pattern, Map<Long, List<Event>>
5050
List<Occurrences> occurrences = new ArrayList<>();
5151
for (Map.Entry<Long, List<Event>> e : events.entrySet()) {
5252
ec.initializeEngine();
53+
if(e.getValue().isEmpty()){ // in case that events have been removed due to filters
54+
continue;
55+
}
5356
Stream s = this.getStream(new ArrayList<>(e.getValue()));
5457
ec.setInput(s);
5558
try {
@@ -106,6 +109,40 @@ public List<GroupOccurrences> evaluateGroups(SIESTAPattern pattern, Map<Integer,
106109
return occurrences;
107110
}
108111

112+
/**
113+
* Similar to the above method but it is used to evaluate the appearence of a pattern in the events when they
114+
* are grouped for different trace-groups
115+
* @param pattern the user defined pattern
116+
* @param events the required events for each trace in order to determine if pattern occurs
117+
* @return where the pattern occur
118+
*/
119+
public List<Occurrences> evaluateSmallPatterns(SIESTAPattern pattern, Map<Long, List<EventBoth>> events){
120+
EngineController ec = this.getEngineController(pattern,false);
121+
List<Occurrences> occurrences = new ArrayList<>();
122+
for (Map.Entry<Long, List<EventBoth>> e : events.entrySet()) {
123+
ec.initializeEngine();
124+
Stream s = this.getStream(new ArrayList<>(e.getValue()));
125+
ec.setInput(s);
126+
try {
127+
ec.runEngine();
128+
} catch (CloneNotSupportedException | EvaluationException exe) {
129+
throw new RuntimeException(exe);
130+
}
131+
if (!ec.getMatches().isEmpty()) {
132+
Occurrences ocs = new Occurrences();
133+
ocs.setTraceID(e.getKey());
134+
for (Match m : ec.getMatches()) {
135+
ocs.addOccurrence(new Occurrence(Arrays.stream(m.getEvents()).parallel()
136+
.map(x -> (SaseEvent) x)
137+
.map(SaseEvent::getEventBoth)
138+
.collect(Collectors.toList())));
139+
}
140+
occurrences.add(ocs);
141+
}
142+
}
143+
return occurrences;
144+
}
145+
109146
/**
110147
* Based on the pattern it creates a NFA that contains the states and the transitions of a state machine
111148
* that will be used to detect the occurrences of the pattern
@@ -122,6 +159,7 @@ private EngineController getEngineController(SIESTAPattern pattern, boolean only
122159
} else {
123160
nfaWrapper.setStates(pattern.getNfa());
124161
}
162+
nfaWrapper.setSize(nfaWrapper.getStates().length);
125163
ec.setNfa(new NFA(nfaWrapper));
126164
return ec;
127165
}

‎src/main/java/com/datalab/siesta/queryprocessor/Signatures/CassandraConnectionSignature.java‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -73,24 +73,26 @@ public Signature getSignature(String logname) {
7373
}
7474

7575
public List<Long> getPossibleTraceIds(ComplexPattern pattern, String logname, Signature s) {
76-
String path = String.format("%s_sign_idx", logname);
77-
ExtractedPairsForPatternDetection pairs = pattern.extractPairsForPatternDetection(false);
78-
Set<Integer> positions1 = s.findPositionsWith1(pattern.getEventTypes(), pairs.getAllPairs());
79-
Iterator<Integer> iter = positions1.iterator();
80-
List<String> conditions = new ArrayList<>();
81-
while (iter.hasNext()) {
82-
conditions.add(" signature[" + iter.next().toString() + "]=" + "'1' ");
83-
}
84-
return sparkSession.read()
85-
.format("org.apache.spark.sql.cassandra")
86-
.options(Map.of("table", path, "keyspace", "siesta"))
87-
.load()
88-
.where(String.join("and", conditions))
89-
.toJavaRDD()
90-
.flatMap((FlatMapFunction<Row, Long>) row -> {
91-
List<String> traces = JavaConverters.seqAsJavaList(row.getSeq(1));
92-
return traces.stream().map(Long::parseLong).collect(Collectors.toList()).iterator();
93-
}).collect();
76+
return new ArrayList<>();
77+
//TODO: removed
78+
// String path = String.format("%s_sign_idx", logname);
79+
// ExtractedPairsForPatternDetection pairs = pattern.extractPairsForPatternDetection(false);
80+
// Set<Integer> positions1 = s.findPositionsWith1(pattern.getEventTypes(), pairs.getAllPairs());
81+
// Iterator<Integer> iter = positions1.iterator();
82+
// List<String> conditions = new ArrayList<>();
83+
// while (iter.hasNext()) {
84+
// conditions.add(" signature[" + iter.next().toString() + "]=" + "'1' ");
85+
// }
86+
// return sparkSession.read()
87+
// .format("org.apache.spark.sql.cassandra")
88+
// .options(Map.of("table", path, "keyspace", "siesta"))
89+
// .load()
90+
// .where(String.join("and", conditions))
91+
// .toJavaRDD()
92+
// .flatMap((FlatMapFunction<Row, Long>) row -> {
93+
// List<String> traces = JavaConverters.seqAsJavaList(row.getSeq(1));
94+
// return traces.stream().map(Long::parseLong).collect(Collectors.toList()).iterator();
95+
// }).collect();
9496
}
9597

9698
public Map<Long, List<Event>> getOriginalTraces(List<Long> traces, String logname) {

‎src/main/java/com/datalab/siesta/queryprocessor/controllers/QueryResponseController.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.datalab.siesta.queryprocessor.model.DBModel.Metadata;
44
import com.datalab.siesta.queryprocessor.model.Queries.QueryPlans.QueryPlan;
55
import com.datalab.siesta.queryprocessor.model.Queries.QueryResponses.QueryResponse;
6+
import com.datalab.siesta.queryprocessor.model.Queries.QueryResponses.QueryResponseGroups;
67
import com.datalab.siesta.queryprocessor.model.Queries.QueryResponses.QueryResponsePatternDetection;
78
import com.datalab.siesta.queryprocessor.model.Queries.QueryTypes.QueryExploration;
89
import com.datalab.siesta.queryprocessor.model.Queries.QueryTypes.QueryPatternDetection;
@@ -107,7 +108,7 @@ public ResponseEntity<MappingJacksonValue> getMetadata(@RequestBody QueryMetadat
107108
}
108109

109110
/**
110-
* Returns the stats for each concecutive event-pair. For example for the pattern ABC, it will return the stats
111+
* Returns the stats for each consecutive event-pair. For example for the pattern ABC, it will return the stats
111112
* for the event pairs A-B and B-C. Stats include min,max, average duration and number of completions.
112113
*/
113114
@RequestMapping(path = "/stats", method = RequestMethod.POST)
@@ -134,7 +135,7 @@ public ResponseEntity<String> patternDetection(@RequestBody QueryPatternDetectio
134135
} else {
135136
QueryPlan qp = qpd.createQueryPlan(qpdw, m);
136137
QueryResponse qrs = qp.execute(qpdw);
137-
if(qrs instanceof QueryResponsePatternDetection) {
138+
if(qrs instanceof QueryResponsePatternDetection || qrs instanceof QueryResponseGroups) {
138139
return new ResponseEntity<>(objectMapper.writeValueAsString(qrs), HttpStatus.OK);
139140
}else{
140141
return new ResponseEntity<>(objectMapper.writeValueAsString(qrs), HttpStatus.BAD_REQUEST);

‎src/main/java/com/datalab/siesta/queryprocessor/controllers/WebConfig.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package com.datalab.siesta.queryprocessor.controllers;
2+
3+
import org.springframework.context.annotation.Bean;
4+
import org.springframework.context.annotation.Configuration;
5+
import org.springframework.web.servlet.config.annotation.CorsRegistry;
6+
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
7+
8+
/**
9+
* Sets which origins are allowed to perform what requests
10+
*/
11+
@Configuration
12+
public class WebConfig {
13+
14+
@Bean
15+
public WebMvcConfigurer corsConfigurer() {
16+
return new WebMvcConfigurer() {
17+
@Override
18+
public void addCorsMappings(CorsRegistry registry) {
19+
registry.addMapping("/**")
20+
.allowedOrigins("*")
21+
.allowedMethods("GET", "POST", "PUT", "DELETE", "OPTIONS")
22+
.allowedHeaders("*");
23+
}
24+
};
25+
}
26+
}

‎src/main/java/com/datalab/siesta/queryprocessor/model/Constraints/TimeConstraint.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,17 @@ public TimeConstraint(int posA, int posB, long constraint, String granularity) {
3232

3333
public boolean isConstraintHolds(Count c) {
3434
if (method.equals("within")) {
35-
return c.getMin_duration() <= this.getConstraint();
35+
return c.getMin_duration() <= this.getConstraintInSeconds();
3636
} else if (method.equals("atleast")) {
37-
return this.getConstraint() <= c.getMax_duration();
37+
return this.getConstraintInSeconds() <= c.getMax_duration();
3838
} else return false;
3939
}
4040

4141
public long getConstraint() {
42+
return constraint;
43+
}
44+
45+
public long getConstraintInSeconds(){
4246
if(granularity.equals("minutes")) return constraint*60;
4347
else if (granularity.equals("hours")) return constraint*60*60;
4448
else return constraint;
@@ -72,8 +76,8 @@ public void setGranularity(String granularity) {
7276
public boolean isCorrect(EventBoth a, EventBoth b) {
7377
if (a.getTimestamp() == null || b.getTimestamp() == null) return false;
7478
if (this.method.equals("within")) {
75-
return (b.getTimestamp().getTime() - a.getTimestamp().getTime()) / 1000 <= this.getConstraint();
76-
} else return (b.getTimestamp().getTime() - a.getTimestamp().getTime()) / 1000 >= this.getConstraint();
79+
return (b.getTimestamp().getTime() - a.getTimestamp().getTime()) / 1000 <= this.getConstraintInSeconds();
80+
} else return (b.getTimestamp().getTime() - a.getTimestamp().getTime()) / 1000 >= this.getConstraintInSeconds();
7781
}
7882

7983
/**
@@ -83,7 +87,7 @@ public boolean isCorrect(EventBoth a, EventBoth b) {
8387
public long minimumChangeRequired(EventBoth a, EventBoth b) {
8488
if (a.getTimestamp() == null || b.getTimestamp() == null) return -1;
8589
if (this.method.equals("within")) {
86-
return (b.getTimestamp().getTime() / 1000) - (a.getTimestamp().getTime() / 1000) - this.getConstraint();
87-
} else return (a.getTimestamp().getTime() / 1000) + this.getConstraint() - (b.getTimestamp().getTime() / 1000);
90+
return (b.getTimestamp().getTime() / 1000) - (a.getTimestamp().getTime() / 1000) - this.getConstraintInSeconds();
91+
} else return (a.getTimestamp().getTime() / 1000) + this.getConstraintInSeconds() - (b.getTimestamp().getTime() / 1000);
8892
}
8993
}

‎src/main/java/com/datalab/siesta/queryprocessor/model/DBModel/Metadata.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,16 @@ public class Metadata {
5959
*/
6060
private Long traces;
6161

62+
/**
63+
* The first timestamp of a log database
64+
*/
65+
private String start_ts;
66+
67+
/**
68+
* The last timestamp of a log database
69+
*/
70+
private String last_ts;
71+
6272

6373
/**
6474
* Parse a json row. Utilized in S3, as metadata stored in json format
@@ -142,4 +152,20 @@ public Long getSplit_every_days() {
142152
public Long getTraces() {
143153
return traces;
144154
}
155+
156+
public String getStart_ts() {
157+
return start_ts;
158+
}
159+
160+
public void setStart_ts(String start_ts) {
161+
this.start_ts = start_ts;
162+
}
163+
164+
public String getLast_ts() {
165+
return last_ts;
166+
}
167+
168+
public void setLast_ts(String last_ts) {
169+
this.last_ts = last_ts;
170+
}
145171
}

‎src/main/java/com/datalab/siesta/queryprocessor/model/GroupOccurrences.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
package com.datalab.siesta.queryprocessor.model;
22

3-
import com.datalab.siesta.queryprocessor.model.Serializations.EventBothSerializer;
4-
import com.datalab.siesta.queryprocessor.model.Serializations.GroupOccurrencesSerialization;
3+
import com.amazonaws.thirdparty.jackson.annotation.JsonProperty;
4+
55
import org.codehaus.jackson.map.annotate.JsonSerialize;
6+
import com.datalab.siesta.queryprocessor.model.Serializations.CustomGroupOccurrencesSerializer;
67

78
import java.util.List;
89

@@ -11,9 +12,10 @@
1112
* defines the group id that the occurrences refer to. The trace id it was set to -1 and due to the
1213
* GroupOccurrencesSerialization.class it is not a member of the response json
1314
*/
14-
@JsonSerialize(using = GroupOccurrencesSerialization.class)
15+
@JsonSerialize(using = CustomGroupOccurrencesSerializer.class)
1516
public class GroupOccurrences extends Occurrences{
1617

18+
@JsonProperty("Group ID")
1719
private int groupId;
1820

1921
public GroupOccurrences() {

0 commit comments

Comments
 (0)