Skip to content

Commit 16681ba

Browse files
bbimberbbimber
committed
Auto-create demographics records from SIV study assignment
1 parent 0496cb3 commit 16681ba

File tree

5 files changed

+192
-2
lines changed

5 files changed

+192
-2
lines changed

‎SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,30 @@
11
package org.labkey.sivstudies.notification;
22

3+
import org.apache.logging.log4j.Logger;
34
import org.jetbrains.annotations.Nullable;
5+
import org.labkey.api.data.CompareType;
46
import org.labkey.api.data.Container;
7+
import org.labkey.api.data.SimpleFilter;
58
import org.labkey.api.data.TableInfo;
69
import org.labkey.api.data.TableSelector;
710
import org.labkey.api.ldk.notification.AbstractNotification;
811
import org.labkey.api.module.ModuleLoader;
12+
import org.labkey.api.query.FieldKey;
913
import org.labkey.api.query.QueryService;
1014
import org.labkey.api.query.UserSchema;
1115
import org.labkey.api.security.User;
16+
import org.labkey.api.study.Study;
17+
import org.labkey.api.study.StudyService;
18+
import org.labkey.api.util.logging.LogHelper;
1219
import org.labkey.sivstudies.SivStudiesModule;
20+
import org.labkey.sivstudies.query.DefaultDatasetTrigger;
1321

1422
import java.util.Date;
1523

1624
public class SivStudiesDataValidationNotification extends AbstractNotification
1725
{
26+
protected static final Logger _log = LogHelper.getLogger(DefaultDatasetTrigger.class, "Messages related to SivStudiesDataValidationNotification");
27+
1828
public SivStudiesDataValidationNotification()
1929
{
2030
super(ModuleLoader.getInstance().getModule(SivStudiesModule.class));
@@ -65,6 +75,7 @@ public String getEmailSubject(Container c)
6575
duplicateInfectionCheck(c, u, msg);
6676
infectionAnchorDateDiscordance(c, u, msg);
6777
pvlWithoutInfectionDate(c, u, msg);
78+
idsMissingFromDemographics(c, u, msg);
6879

6980
if (!msg.isEmpty())
7081
{
@@ -102,10 +113,15 @@ private void infectionAnchorDateDiscordance(Container c, User u, StringBuilder m
102113
}
103114

104115
private void genericQueryCheck(Container c, User u, StringBuilder msg, String schemaName, String queryName, String message)
116+
{
117+
genericQueryCheck(c, u, msg, schemaName, queryName, message, null);
118+
}
119+
120+
private void genericQueryCheck(Container c, User u, StringBuilder msg, String schemaName, String queryName, String message, @Nullable SimpleFilter filter)
105121
{
106122
TableInfo ti = getTableInfo(u, c, schemaName, queryName);
107123

108-
TableSelector ts = new TableSelector(ti);
124+
TableSelector ts = new TableSelector(ti, filter, null);
109125
long count = ts.getRowCount();
110126
if (count > 0)
111127
{
@@ -119,4 +135,21 @@ private void pvlWithoutInfectionDate(Container c, User u, StringBuilder msg)
119135
{
120136
genericQueryCheck(c, u, msg, "study", "pvlWithoutInfectionDate", "animals with PVL data but no record of SIV infection");
121137
}
138+
139+
private void idsMissingFromDemographics(Container c, User u, StringBuilder msg)
140+
{
141+
Study s = StudyService.get().getStudy(getTargetContainer(c));
142+
if (s == null)
143+
{
144+
return;
145+
}
146+
147+
SimpleFilter filter = new SimpleFilter(FieldKey.fromString("DataSet/Demographics/" + s.getSubjectColumnName()), null, CompareType.ISBLANK);
148+
genericQueryCheck(c, u, msg, "study", s.getSubjectNounSingular(), "IDs with data in the study not present in the demographics table", filter);
149+
}
150+
151+
protected Container getTargetContainer(Container c)
152+
{
153+
return c.isWorkbookOrTab() ? c.getParent() : c;
154+
}
122155
}

‎SivStudies/src/org/labkey/sivstudies/query/AutoCreateDemographicsTrigger.java‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package org.labkey.sivstudies.query;
2+
3+
import org.apache.logging.log4j.Logger;
4+
import org.jetbrains.annotations.NotNull;
5+
import org.jetbrains.annotations.Nullable;
6+
import org.labkey.api.collections.CaseInsensitiveHashSet;
7+
import org.labkey.api.data.CompareType;
8+
import org.labkey.api.data.Container;
9+
import org.labkey.api.data.SimpleFilter;
10+
import org.labkey.api.data.TableInfo;
11+
import org.labkey.api.data.TableSelector;
12+
import org.labkey.api.data.triggers.Trigger;
13+
import org.labkey.api.data.triggers.TriggerFactory;
14+
import org.labkey.api.query.BatchValidationException;
15+
import org.labkey.api.query.DuplicateKeyException;
16+
import org.labkey.api.query.FieldKey;
17+
import org.labkey.api.query.QueryService;
18+
import org.labkey.api.query.QueryUpdateServiceException;
19+
import org.labkey.api.query.ValidationException;
20+
import org.labkey.api.security.User;
21+
import org.labkey.api.study.Study;
22+
import org.labkey.api.study.StudyService;
23+
import org.labkey.api.util.PageFlowUtil;
24+
import org.labkey.api.util.logging.LogHelper;
25+
26+
import java.sql.SQLException;
27+
import java.util.Collection;
28+
import java.util.List;
29+
import java.util.Map;
30+
31+
public class AutoCreateDemographicsTrigger extends DefaultDatasetTrigger
32+
{
33+
protected static final Logger _log = LogHelper.getLogger(DefaultDatasetTrigger.class, "Messages related to CreateDemographicsTrigger");
34+
35+
public static class Factory implements TriggerFactory
36+
{
37+
public Factory()
38+
{
39+
40+
}
41+
42+
@Override
43+
public @NotNull Collection<Trigger> createTrigger(@Nullable Container c, TableInfo table, Map<String, Object> extraContext)
44+
{
45+
return List.of(new AutoCreateDemographicsTrigger());
46+
}
47+
}
48+
49+
private static final String CACHE_KEY = "~~AutoCreateDemographicsTrigger.IdsToCreate~~";
50+
51+
@Override
52+
protected void afterUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
53+
{
54+
if (extraContext == null)
55+
{
56+
_log.error("extraContext is null in AutoCreateDemographicsTrigger.afterUpsert()");
57+
return;
58+
}
59+
60+
if (!extraContext.containsKey(AutoCreateDemographicsTrigger.CACHE_KEY))
61+
{
62+
extraContext.put(CACHE_KEY, new CaseInsensitiveHashSet());
63+
}
64+
65+
if (extraContext.get(CACHE_KEY) instanceof CaseInsensitiveHashSet s)
66+
{
67+
String idField = getIdField(c);
68+
String id = newRow.get(idField) != null ? newRow.get(idField).toString() : null;
69+
if (id != null)
70+
{
71+
s.add(id);
72+
}
73+
}
74+
}
75+
76+
@Override
77+
public void complete(TableInfo table, Container c, User user, TableInfo.TriggerType event, BatchValidationException errors, Map<String, Object> extraContext)
78+
{
79+
if (extraContext == null)
80+
{
81+
_log.error("extraContext is null in AutoCreateDemographicsTrigger.complete()");
82+
return;
83+
}
84+
85+
if (extraContext.get(CACHE_KEY) instanceof CaseInsensitiveHashSet s)
86+
{
87+
s = new CaseInsensitiveHashSet(s);
88+
89+
String idField = getIdField(c);
90+
TableInfo ti = QueryService.get().getUserSchema(user, getTargetContainer(c), "study").getTable("demographics");
91+
List<String> existingIds = new TableSelector(ti, PageFlowUtil.set(idField), new SimpleFilter(FieldKey.fromString(idField), s, CompareType.IN), null).getArrayList(String.class);
92+
93+
s.removeAll(existingIds);
94+
95+
if (!s.isEmpty())
96+
{
97+
List<Map<String, Object>> toInsert = s.stream().map(id -> Map.of(idField, (Object)id)).toList();
98+
try
99+
{
100+
ti.getUpdateService().insertRows(user, c, toInsert, null, null, null);
101+
}
102+
catch (SQLException | BatchValidationException | QueryUpdateServiceException | DuplicateKeyException e)
103+
{
104+
_log.error("Error creating demographics records", e);
105+
}
106+
}
107+
}
108+
}
109+
110+
private String _idField = null;
111+
112+
private String getIdField(Container c)
113+
{
114+
if (_idField == null)
115+
{
116+
Study s = StudyService.get().getStudy(getTargetContainer(c));
117+
if (s == null)
118+
{
119+
return null;
120+
}
121+
122+
_idField = s.getSubjectColumnName();
123+
}
124+
125+
return _idField;
126+
}
127+
}

‎SivStudies/src/org/labkey/sivstudies/query/DefaultDatasetTrigger.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,29 @@ public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map<
4040
beforeInsert(table, c, user, newRow, errors, extraContext, null);
4141
}
4242

43+
@Override
44+
public void afterInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext, @Nullable Map<String, Object> existingRecord) throws ValidationException
45+
{
46+
afterUpsert(table, c, user, newRow, existingRecord, errors, extraContext);
47+
}
48+
49+
@Override
50+
public void afterInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
51+
{
52+
afterInsert(table, c, user, newRow, errors, extraContext, null);
53+
}
54+
55+
@Override
56+
public void afterUpdate(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
57+
{
58+
afterUpsert(table, c, user, newRow, oldRow, errors, extraContext);
59+
}
60+
61+
protected void afterUpsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, @Nullable Map<String, Object> oldRow, ValidationException errors, Map<String, Object> extraContext) throws ValidationException
62+
{
63+
64+
}
65+
4366
@Override
4467
public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map<String, Object> newRow, ValidationException errors, Map<String, Object> extraContext, @Nullable Map<String, Object> existingRecord) throws ValidationException
4568
{
@@ -84,4 +107,10 @@ private void mergeOldToNewRow(@NotNull Map<String, Object> newRow, @Nullable Map
84107
}
85108
}
86109
}
110+
111+
protected Container getTargetContainer(Container c)
112+
{
113+
return c.isWorkbookOrTab() ? c.getParent() : c;
114+
}
115+
87116
}

‎SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ public void performDatasetCustomization(DatasetTable ds)
7979
if ("assignment".equalsIgnoreCase(ds.getName()))
8080
{
8181
ati.addTriggerFactory(StudiesService.get().getStudiesTriggerFactory());
82+
ati.addTriggerFactory(new AutoCreateDemographicsTrigger.Factory());
8283
}
8384
}
8485
else

‎mcc/resources/etls/mcc.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
<column>objectid</column>
3939
</sourceColumns>
4040
</source>
41-
<destination schemaName="study" queryName="Demographics" targetOption="truncate" bulkLoad="true" batchSize="5000">
41+
<destination schemaName="study" queryName="Demographics" targetOption="truncate" bulkLoad="true" batchSize="2500">
4242
<alternateKeys>
4343
<column name="objectid"/>
4444
</alternateKeys>

0 commit comments

Comments
 (0)