Skip to content

Speed up cloning/migrating of loaded GO schemas #926

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
labkey-adam merged 6 commits into from
Sep 21, 2025
Merged

Speed up cloning/migrating of loaded GO schemas #926

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2520253
Speed up cloning/migrating of loaded GO schemas by dropping indices a…
labkey-adam Sep 11, 2025
3b0b2d5
Refactor MigrationHandler. Add some (virtual) foreign keys and provid…
labkey-adam Sep 12, 2025
7d2aeaf
Identify container field key in more tables. Exclude some tables.
labkey-adam Sep 12, 2025
5562fa1
Find container field keys in nearly every table
labkey-adam Sep 13, 2025
9a4ac54
Skip GO tables for now
labkey-adam Sep 17, 2025
ab13b79
Implement SkipSchemas property. Fix PropertyDomain and ProtocolApplic…
labkey-adam Sep 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions protein/api-src/org/labkey/api/protein/go/GoLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,25 @@ public void load()
});
}

private void loadGoFromGz() throws SQLException, IOException, ServletException
public static void dropGoIndexes()
{
DbSchema schema = ProteinSchema.getSchema();
new SqlExecutor(schema).execute(schema.getSqlDialect().execute(schema, "drop_go_indexes", ""));
}

public static void createGoIndexes()
{
DbSchema schema = ProteinSchema.getSchema();
new SqlExecutor(schema).execute(schema.getSqlDialect().execute(schema, "create_go_indexes", ""));
}

private void loadGoFromGz() throws SQLException, IOException, ServletException
{
Map<String, GoLoadBean> map = getGoLoadMap();
long start = System.currentTimeMillis();

clearGoLoaded();
new SqlExecutor(schema).execute(schema.getSqlDialect().execute(schema, "drop_go_indexes", ""));
dropGoIndexes();

logStatus("Starting to load GO annotation files");
logStatus("");
Expand All @@ -175,7 +186,7 @@ private void loadGoFromGz() throws SQLException, IOException, ServletException
}
}

new SqlExecutor(schema).execute(schema.getSqlDialect().execute(schema, "create_go_indexes", ""));
createGoIndexes();
long elapsed = System.currentTimeMillis() - start;

logStatus("Successfully loaded all GO annotation files (" + DateUtil.formatDuration(elapsed) + ")");
Expand Down
26 changes: 20 additions & 6 deletions protein/src/org/labkey/protein/ProteinModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import org.labkey.api.data.ContainerManager;
import org.labkey.api.data.DatabaseMigrationService;
import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationHandler;
import org.labkey.api.data.DbSchema;
import org.labkey.api.data.SqlExecutor;
import org.labkey.api.data.TableInfo;
import org.labkey.api.data.TableSelector;
import org.labkey.api.files.FileContentService;
import org.labkey.api.files.TableUpdaterFileListener;
Expand All @@ -36,9 +36,11 @@
import org.labkey.api.protein.annotation.CustomAnnotationSetManager;
import org.labkey.api.protein.annotation.ProteinAnnotationPipelineProvider;
import org.labkey.api.protein.fasta.FastaDbLoader;
import org.labkey.api.protein.go.GoLoader;
import org.labkey.api.protein.query.CustomAnnotationSchema;
import org.labkey.api.protein.query.ProteinUserSchema;
import org.labkey.api.protein.search.MSSearchWebpart;
import org.labkey.api.query.FieldKey;
import org.labkey.api.usageMetrics.UsageMetricsService;
import org.labkey.api.view.BaseWebPartFactory;
import org.labkey.api.view.Portal;
Expand Down Expand Up @@ -129,18 +131,30 @@ public void doStartup(ModuleContext moduleContext)
}

ProteinService.get().registerProteinSearchView(new ProteinSearchViewProvider());
DatabaseMigrationService.get().registerHandler(ProteinSchema.getSchema(), new DefaultMigrationHandler()
DatabaseMigrationService.get().registerHandler(new DefaultMigrationHandler(ProteinSchema.getSchema())
{
@Override
public void beforeSchema(DbSchema targetSchema)
public void beforeSchema()
{
new SqlExecutor(targetSchema).execute("ALTER TABLE prot.Organisms DROP CONSTRAINT FK_ProtOrganisms_ProtIdentifiers");
new SqlExecutor(getSchema()).execute("ALTER TABLE prot.Organisms DROP CONSTRAINT FK_ProtOrganisms_ProtIdentifiers");
GoLoader.dropGoIndexes();
}

@Override
public void afterSchema(DbSchema targetSchema)
public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable)
{
new SqlExecutor(targetSchema).execute("ALTER TABLE prot.Organisms ADD CONSTRAINT FK_ProtOrganisms_ProtIdentifiers FOREIGN KEY (IdentId) REFERENCES prot.Identifiers (IdentId)");
return switch (sourceTable.getName())
{
case "CustomAnnotation", "CustomAnnotationSet" -> super.getContainerFieldKey(sourceTable);
default -> SITE_WIDE_TABLE;
};
}

@Override
public void afterSchema()
{
new SqlExecutor(getSchema()).execute("ALTER TABLE prot.Organisms ADD CONSTRAINT FK_ProtOrganisms_ProtIdentifiers FOREIGN KEY (IdentId) REFERENCES prot.Identifiers (IdentId)");
GoLoader.createGoIndexes();
}
});
}
Expand Down