Skip to content

Globus patch - no need to lock the dataset while an upload is in progress, when tasks are monitored asynchronously. #11971

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
landreev wants to merge 6 commits into
base: develop
Choose a base branch
from
+63 −14
Open

Globus patch - no need to lock the dataset while an upload is in progress, when tasks are monitored asynchronously. #11971

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
69198c5
With this patch Globus uploads are no longer locking datasets for edi…
landreev Nov 4, 2025
ccf04b3
Merge branch 'develop' into prodpatch-6.8-globus-no-locking
landreev Nov 12, 2025
0256a0f
Merge branch 'develop' into prodpatch-6.8-globus-no-locking
stevenwinship Nov 12, 2025
93478fa
Unused commented-out code removed.
landreev Nov 18, 2025
21d8143
Merge branch 'prodpatch-6.8-globus-no-locking' of https://github.com/…
landreev Nov 18, 2025
afba69b
Merge branch 'develop' into prodpatch-6.8-globus-no-locking
landreev Nov 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@
import edu.harvard.iq.dataverse.search.SearchFields;
import edu.harvard.iq.dataverse.search.SearchUtil;
import edu.harvard.iq.dataverse.search.SolrClientService;
import edu.harvard.iq.dataverse.settings.FeatureFlags;
import edu.harvard.iq.dataverse.settings.JvmSettings;
import edu.harvard.iq.dataverse.util.SignpostingResources;
import edu.harvard.iq.dataverse.util.FileMetadataUtil;
Expand Down Expand Up @@ -2391,7 +2392,8 @@ public boolean isValidOrCanReviewIncomplete() {

private void displayLockInfo(Dataset dataset) {
// Various info messages, when the dataset is locked (for various reasons):
if (dataset.isLocked() && canUpdateDataset()) {
boolean globusUploadInProgress = globusService.isUploadTaskInProgressForDataset(dataset.getId());
if ((dataset.isLocked() || globusUploadInProgress) && canUpdateDataset()) {
if (dataset.isLockedFor(DatasetLock.Reason.Workflow)) {
JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("dataset.locked.message"),
BundleUtil.getStringFromBundle("dataset.locked.message.details"));
Expand All @@ -2405,9 +2407,17 @@ private void displayLockInfo(Dataset dataset) {
BundleUtil.getStringFromBundle("file.rsyncUpload.inProgressMessage.details"));
lockedDueToDcmUpload = true;
}
if (dataset.isLockedFor(DatasetLock.Reason.GlobusUpload)) {
JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.globusUpload.inProgressMessage.summary"),
BundleUtil.getStringFromBundle("file.globusUpload.inProgressMessage.details"));
if (dataset.isLockedFor(DatasetLock.Reason.GlobusUpload)
|| globusUploadInProgress) {
// (prod. patch 6.8) fall back to the old-style Globus lock message unless
// the new, async task mgmt model is used.
if (FeatureFlags.GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK.enabled()) {
JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.globusUpload.inProgressMessage.summary"),
BundleUtil.getStringFromBundle("file.globusUpload.inProgressMessageAsync.details"));
} else {
JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.globusUpload.inProgressMessage.summary"),
BundleUtil.getStringFromBundle("file.globusUpload.inProgressMessage.details"));
}
}
//This is a hack to remove dataset locks for File PID registration if
//the dataset is released
Expand Down Expand Up @@ -4381,10 +4391,10 @@ public boolean isStillLockedForAnyReason() {
if (dataset.getId() != null) {
Dataset testDataset = datasetService.find(dataset.getId());
if (testDataset != null && testDataset.getId() != null) {
// Refresh the info messages, in case the dataset has been
// re-locked with a different lock type (or a Globus upload task is in progress):
displayLockInfo(testDataset);
if (testDataset.getLocks().size() > 0) {
// Refresh the info messages, in case the dataset has been
// re-locked with a different lock type:
displayLockInfo(testDataset);
return true;
}
}
Expand Down
20 changes: 17 additions & 3 deletions src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
import edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetCommand;
import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand;
import edu.harvard.iq.dataverse.globus.GlobusServiceBean;
import edu.harvard.iq.dataverse.settings.FeatureFlags;
import edu.harvard.iq.dataverse.util.BundleUtil;
import edu.harvard.iq.dataverse.workflow.PendingWorkflowInvocation;
import edu.harvard.iq.dataverse.workflow.WorkflowServiceBean;
Expand Down Expand Up @@ -85,6 +87,9 @@ public class PermissionServiceBean {

@EJB
GroupServiceBean groupService;

@EJB
GlobusServiceBean globusService;

@Inject
DataverseSession session;
Expand Down Expand Up @@ -779,8 +784,14 @@ else if (dataset.isLockedFor(DatasetLock.Reason.Workflow)) {
else if (dataset.isLockedFor(DatasetLock.Reason.DcmUpload)) {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.message.locked.editNotAllowed"), command);
}
/**
* prod. patch 6.8: as an experiment, not locking datasets for edits while Globus uploads in progress:
*/
else if (dataset.isLockedFor(DatasetLock.Reason.GlobusUpload)) {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.message.locked.editNotAllowed"), command);
// ... but we'll keep it locked for edits unless the new, async task mgmt. is in use
if (!FeatureFlags.GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK.enabled()) {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.message.locked.editNotAllowed"), command);
}
}
else if (dataset.isLockedFor(DatasetLock.Reason.EditInProgress)) {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.message.locked.editNotAllowed"), command);
Expand All @@ -806,7 +817,9 @@ public void checkUpdateDatasetVersionLock(Dataset dataset, DataverseRequest data
}

public void checkPublishDatasetLock(Dataset dataset, DataverseRequest dataverseRequest, Command command) throws IllegalCommandException {
if (dataset.isLocked()) {
// prod. patch 6.8:
boolean globusUploadInProgress = globusService.isUploadTaskInProgressForDataset(dataset.getId());
if (dataset.isLocked() || globusUploadInProgress) {
if (dataset.isLockedFor(DatasetLock.Reason.Ingest)) {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.message.locked.publishNotAllowed"), command);
}
Expand All @@ -825,7 +838,8 @@ else if (dataset.isLockedFor(DatasetLock.Reason.Workflow)) {
else if (dataset.isLockedFor(DatasetLock.Reason.DcmUpload)) {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.message.locked.publishNotAllowed"), command);
}
else if (dataset.isLockedFor(DatasetLock.Reason.GlobusUpload)) {
else if (dataset.isLockedFor(DatasetLock.Reason.GlobusUpload)
|| globusUploadInProgress) {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.message.locked.publishNotAllowed"), command);
}
else if (dataset.isLockedFor(DatasetLock.Reason.EditInProgress)) {
Expand Down
20 changes: 20 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import org.primefaces.PrimeFaces;

import com.google.gson.Gson;
import com.rometools.utils.Lists;
import edu.harvard.iq.dataverse.api.ApiConstants;
import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
import edu.harvard.iq.dataverse.authorization.users.ApiToken;
Expand Down Expand Up @@ -1700,6 +1701,25 @@ public List<GlobusTaskInProgress> findAllOngoingTasks(GlobusTaskInProgress.TaskT
return em.createQuery("select object(o) from GlobusTaskInProgress as o where o.taskType=:taskType order by o.startTime", GlobusTaskInProgress.class).setParameter("taskType", taskType).getResultList();
}

public List<GlobusTaskInProgress> findAllOngoingTasksForDataset(GlobusTaskInProgress.TaskType taskType, Long datasetId) {
return em.createQuery("select object(o) from GlobusTaskInProgress as o where o.taskType=:taskType and o.dataset.id=:datasetId order by o.startTime", GlobusTaskInProgress.class)
.setParameter("taskType", taskType)
.setParameter("datasetId", datasetId)
.getResultList();
}

/**
* (prod. patch 6.8)
* @param datasetId
* @return
*/
public boolean isUploadTaskInProgressForDataset(Long datasetId) {
if (!FeatureFlags.GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK.enabled()) {
return false;
}
return Lists.isNotEmpty(findAllOngoingTasksForDataset(GlobusTaskInProgress.TaskType.UPLOAD, datasetId));
}

public boolean isRuleInUseByOtherTasks(String ruleId) {
Long numTask = em.createQuery("select count(o) from GlobusTaskInProgress as o where o.ruleId=:ruleId", Long.class).setParameter("ruleId", ruleId).getSingleResult();
return numTask > 1;
Expand Down
1 change: 1 addition & 0 deletions src/main/java/propertyFiles/Bundle.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1956,6 +1956,7 @@ file.rsyncUpload.rsyncUploadDisabledDueFileUploadedViaHttp=Upload with rsync + S
file.rsyncUpload.rsyncUploadDisabledDueFileUploadedViaHttpAndPublished=Upload with rsync + SSH is disabled for this dataset because you have already uploaded files via HTTP and published the dataset.
file.globusUpload.inProgressMessage.summary=Globus Transfer in Progress
file.globusUpload.inProgressMessage.details=This dataset is locked while the data files are being transferred and verified. Large transfers may take significant time. You can check transfer status at <a href="https://app.globus.org/activity" target="_blank">https://app.globus.org/activity</a>.
file.globusUpload.inProgressMessageAsync.details=This dataset cannot be published while Globus transfers are in progress. Large transfers may take significant time. You can check transfer status at <a href="https://app.globus.org/activity" target="_blank">https://app.globus.org/activity</a>.
file.metaData.checksum.copy=Click to copy
file.metaData.dataFile.dataTab.unf=UNF
file.metaData.dataFile.dataTab.variables=Variables
Expand Down
12 changes: 8 additions & 4 deletions src/main/webapp/dataset.xhtml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,12 +360,16 @@
<div class="btn-group">
<!-- Publish BTN -->
<h:outputLink value="#" disabled="#{DatasetPage.lockedFromPublishing or !DatasetPage.hasValidTermsOfAccess or !valid}">
<c:if test="#{!((showSubmitForReviewLink or showReturnToAuthorLink) and showPublishLink )}">
<f:passThroughAttribute name="class" value="btn btn-default btn-access btn-publish #{DatasetPage.lockedFromPublishing or !DatasetPage.hasValidTermsOfAccess or !valid ? 'disabled' : ''}"/>
<c:if test="#{!((showSubmitForReviewLink or showReturnToAuthorLink) and showPublishLink ) and !DatasetPage.lockedFromPublishing}">
<f:passThroughAttribute name="class" value="btn btn-default btn-access btn-publish #{!DatasetPage.hasValidTermsOfAccess or !valid ? 'disabled' : ''}"/>
<f:passThroughAttribute name="onclick" value="$(this).parent().find( 'li > a' ).trigger( 'click' );"/>
</c:if>
<c:if test="#{(showSubmitForReviewLink or showReturnToAuthorLink) and showPublishLink}">
<f:passThroughAttribute name="class" value="btn btn-default btn-access btn-publish dropdown-toggle #{DatasetPage.lockedFromPublishing or !DatasetPage.hasValidTermsOfAccess or !valid ? 'disabled' : ''}"/>
<!-- todo: make sure it's working properly with submit-for-review in place as well? -->
<c:if test="#{!((showSubmitForReviewLink or showReturnToAuthorLink) and showPublishLink ) and DatasetPage.lockedFromPublishing}">
<f:passThroughAttribute name="class" value="btn btn-default btn-access btn-publish disabled"/>
</c:if>
<c:if test="#{(showSubmitForReviewLink or showReturnToAuthorLink) and showPublishLink and !DatasetPage.lockedFromPublishing}">
<f:passThroughAttribute name="class" value="btn btn-default btn-access btn-publish dropdown-toggle #{!DatasetPage.hasValidTermsOfAccess or !valid ? 'disabled' : ''}"/>
<f:passThroughAttribute name="data-toggle" value="dropdown"/>
<f:passThroughAttribute name="aria-haspopup" value="true"/>
<f:passThroughAttribute name="aria-expanded" value="false"/>
Expand Down