Skip to content

Fix: Resolve watcher service deadlock during self-restart on folder deletion #1269

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
DeveloperAmrit wants to merge 1 commit into
base: main
Choose a base branch
from
+132 −118
Open

Fix: Resolve watcher service deadlock during self-restart on folder deletion #1269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 113 additions & 113 deletions docs/backend/backend_python/openapi.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2073,119 +2073,6 @@
],
"title": "DeleteFoldersResponse"
},
"app__schemas__folders__ErrorResponse": {
"properties": {
"success": {
"type": "boolean",
"title": "Success",
"default": false
},
"message": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Message"
},
"error": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Error"
}
},
"type": "object",
"title": "ErrorResponse"
},
"app__schemas__face_clusters__ErrorResponse": {
"properties": {
"success": {
"type": "boolean",
"title": "Success",
"default": false
},
"message": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Message"
},
"error": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Error"
}
},
"type": "object",
"title": "ErrorResponse"
},
"app__schemas__images__ErrorResponse": {
"properties": {
"success": {
"type": "boolean",
"title": "Success",
"default": false
},
"message": {
"type": "string",
"title": "Message"
},
"error": {
"type": "string",
"title": "Error"
}
},
"type": "object",
"required": [
"message",
"error"
],
"title": "ErrorResponse"
},
"app__schemas__user_preferences__ErrorResponse": {
"properties": {
"success": {
"type": "boolean",
"title": "Success"
},
"error": {
"type": "string",
"title": "Error"
},
"message": {
"type": "string",
"title": "Message"
}
},
"type": "object",
"required": [
"success",
"error",
"message"
],
"title": "ErrorResponse",
"description": "Error response model"
},
"FaceSearchRequest": {
"properties": {
"path": {
Expand Down Expand Up @@ -3407,6 +3294,119 @@
"type"
],
"title": "ValidationError"
},
"app__schemas__face_clusters__ErrorResponse": {
"properties": {
"success": {
"type": "boolean",
"title": "Success",
"default": false
},
"message": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Message"
},
"error": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Error"
}
},
"type": "object",
"title": "ErrorResponse"
},
"app__schemas__folders__ErrorResponse": {
"properties": {
"success": {
"type": "boolean",
"title": "Success",
"default": false
},
"message": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Message"
},
"error": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Error"
}
},
"type": "object",
"title": "ErrorResponse"
},
"app__schemas__images__ErrorResponse": {
"properties": {
"success": {
"type": "boolean",
"title": "Success",
"default": false
},
"message": {
"type": "string",
"title": "Message"
},
"error": {
"type": "string",
"title": "Error"
}
},
"type": "object",
"required": [
"message",
"error"
],
"title": "ErrorResponse"
},
"app__schemas__user_preferences__ErrorResponse": {
"properties": {
"success": {
"type": "boolean",
"title": "Success"
},
"error": {
"type": "string",
"title": "Error"
},
"message": {
"type": "string",
"title": "Message"
}
},
"type": "object",
"required": [
"success",
"error",
"message"
],
"title": "ErrorResponse",
"description": "Error response model"
}
}
}
Expand Down
24 changes: 19 additions & 5 deletions sync-microservice/app/utils/watcher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,13 +311,16 @@ def watcher_util_stop_folder_watcher() -> None:
# Signal the watcher to stop
stop_event.set()

# Wait for thread to finish
watcher_thread.join(timeout=5.0)
# Wait for thread to finish if not called from the thread itself
if threading.current_thread() != watcher_thread:
watcher_thread.join(timeout=5.0)

if watcher_thread.is_alive():
logger.warning("Warning: Watcher thread did not stop gracefully")
if watcher_thread.is_alive():
logger.warning("Warning: Watcher thread did not stop gracefully")
else:
logger.info("Watcher stopped successfully")
else:
logger.info("Watcher stopped successfully")
logger.info("Watcher stopped successfully (called from within watcher thread)")

except Exception as e:
logger.error(f"Error stopping watcher: {e}")
Expand All @@ -336,6 +339,17 @@ def watcher_util_restart_folder_watcher() -> bool:
True if restart was successful, False otherwise
"""
logger.info("Restarting folder watcher...")

if watcher_thread and threading.current_thread() == watcher_thread:
# We are inside the watcher thread, so we can't join ourselves.
# Launch a background thread to do the restart.
def restart_worker():
watcher_util_stop_folder_watcher()
watcher_util_start_folder_watcher()

threading.Thread(target=restart_worker, daemon=True).start()
return True
Comment on lines +343 to +351
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot May 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

Add an automated regression test for the self-restart path.

This is critical thread-lifecycle logic. Please add a test that triggers restart from inside the watcher worker path and verifies there is no self-join hang and restart completes.

As per coding guidelines, "Ensure that test code is automated, comprehensive, and follows testing best practices" and "Verify that all critical functionality is covered by tests."

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@sync-microservice/app/utils/watcher.py` around lines 343 - 351, Add an
automated regression test that exercises the self-restart path inside the
watcher worker so we never hang on a self-join: spawn a real watcher_thread (or
simulate it via running the watcher worker function on a new Thread), ensure
threading.current_thread() == watcher_thread when calling the restart logic, and
invoke the code path that creates restart_worker (the branch referencing
watcher_thread and threading.current_thread()). In the test, mock or patch
watcher_util_stop_folder_watcher and watcher_util_start_folder_watcher to record
calls, start the restart path from inside that watcher thread, and assert within
a short timeout that (a) the background restart thread was started, (b) mocked
stop/start functions were called, and (c) the test thread does not block (use a
join with timeout or an event to fail on hang). Target functions/symbols to
locate code: watcher_thread, restart_worker, watcher_util_stop_folder_watcher,
watcher_util_start_folder_watcher, and the branch that checks
threading.current_thread() == watcher_thread.


watcher_util_stop_folder_watcher()
return watcher_util_start_folder_watcher()

Expand Down
Loading