Skip to content

Commit 29b24cb

Browse files
luke-mino-altherrluke-mino-altherr
authored
refactor(assets): modular architecture + async two-phase scanner & background seeder (Comfy-Org#12621)
1 parent a7a6335 commit 29b24cb

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+10732
-2873
lines changed

‎alembic_db/versions/0002_merge_to_asset_references.py‎

Lines changed: 267 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,267 @@
1+
"""
2+
Merge AssetInfo and AssetCacheState into unified asset_references table.
3+
4+
This migration drops old tables and creates the new unified schema.
5+
All existing data is discarded.
6+
7+
Revision ID: 0002_merge_to_asset_references
8+
Revises: 0001_assets
9+
Create Date: 2025-02-11
10+
"""
11+
12+
from alembic import op
13+
import sqlalchemy as sa
14+
15+
revision = "0002_merge_to_asset_references"
16+
down_revision = "0001_assets"
17+
branch_labels = None
18+
depends_on = None
19+
20+
21+
def upgrade() -> None:
22+
# Drop old tables (order matters due to FK constraints)
23+
op.drop_index("ix_asset_info_meta_key_val_bool", table_name="asset_info_meta")
24+
op.drop_index("ix_asset_info_meta_key_val_num", table_name="asset_info_meta")
25+
op.drop_index("ix_asset_info_meta_key_val_str", table_name="asset_info_meta")
26+
op.drop_index("ix_asset_info_meta_key", table_name="asset_info_meta")
27+
op.drop_table("asset_info_meta")
28+
29+
op.drop_index("ix_asset_info_tags_asset_info_id", table_name="asset_info_tags")
30+
op.drop_index("ix_asset_info_tags_tag_name", table_name="asset_info_tags")
31+
op.drop_table("asset_info_tags")
32+
33+
op.drop_index("ix_asset_cache_state_asset_id", table_name="asset_cache_state")
34+
op.drop_index("ix_asset_cache_state_file_path", table_name="asset_cache_state")
35+
op.drop_table("asset_cache_state")
36+
37+
op.drop_index("ix_assets_info_owner_name", table_name="assets_info")
38+
op.drop_index("ix_assets_info_last_access_time", table_name="assets_info")
39+
op.drop_index("ix_assets_info_created_at", table_name="assets_info")
40+
op.drop_index("ix_assets_info_name", table_name="assets_info")
41+
op.drop_index("ix_assets_info_asset_id", table_name="assets_info")
42+
op.drop_index("ix_assets_info_owner_id", table_name="assets_info")
43+
op.drop_table("assets_info")
44+
45+
# Truncate assets table (cascades handled by dropping dependent tables first)
46+
op.execute("DELETE FROM assets")
47+
48+
# Create asset_references table
49+
op.create_table(
50+
"asset_references",
51+
sa.Column("id", sa.String(length=36), primary_key=True),
52+
sa.Column(
53+
"asset_id",
54+
sa.String(length=36),
55+
sa.ForeignKey("assets.id", ondelete="CASCADE"),
56+
nullable=False,
57+
),
58+
sa.Column("file_path", sa.Text(), nullable=True),
59+
sa.Column("mtime_ns", sa.BigInteger(), nullable=True),
60+
sa.Column(
61+
"needs_verify",
62+
sa.Boolean(),
63+
nullable=False,
64+
server_default=sa.text("false"),
65+
),
66+
sa.Column(
67+
"is_missing", sa.Boolean(), nullable=False, server_default=sa.text("false")
68+
),
69+
sa.Column("enrichment_level", sa.Integer(), nullable=False, server_default="0"),
70+
sa.Column("owner_id", sa.String(length=128), nullable=False, server_default=""),
71+
sa.Column("name", sa.String(length=512), nullable=False),
72+
sa.Column(
73+
"preview_id",
74+
sa.String(length=36),
75+
sa.ForeignKey("assets.id", ondelete="SET NULL"),
76+
nullable=True,
77+
),
78+
sa.Column("user_metadata", sa.JSON(), nullable=True),
79+
sa.Column("created_at", sa.DateTime(timezone=False), nullable=False),
80+
sa.Column("updated_at", sa.DateTime(timezone=False), nullable=False),
81+
sa.Column("last_access_time", sa.DateTime(timezone=False), nullable=False),
82+
sa.Column("deleted_at", sa.DateTime(timezone=False), nullable=True),
83+
sa.CheckConstraint(
84+
"(mtime_ns IS NULL) OR (mtime_ns >= 0)", name="ck_ar_mtime_nonneg"
85+
),
86+
sa.CheckConstraint(
87+
"enrichment_level >= 0 AND enrichment_level <= 2",
88+
name="ck_ar_enrichment_level_range",
89+
),
90+
)
91+
op.create_index(
92+
"uq_asset_references_file_path", "asset_references", ["file_path"], unique=True
93+
)
94+
op.create_index("ix_asset_references_asset_id", "asset_references", ["asset_id"])
95+
op.create_index("ix_asset_references_owner_id", "asset_references", ["owner_id"])
96+
op.create_index("ix_asset_references_name", "asset_references", ["name"])
97+
op.create_index("ix_asset_references_is_missing", "asset_references", ["is_missing"])
98+
op.create_index(
99+
"ix_asset_references_enrichment_level", "asset_references", ["enrichment_level"]
100+
)
101+
op.create_index("ix_asset_references_created_at", "asset_references", ["created_at"])
102+
op.create_index(
103+
"ix_asset_references_last_access_time", "asset_references", ["last_access_time"]
104+
)
105+
op.create_index(
106+
"ix_asset_references_owner_name", "asset_references", ["owner_id", "name"]
107+
)
108+
op.create_index("ix_asset_references_deleted_at", "asset_references", ["deleted_at"])
109+
110+
# Create asset_reference_tags table
111+
op.create_table(
112+
"asset_reference_tags",
113+
sa.Column(
114+
"asset_reference_id",
115+
sa.String(length=36),
116+
sa.ForeignKey("asset_references.id", ondelete="CASCADE"),
117+
nullable=False,
118+
),
119+
sa.Column(
120+
"tag_name",
121+
sa.String(length=512),
122+
sa.ForeignKey("tags.name", ondelete="RESTRICT"),
123+
nullable=False,
124+
),
125+
sa.Column(
126+
"origin", sa.String(length=32), nullable=False, server_default="manual"
127+
),
128+
sa.Column("added_at", sa.DateTime(timezone=False), nullable=False),
129+
sa.PrimaryKeyConstraint(
130+
"asset_reference_id", "tag_name", name="pk_asset_reference_tags"
131+
),
132+
)
133+
op.create_index(
134+
"ix_asset_reference_tags_tag_name", "asset_reference_tags", ["tag_name"]
135+
)
136+
op.create_index(
137+
"ix_asset_reference_tags_asset_reference_id",
138+
"asset_reference_tags",
139+
["asset_reference_id"],
140+
)
141+
142+
# Create asset_reference_meta table
143+
op.create_table(
144+
"asset_reference_meta",
145+
sa.Column(
146+
"asset_reference_id",
147+
sa.String(length=36),
148+
sa.ForeignKey("asset_references.id", ondelete="CASCADE"),
149+
nullable=False,
150+
),
151+
sa.Column("key", sa.String(length=256), nullable=False),
152+
sa.Column("ordinal", sa.Integer(), nullable=False, server_default="0"),
153+
sa.Column("val_str", sa.String(length=2048), nullable=True),
154+
sa.Column("val_num", sa.Numeric(38, 10), nullable=True),
155+
sa.Column("val_bool", sa.Boolean(), nullable=True),
156+
sa.Column("val_json", sa.JSON(), nullable=True),
157+
sa.PrimaryKeyConstraint(
158+
"asset_reference_id", "key", "ordinal", name="pk_asset_reference_meta"
159+
),
160+
)
161+
op.create_index("ix_asset_reference_meta_key", "asset_reference_meta", ["key"])
162+
op.create_index(
163+
"ix_asset_reference_meta_key_val_str", "asset_reference_meta", ["key", "val_str"]
164+
)
165+
op.create_index(
166+
"ix_asset_reference_meta_key_val_num", "asset_reference_meta", ["key", "val_num"]
167+
)
168+
op.create_index(
169+
"ix_asset_reference_meta_key_val_bool",
170+
"asset_reference_meta",
171+
["key", "val_bool"],
172+
)
173+
174+
175+
def downgrade() -> None:
176+
"""Reverse 0002_merge_to_asset_references: drop new tables, recreate old schema.
177+
178+
NOTE: Data is not recoverable. The upgrade discards all rows from the old
179+
tables and truncates assets. After downgrade the old schema will be empty.
180+
A filesystem rescan will repopulate data once the older code is running.
181+
"""
182+
# Drop new tables (order matters due to FK constraints)
183+
op.drop_index("ix_asset_reference_meta_key_val_bool", table_name="asset_reference_meta")
184+
op.drop_index("ix_asset_reference_meta_key_val_num", table_name="asset_reference_meta")
185+
op.drop_index("ix_asset_reference_meta_key_val_str", table_name="asset_reference_meta")
186+
op.drop_index("ix_asset_reference_meta_key", table_name="asset_reference_meta")
187+
op.drop_table("asset_reference_meta")
188+
189+
op.drop_index("ix_asset_reference_tags_asset_reference_id", table_name="asset_reference_tags")
190+
op.drop_index("ix_asset_reference_tags_tag_name", table_name="asset_reference_tags")
191+
op.drop_table("asset_reference_tags")
192+
193+
op.drop_index("ix_asset_references_deleted_at", table_name="asset_references")
194+
op.drop_index("ix_asset_references_owner_name", table_name="asset_references")
195+
op.drop_index("ix_asset_references_last_access_time", table_name="asset_references")
196+
op.drop_index("ix_asset_references_created_at", table_name="asset_references")
197+
op.drop_index("ix_asset_references_enrichment_level", table_name="asset_references")
198+
op.drop_index("ix_asset_references_is_missing", table_name="asset_references")
199+
op.drop_index("ix_asset_references_name", table_name="asset_references")
200+
op.drop_index("ix_asset_references_owner_id", table_name="asset_references")
201+
op.drop_index("ix_asset_references_asset_id", table_name="asset_references")
202+
op.drop_index("uq_asset_references_file_path", table_name="asset_references")
203+
op.drop_table("asset_references")
204+
205+
# Truncate assets (upgrade deleted all rows; downgrade starts fresh too)
206+
op.execute("DELETE FROM assets")
207+
208+
# Recreate old tables from 0001_assets schema
209+
op.create_table(
210+
"assets_info",
211+
sa.Column("id", sa.String(length=36), primary_key=True),
212+
sa.Column("owner_id", sa.String(length=128), nullable=False, server_default=""),
213+
sa.Column("name", sa.String(length=512), nullable=False),
214+
sa.Column("asset_id", sa.String(length=36), sa.ForeignKey("assets.id", ondelete="RESTRICT"), nullable=False),
215+
sa.Column("preview_id", sa.String(length=36), sa.ForeignKey("assets.id", ondelete="SET NULL"), nullable=True),
216+
sa.Column("user_metadata", sa.JSON(), nullable=True),
217+
sa.Column("created_at", sa.DateTime(timezone=False), nullable=False),
218+
sa.Column("updated_at", sa.DateTime(timezone=False), nullable=False),
219+
sa.Column("last_access_time", sa.DateTime(timezone=False), nullable=False),
220+
sa.UniqueConstraint("asset_id", "owner_id", "name", name="uq_assets_info_asset_owner_name"),
221+
)
222+
op.create_index("ix_assets_info_owner_id", "assets_info", ["owner_id"])
223+
op.create_index("ix_assets_info_asset_id", "assets_info", ["asset_id"])
224+
op.create_index("ix_assets_info_name", "assets_info", ["name"])
225+
op.create_index("ix_assets_info_created_at", "assets_info", ["created_at"])
226+
op.create_index("ix_assets_info_last_access_time", "assets_info", ["last_access_time"])
227+
op.create_index("ix_assets_info_owner_name", "assets_info", ["owner_id", "name"])
228+
229+
op.create_table(
230+
"asset_cache_state",
231+
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
232+
sa.Column("asset_id", sa.String(length=36), sa.ForeignKey("assets.id", ondelete="CASCADE"), nullable=False),
233+
sa.Column("file_path", sa.Text(), nullable=False),
234+
sa.Column("mtime_ns", sa.BigInteger(), nullable=True),
235+
sa.Column("needs_verify", sa.Boolean(), nullable=False, server_default=sa.text("false")),
236+
sa.CheckConstraint("(mtime_ns IS NULL) OR (mtime_ns >= 0)", name="ck_acs_mtime_nonneg"),
237+
sa.UniqueConstraint("file_path", name="uq_asset_cache_state_file_path"),
238+
)
239+
op.create_index("ix_asset_cache_state_file_path", "asset_cache_state", ["file_path"])
240+
op.create_index("ix_asset_cache_state_asset_id", "asset_cache_state", ["asset_id"])
241+
242+
op.create_table(
243+
"asset_info_tags",
244+
sa.Column("asset_info_id", sa.String(length=36), sa.ForeignKey("assets_info.id", ondelete="CASCADE"), nullable=False),
245+
sa.Column("tag_name", sa.String(length=512), sa.ForeignKey("tags.name", ondelete="RESTRICT"), nullable=False),
246+
sa.Column("origin", sa.String(length=32), nullable=False, server_default="manual"),
247+
sa.Column("added_at", sa.DateTime(timezone=False), nullable=False),
248+
sa.PrimaryKeyConstraint("asset_info_id", "tag_name", name="pk_asset_info_tags"),
249+
)
250+
op.create_index("ix_asset_info_tags_tag_name", "asset_info_tags", ["tag_name"])
251+
op.create_index("ix_asset_info_tags_asset_info_id", "asset_info_tags", ["asset_info_id"])
252+
253+
op.create_table(
254+
"asset_info_meta",
255+
sa.Column("asset_info_id", sa.String(length=36), sa.ForeignKey("assets_info.id", ondelete="CASCADE"), nullable=False),
256+
sa.Column("key", sa.String(length=256), nullable=False),
257+
sa.Column("ordinal", sa.Integer(), nullable=False, server_default="0"),
258+
sa.Column("val_str", sa.String(length=2048), nullable=True),
259+
sa.Column("val_num", sa.Numeric(38, 10), nullable=True),
260+
sa.Column("val_bool", sa.Boolean(), nullable=True),
261+
sa.Column("val_json", sa.JSON(), nullable=True),
262+
sa.PrimaryKeyConstraint("asset_info_id", "key", "ordinal", name="pk_asset_info_meta"),
263+
)
264+
op.create_index("ix_asset_info_meta_key", "asset_info_meta", ["key"])
265+
op.create_index("ix_asset_info_meta_key_val_str", "asset_info_meta", ["key", "val_str"])
266+
op.create_index("ix_asset_info_meta_key_val_num", "asset_info_meta", ["key", "val_num"])
267+
op.create_index("ix_asset_info_meta_key_val_bool", "asset_info_meta", ["key", "val_bool"])

0 commit comments

Comments
 (0)