[Bug]: Dropout probability > 0 with compiled blocks causes error #1162
Description
What happened?
Trying the Chroma 24GB preset with compile transformer blocks enabled and dropout probability of 0.05 on the LoRA tab is leading to an error. It works fine when it's set to 0.
Was also able to reproduce the issue on WSL2 (clean venv from latest commit).
What did you expect would happen?
It should work.
Relevant log output
Traceback (most recent call last):
File "M:\OneTrainer\modules\ui\TrainUI.py", line 754, in __training_thread_function
trainer.train()
File "M:\OneTrainer\modules\trainer\GenericTrainer.py", line 737, in train
model_output_data = self.model_setup.predict(self.model, batch, self.config, train_progress)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\modules\modelSetup\BaseChromaSetup.py", line 233, in predict
packed_predicted_flow = model.transformer(
^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\nn\modules\module.py", line 1773, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\nn\modules\module.py", line 1784, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\src\diffusers\src\diffusers\models\transformers\transformer_chroma.py", line 577, in forward
encoder_hidden_states, hidden_states = block(
^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\nn\modules\module.py", line 1771, in _wrapped_call_impl
return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\eval_frame.py", line 749, in compile_wrapper
raise e.remove_dynamo_frames() from None # see TORCHDYNAMO_VERBOSE=1
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\eval_frame.py", line 736, in compile_wrapper
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\nn\modules\module.py", line 1784, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\convert_frame.py", line 1495, in __call__
return self._torchdynamo_orig_callable(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\convert_frame.py", line 629, in __call__
return _compile(
^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\convert_frame.py", line 1111, in _compile
guarded_code = compile_inner(code, one_graph, hooks, transform)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_utils_internal.py", line 97, in wrapper_function
return function(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\convert_frame.py", line 793, in compile_inner
return _compile_inner(code, one_graph, hooks, transform)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\convert_frame.py", line 832, in _compile_inner
out_code = transform_code_object(code, transform)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\bytecode_transformation.py", line 1424, in transform_code_object
transformations(instructions, code_options)
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\convert_frame.py", line 267, in _fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\convert_frame.py", line 753, in transform
tracer.run()
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\symbolic_convert.py", line 3497, in run
super().run()
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\symbolic_convert.py", line 1363, in run
while self.step():
^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\symbolic_convert.py", line 1267, in step
self.dispatch_table[inst.opcode](self, inst)
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\symbolic_convert.py", line 3672, in RETURN_VALUE
self._return(inst)
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\symbolic_convert.py", line 3653, in _return
all_stack_locals_metadata = self.output.compile_subgraph(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\output_graph.py", line 1422, in compile_subgraph
self.compile_and_call_fx_graph(tx, pass2.graph_output_vars(), root)
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\output_graph.py", line 1696, in compile_and_call_fx_graph
compiled_fn = self.call_user_compiler(gm, self.example_inputs())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\output_graph.py", line 1811, in call_user_compiler
return self._call_user_compiler(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\output_graph.py", line 1871, in _call_user_compiler
raise BackendCompilerFailed(
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\output_graph.py", line 1846, in _call_user_compiler
compiled_fn = compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\repro\after_dynamo.py", line 150, in __call__
compiled_gm = compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\__init__.py", line 2380, in __call__
return compile_fx(model_, inputs_, config_patches=self.config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_inductor\compile_fx.py", line 2418, in compile_fx
return aot_autograd(
^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_dynamo\backends\common.py", line 109, in __call__
cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_functorch\aot_autograd.py", line 1199, in aot_module_simplified
compiled_fn = AOTAutogradCache.load(
^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_functorch\_aot_autograd\autograd_cache.py", line 1140, in load
compiled_fn = dispatch_and_compile()
^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_functorch\aot_autograd.py", line 1184, in dispatch_and_compile
compiled_fn, _ = create_aot_dispatcher_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_functorch\aot_autograd.py", line 576, in create_aot_dispatcher_function
return _create_aot_dispatcher_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_functorch\aot_autograd.py", line 836, in _create_aot_dispatcher_function
compiled_fn, fw_metadata = compiler_fn(
^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_functorch\_aot_autograd\jit_compile_runtime_wrappers.py", line 1320, in aot_dispatch_autograd
fw_module, bw_module = aot_config.partition_fn(
^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_inductor\compile_fx.py", line 2300, in partition_fn
return min_cut_rematerialization_partition(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_functorch\partitioners.py", line 2640, in min_cut_rematerialization_partition
fw_module, bw_module = _extract_fwd_bwd_modules(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_functorch\partitioners.py", line 817, in _extract_fwd_bwd_modules
bwd_graph = _extract_graph_with_inputs_outputs(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "M:\OneTrainer\venv\Lib\site-packages\torch\_functorch\partitioners.py", line 232, in _extract_graph_with_inputs_outputs
assert not isinstance(
^^^^^^^^^^^^^^^
torch._dynamo.exc.BackendCompilerFailed: backend='inductor' raised:
AssertionError: Node convert_element_type_265 was invalid, but is outputGenerate and upload debug_report.log
=== System Information ===
OS: Windows 11
Version: 10.0.26200
=== Hardware Information ===
CPU: AMD Ryzen 9 9950X 16-Core Processor (Cores: 16)
Total RAM: 119.6 GB
=== GPU Information ===
NVIDIA GPU (Index 0): NVIDIA GeForce RTX 5090 [NVIDIA]
Driver version: 581.29
Power Limit: 600.00 W
=== Python Environment ===
Global Python Version: 3.12.12
Python Executable Path: M:\OneTrainer\venv\Scripts\python.exe
PyTorch Info: torch==2.8.0+cu128
pip freeze output:
absl-py==2.3.1
accelerate==1.7.0
adv_optm==1.1.3
aiodns==3.5.0
aiohappyeyeballs==2.6.1
aiohttp==3.13.2
aiohttp-retry==2.9.1
aiosignal==1.4.0
annotated-doc==0.0.4
annotated-types==0.7.0
antlr4-python3-runtime==4.9.3
anyio==4.11.0
attrs==25.4.0
av==14.4.0
backoff==2.2.1
backports.zstd==1.0.0
bcrypt==5.0.0
bitsandbytes==0.46.0
boto3==1.41.2
botocore==1.41.2
brotli==1.2.0
certifi==2025.11.12
cffi==2.0.0
charset-normalizer==3.4.4
click==8.3.1
cloudpickle==3.1.2
colorama==0.4.6
coloredlogs==15.0.1
contourpy==1.3.3
cryptography==45.0.7
customtkinter==5.2.2
cycler==0.12.1
dadaptation==3.2
darkdetect==0.8.0
decorator==5.2.1
deepdiff==8.6.1
Deprecated==1.3.1
-e git+https://github.com/huggingface/diffusers.git@9b721db205729d5a6e97a72312c3a0f4534064f1#egg=diffusers
dnspython==2.8.0
einops==0.8.1
email-validator==2.3.0
fabric==3.2.2
fastapi==0.121.3
fastapi-cli==0.0.16
fastapi-cloud-cli==0.5.1
fastar==0.6.0
filelock==3.20.0
flatbuffers==25.9.23
fonttools==4.60.1
frozenlist==1.8.0
fsspec==2025.10.0
ftfy==6.3.1
gguf==0.17.1
grpcio==1.76.0
h11==0.16.0
httpcore==1.0.9
httptools==0.7.1
httpx==0.28.1
huggingface-hub==0.34.4
humanfriendly==10.0
idna==3.11
imagesize==1.4.1
importlib_metadata==8.7.0
inquirerpy==0.3.4
invisible-watermark==0.2.0
invoke==2.2.1
itsdangerous==2.2.0
Jinja2==3.1.6
jmespath==1.0.1
kiwisolver==1.4.9
lightning-utilities==0.15.2
lion-pytorch==0.2.3
Markdown==3.10
markdown-it-py==4.0.0
MarkupSafe==3.0.3
matplotlib==3.10.3
mdurl==0.1.2
-e git+https://github.com/Nerogar/mgds.git@efa0f6b0ebceb24208ab0449dbc71c2ef99e95ae#egg=mgds
mpmath==1.3.0
multidict==6.7.0
networkx==3.5
numpy==2.2.6
nvidia-ml-py==13.580.82
omegaconf==2.3.0
-e git+https://github.com/Open-Model-Initiative/OMI-Model-Standards.git@f14b1da606811d2004f9241c3463c240eaf09ac5#egg=omi_model_standards
onnxruntime-gpu==1.22.0
open_clip_torch==2.32.0
opencv-python==4.11.0.86
orderly-set==5.5.0
orjson==3.11.4
packaging==25.0
paramiko==4.0.0
pfzy==0.3.4
pillow==11.3.0
platformdirs==4.5.0
pooch==1.8.2
prettytable==3.17.0
prodigy-plus-schedule-free==2.0.1
prodigyopt==1.1.2
prompt_toolkit==3.0.52
propcache==0.4.1
protobuf==6.33.1
psutil==7.0.0
py-cpuinfo==9.0.0
pycares==4.11.0
pycparser==2.23
pydantic==2.12.4
pydantic-extra-types==2.10.6
pydantic-settings==2.12.0
pydantic_core==2.41.5
Pygments==2.19.2
PyNaCl==1.6.1
pyparsing==3.2.5
pyreadline3==3.5.4
python-dateutil==2.9.0.post0
python-dotenv==1.2.1
python-multipart==0.0.20
pytorch-lightning==2.5.1.post0
pytorch_optimizer==3.6.0
PyWavelets==1.9.0
PyYAML==6.0.2
regex==2025.11.3
requests==2.32.3
rich==14.2.0
rich-toolkit==0.16.0
rignore==0.7.6
runpod==1.7.10
s3transfer==0.15.0
safetensors==0.5.3
scalene==1.5.51
scenedetect==0.6.6
schedulefree==1.4.1
scipy==1.15.3
sentencepiece==0.2.1
sentry-sdk==2.45.0
setuptools==80.9.0
shellingham==1.5.4
six==1.17.0
sniffio==1.3.1
starlette==0.50.0
sympy==1.14.0
tensorboard==2.19.0
tensorboard-data-server==0.7.2
timm==1.0.22
tokenizers==0.22.1
tomli==2.3.0
tomlkit==0.13.3
torch==2.8.0+cu128
torchmetrics==1.8.2
torchvision==0.23.0+cu128
tqdm==4.67.1
tqdm-loggable==0.2
transformers==4.56.2
triton-windows==3.4.0.post20
typer==0.20.0
typing-inspection==0.4.2
typing_extensions==4.15.0
ujson==5.11.0
urllib3==2.5.0
uvicorn==0.38.0
watchdog==6.0.0
watchfiles==1.1.1
wcwidth==0.2.14
websockets==15.0.1
Werkzeug==3.1.3
wheel==0.45.1
wrapt==2.0.1
yarl==1.22.0
yt-dlp==2025.11.12
zipp==3.23.0
=== Git Information ===
Repo: Nerogar/OneTrainer
Branch: master
Commit: 335560bd1044b5b2c9e2a7002836db79e9312084
No deleted, unmerged, or modified files relative to origin/master.
=== Network Connectivity ===
PyPI (https://pypi.org/): Ping to pypi.org successful: Packet Loss: 0%
HuggingFace (https://huggingface.co): Ping to huggingface.co successful: Packet Loss: 0%
Google (https://www.google.com): Ping to www.google.com successful: Packet Loss: 0%
=== Intel Microcode Information ===
CPU is not detected as 13th or 14th Gen Intel - microcode info not applicable.