Scene crash fixes

.github/workflows/all_builds.yml

@@ -28,7 +28,7 @@ env:
   # TODO: change this back to godotengine/godot and target master when #109685 and #109475 are merged
   GODOT_REPOSITORY: nikitalita/godot
   # Change the README too
-  GODOT_MAIN_SYNC_REF: gdre-wb-63227bb
+  GODOT_MAIN_SYNC_REF: gdre-wb-1559ab34c6
   SCONSFLAGS: verbose=yes warnings=all werror=no module_text_server_fb_enabled=yes minizip=yes deprecated=yes
   SCONSFLAGS_TEMPLATE: no_editor_splash=yes module_camera_enabled=no module_mobile_vr_enabled=no module_upnp_enabled=no module_websocket_enabled=no module_csg_enabled=yes module_gridmap_enabled=yes use_static_cpp=yes builtin_freetype=yes builtin_libpng=yes builtin_zlib=yes builtin_libwebp=yes builtin_libvorbis=yes builtin_libogg=yes disable_3d=no
   SCONS_CACHE_MSVC_CONFIG: true

.scripts/rebase_godot.sh

@@ -1,21 +1,30 @@
 #!/bin/sh
 
 SCRIPT_PATH=$(dirname "$0")
+GDRE_PATH=$(realpath "$SCRIPT_PATH/..")
 cd "$SCRIPT_PATH/../../.."
 
 git fetch --all
 git checkout master
 git pull
 
 HEAD=$(git rev-parse HEAD)
+SHORT_HASH=$(git rev-parse --short HEAD)
+NEW_BRANCH_NAME="gdre-wb-$SHORT_HASH"
 
 # check for the existence of the 'nikitalita' remote
 if ! git remote | grep -q "nikitalita"; then
     git remote add nikitalita https://github.com/nikitalita/godot.git
 	git fetch nikitalita
 fi
 
-git checkout working-branch
+# check if the branch already exists
+if git branch -a | grep -q $NEW_BRANCH_NAME; then
+    git branch -D $NEW_BRANCH_NAME
+fi
+git branch -C $NEW_BRANCH_NAME
+
+git checkout $NEW_BRANCH_NAME
 git reset --hard $HEAD
 
 BRANCHES_TO_MERGE=(
@@ -26,9 +35,29 @@ BRANCHES_TO_MERGE=(
 	convert-3.x-escn
 	fix-svg
 	fix-compat-array-shapes
+	gltf-buffer-view-encode-fix
 )
 
+# set fail on error
 for branch in "${BRANCHES_TO_MERGE[@]}"; do
     # merge branch, use a merge commit
     git merge nikitalita/$branch -m "Merge branch '$branch'"
+	if [ $? -ne 0 ]; then
+		echo "Error: Failed to merge branch '$branch'"
+		exit 1
+	fi
 done
+
+# detect OS for cross-platform sed compatibility
+# macOS (BSD sed) requires -i '' while Linux (GNU sed) uses -i
+if [ "$(uname)" = "Darwin" ]; then
+    sed_in_place() { sed -i '' "$@"; }
+else
+    sed_in_place() { sed -i "$@"; }
+fi
+
+git push nikitalita $NEW_BRANCH_NAME --set-upstream
+
+# change the branch name in .github/workflows/all_builds.yml and the README.md
+sed_in_place "s/GODOT_MAIN_SYNC_REF: .*/GODOT_MAIN_SYNC_REF: $NEW_BRANCH_NAME/" "$GDRE_PATH/.github/workflows/all_builds.yml"
+sed_in_place "s/ @ branch \`.*\`/ @ branch \`$NEW_BRANCH_NAME\`/" "$GDRE_PATH/README.md"

README.md

@@ -149,7 +149,7 @@ Note: Make sure to build the editor build first, and to launch the editor to edi
 
 ### Requirements
 
-[Our fork of godot](https://github.com/nikitalita/godot/tree/working-branch) @ `gdre-wb-63227bb`
+[Our fork of godot](https://github.com/nikitalita/godot) @ branch `gdre-wb-1559ab34c6`
 
 - Support for building on 3.x has been dropped and no new features are being pushed
   - Godot RE Tools still retains the ability to decompile 3.x and 2.x projects, however.

exporters/scene_exporter.cpp

@@ -29,6 +29,7 @@
 #include "utility/gdre_logger.h"
 #include "utility/gdre_settings.h"
 
+#include "core/crypto/crypto_core.h"
 #include "core/error/error_list.h"
 #include "core/error/error_macros.h"
 #include "main/main.h"
@@ -38,8 +39,12 @@
 #include "utility/task_manager.h"
 
 #ifndef MAKE_GLTF_COPY
+#ifdef DEBUG_ENABLED
+#define MAKE_GLTF_COPY 1
+#else
 #define MAKE_GLTF_COPY 0
 #endif
+#endif
 #ifndef PRINT_PERF_CSV
 #define PRINT_PERF_CSV 0
 #endif
@@ -1695,7 +1700,7 @@
 				}
 				if (meshes.size() > 1) {
 					SurfaceTool surface_tool;
 					for (size_t i = 0; i < meshes.size(); i++) {
 						for (size_t j = 0; j < meshes[i]->get_surface_count(); j++) {
 							if (i == 0 && j == 0) {
 								surface_tool.create_from(meshes[i], j);
@@ -2408,6 +2413,85 @@
 	}
 }
 
+Ref<Image> GLBExporterInstance::_parse_image_bytes_into_image(const Ref<GLTFState> &p_state, const Vector<uint8_t> &p_bytes, const String &p_mime_type, int p_index, String &r_file_extension) {
+	Ref<Image> r_image;
+	r_image.instantiate();
+	// Check if any GLTFDocumentExtensions want to import this data as an image.
+	for (auto &ext : GLTFDocument::get_all_gltf_document_extensions()) {
+		ERR_CONTINUE(ext.is_null());
+		Error err = ext->parse_image_data(p_state, p_bytes, p_mime_type, r_image);
+		ERR_CONTINUE_MSG(err != OK, "glTF: Encountered error " + itos(err) + " when parsing image " + itos(p_index) + " in file " + p_state->get_filename() + ". Continuing.");
+		if (!r_image->is_empty()) {
+			r_file_extension = ext->get_image_file_extension();
+			return r_image;
+		}
+	}
+	// If no extension wanted to import this data as an image, try to load a PNG or JPEG.
+	// First we honor the mime types if they were defined.
+	if (p_mime_type == "image/png") { // Load buffer as PNG.
+		r_image->load_png_from_buffer(p_bytes);
+		r_file_extension = ".png";
+	} else if (p_mime_type == "image/jpeg") { // Loader buffer as JPEG.
+		r_image->load_jpg_from_buffer(p_bytes);
+		r_file_extension = ".jpg";
+	}
+	// If we didn't pass the above tests, we attempt loading as PNG and then JPEG directly.
+	// This covers URIs with base64-encoded data with application/* type but
+	// no optional mimeType property, or bufferViews with a bogus mimeType
+	// (e.g. `image/jpeg` but the data is actually PNG).
+	// That's not *exactly* what the spec mandates but this lets us be
+	// lenient with bogus glb files which do exist in production.
+	if (r_image->is_empty()) { // Try PNG first.
+		r_image->load_png_from_buffer(p_bytes);
+	}
+	if (r_image->is_empty()) { // And then JPEG.
+		r_image->load_jpg_from_buffer(p_bytes);
+	}
+	// If it still can't be loaded, give up and insert an empty image as placeholder.
+	if (r_image->is_empty()) {
+		ERR_PRINT(vformat("glTF: Couldn't load image index '%d' with its given mimetype: %s.", p_index, p_mime_type));
+	}
+	return r_image;
+}
+
+String GLBExporterInstance::get_gltf_image_hash(const Ref<GLTFState> &p_state, const Dictionary &dict, int p_index) {
+	String image_hash;
+	String mime_type;
+	if (dict.has("mimeType")) { // Should be "image/png", "image/jpeg", or something handled by an extension.
+		mime_type = dict["mimeType"];
+	}
+
+	String resource_uri;
+	// We only need the image hash if it's embedded in the glTF file, so if it's not a bufferView, we can return early.
+	Vector<uint8_t> data;
+	if (dict.has("bufferView")) {
+		// Handles the third bullet point from the spec (bufferView).
+		ERR_FAIL_COND_V_MSG(mime_type.is_empty(), "", vformat("glTF: Image index '%d' specifies 'bufferView' but no 'mimeType', which is invalid.", p_index));
+		const GLTFBufferViewIndex bvi = dict["bufferView"];
+		auto buffer_views = p_state->get_buffer_views();
+		ERR_FAIL_INDEX_V(bvi, buffer_views.size(), "");
+		Ref<GLTFBufferView> bv = buffer_views[bvi];
+		const GLTFBufferIndex bi = bv->get_buffer();
+		auto buffers = p_state->get_buffers();
+		ERR_FAIL_INDEX_V(bi, buffers.size(), "");
+		ERR_FAIL_COND_V(bv->get_byte_offset() + bv->get_byte_length() > buffers[bi].size(), "");
+		const PackedByteArray &buffer = buffers[bi];
+		data = buffer.slice(bv->get_byte_offset(), bv->get_byte_offset() + bv->get_byte_length());
+	}
+	if (data.is_empty()) {
+		return image_hash;
+	}
+	String ext;
+	Ref<Image> img = _parse_image_bytes_into_image(p_state, data, mime_type, p_index, ext);
+	if (img.is_valid()) {
+		auto img_data = img->get_data();
+		unsigned char md5_hash[16];
+		CryptoCore::md5(img_data.ptr(), img_data.size(), md5_hash);
+		image_hash = String::hex_encode_buffer(md5_hash, 16);
+	}
+	return image_hash;
+}
+
 Error GLBExporterInstance::_export_instanced_scene(Node *root, const String &p_dest_path) {
 	{
 		GDRE_SCN_EXP_CHECK_CANCEL();
@@ -2564,7 +2648,9 @@
 				} else {
 					name = path.get_file().get_basename();
 					external_deps_updated.insert(path);
-					image_path_to_data_hash[path] = FileAccess::get_md5(path);
+					if (updating_import_info && is_batch_export) {
+						image_path_to_data_hash[path] = get_gltf_image_hash(state, image_dict, i);
+					}
 				}
 				check_unique(name, image_map);
 
@@ -3008,7 +3094,9 @@
 
 	iinfo->set_param("_subresources", _subresources_dict);
 	Dictionary extra_info = report->get_extra_info();
-	extra_info["image_path_to_data_hash"] = image_path_to_data_hash;
+	if (!image_path_to_data_hash.is_empty()) {
+		extra_info["image_path_to_data_hash"] = image_path_to_data_hash;
+	}
 	report->set_extra_info(extra_info);
 }
 
@@ -3447,18 +3535,17 @@
 		if (check_unsupported()) {
 			err = ERR_UNAVAILABLE;
 			_set_unsupported(report, ver_major, is_obj_output());
-			preload_done = true;
+			after_preload();
 			return false;
 		}
 
 		if (is_text_output()) {
-			preload_done = true;
 			return false;
 		}
 		err = _check_cancelled();
 		if (err != OK) {
 			report->set_error(err);
-			preload_done = true;
+			after_preload();
 			return false;
 		}
 		instance._initial_set(p_src_path, report);
@@ -3474,7 +3561,7 @@
 			}
 			if (err != OK) {
 				report->set_error(err);
-				preload_done = true;
+				after_preload();
 				return false;
 			}
 			_scene = scene;
@@ -3484,7 +3571,7 @@
 					_scene = nullptr;
 					err = ERR_CANT_ACQUIRE_RESOURCE;
 					report->set_error(err);
-					preload_done = true;
+					after_preload();
 					return false;
 				}
 				root = instance._set_stuff_from_instanced_scene(root);
@@ -3501,10 +3588,19 @@
 			surface_count = get_total_surface_count(meshes);
 		}
 		// print_line("Preloaded scene " + p_src_path);
-		preload_done = true;
+		after_preload();
 		return do_on_main_thread;
 	}
 
+	void after_preload() {
+		if (Thread::is_main_thread() && _scene.is_valid()) {
+			// We have to flush the message queue after the scene is loaded;
+			// Certain resources like NoiseTexture2D can queue up deferred calls that will cause crashes if not flushed before the scene is manipulated or freed
+			GDRESettings::main_iteration();
+		}
+		preload_done = true;
+	}
+
 	void batch_export_instanced_scene() {
 		while (!preload_done && _check_cancelled() == OK) {
 			OS::get_singleton()->delay_usec(10000);
@@ -3738,7 +3834,7 @@
 Vector<String> GLBExporterInstance::_get_logged_error_messages() {
 	auto errors = supports_multithread() ? GDRELogger::get_thread_errors() : GDRELogger::get_errors();
 	Vector<String> ret;
 	for (auto &err : errors) {
 		String lstripped = err.strip_edges(true, false);
 		if (!lstripped.begins_with("GDScript backtrace")) {
 			ret.push_back(err.strip_edges(false, true));

exporters/scene_exporter.h

@@ -198,6 +198,9 @@ class GLBExporterInstance {
 
 	String demangle_name(const String &obj_name);
 
+	static Ref<Image> _parse_image_bytes_into_image(const Ref<GLTFState> &p_state, const Vector<uint8_t> &p_bytes, const String &p_mime_type, int p_index, String &r_file_extension);
+	static String get_gltf_image_hash(const Ref<GLTFState> &p_state, const Dictionary &dict, int p_index);
+
 public:
 	GLBExporterInstance(String p_output_dir, Dictionary curr_options = {}, bool p_project_recovery = false);
 

standalone/gdre_main.gd

@@ -126,6 +126,13 @@ func end_recovery():
 func extract_and_recover(files_to_extract: PackedStringArray, output_dir: String, extract_only: bool):
 	%GdreRecover.hide_win()
 	if not extract_only:
+		if GDREConfig.get_setting("Recovery/clear_output_dir_except_git_before_full_recovery"):
+			if files_to_extract.size() >= GDRESettings.get_file_info_array().size():
+				var err = GDRECommon.clear_dir_except_for(output_dir, [".git", ".gitignore"])
+				if err != OK:
+					print("Error: failed to clear output directory except for git")
+					return 1
+
 		GDRESettings.open_log_file(output_dir)
 	var log_path = GDRESettings.get_log_file_path()
 	GDRESettings.get_errors()
@@ -973,6 +980,15 @@ func recovery(  input_files:PackedStringArray,
 		if (da.file_exists(output_dir)):
 			print("Error: output dir appears to be a file, not extracting...")
 			return 1
+
+	var not_full_recovery = extract_only or translation_only or scripts_only or includes.size() > 0 or excludes.size() > 0
+	if !not_full_recovery and GDREConfig.get_setting("Recovery/clear_output_dir_except_git_before_full_recovery"):
+		var log_file = GDRESettings.get_log_file_path().get_file()
+		err = GDRECommon.clear_dir_except_for(output_dir, [".git", ".gitignore", log_file])
+		if (err != OK):
+			print("Error: failed to clear output directory except for git")
+			return 1
+
 	if is_dir:
 		if extract_only:
 			print("Why did you open a folder to extract it??? What's wrong with you?!!?")

utility/common.cpp

@@ -1537,6 +1537,29 @@ Ref<Image> gdre::load_image_from_file(const String &p_path) {
 	return image;
 }
 
+Error gdre::clear_dir_except_for(const String &p_dir, const Vector<String> &p_files_or_dirs) {
+	if (!DirAccess::dir_exists_absolute(p_dir)) {
+		return OK;
+	}
+	HashSet<String> files_or_dirs_set = gdre::vector_to_hashset(p_files_or_dirs);
+	auto da = DirAccess::create_for_path(p_dir);
+	if (da.is_null()) {
+		return ERR_FILE_CANT_OPEN;
+	}
+	da->change_dir(p_dir);
+	da->list_dir_begin();
+	String file = da->get_next();
+	while (!file.is_empty()) {
+		if (file == "." || file == ".." || files_or_dirs_set.has(file)) {
+			file = da->get_next();
+			continue;
+		}
+		rimraf(p_dir.path_join(file));
+		file = da->get_next();
+	}
+	return OK;
+}
+
 void GDRECommon::_bind_methods() {
 	//	ClassDB::bind_static_method("GLTFCamera", D_METHOD("from_node", "camera_node"), &GLTFCamera::from_node);
 
@@ -1565,4 +1588,5 @@ void GDRECommon::_bind_methods() {
 	ClassDB::bind_static_method("GDRECommon", D_METHOD("get_dirs_at", "dir", "wildcards", "absolute"), &gdre::get_dirs_at);
 	ClassDB::bind_static_method("GDRECommon", D_METHOD("get_recursive_dir_list_multithread", "dir", "wildcards", "absolute", "include_hidden", "exclude_filters", "files_first", "exclude_dot_prefix_and_gdignore", "show_progress"), &gdre::get_recursive_dir_list_multithread, DEFVAL(PackedStringArray()), DEFVAL(true), DEFVAL(true), DEFVAL(PackedStringArray()), DEFVAL(false), DEFVAL(false), DEFVAL(false));
 	ClassDB::bind_static_method("GDRECommon", D_METHOD("get_safe_dir_name", "dir_name", "allow_paths"), &gdre::get_safe_dir_name, DEFVAL(false));
+	ClassDB::bind_static_method("GDRECommon", D_METHOD("clear_dir_except_for", "dir", "files_or_dirs"), &gdre::clear_dir_except_for);
 }

utility/common.h

@@ -375,6 +375,7 @@ String path_to_uri(const String &p_path);
 bool is_zip_file(const String &p_path);
 String get_safe_dir_name(const String &p_dir_name, bool p_allow_paths = false);
 Ref<Image> load_image_from_file(const String &p_path);
+Error clear_dir_except_for(const String &p_dir, const Vector<String> &p_files_or_dirs);
 } // namespace gdre
 
 class GDRECommon : public Object {

utility/gdre_config.cpp

@@ -249,6 +249,12 @@ Vector<Ref<GDREConfigSetting>> GDREConfig::_init_default_settings() {
 				"Use scene view by default",
 				"Use scene view by default instead of the text preview.\nWARNING: Scene view is still experimental and certain scenes may cause the program to become unresponsive.",
 				false)),
+		memnew(GDREConfigSetting(
+				"Recovery/clear_output_dir_except_git_before_full_recovery",
+				"Clear output directory except git before full recovery",
+				"Clear the output directory except for the .git directory before running a full recovery (i.e. no includes/excludes)",
+				false,
+				true)),
 		memnew(GDREConfigSetting(
 				"Recovery/git/create_git_repo",
 				"Create git repo in recovered project",

utility/gdre_settings.cpp

@@ -685,6 +685,7 @@ Error GDRESettings::_add_pack(const String &path) {
 
 Error GDRESettings::load_project(const Vector<String> &p_paths, bool _cmd_line_extract, const String &csharp_assembly_override) {
 	GDRELogger::clear_error_queues();
+	error_encryption = false;
 	if (is_pack_loaded()) {
 		return ERR_ALREADY_IN_USE;
 	}