/**************************************************************************/ /* editor_file_system.cpp */ /**************************************************************************/ /* This file is part of: */ /* GODOT ENGINE */ /* https://godotengine.org */ /**************************************************************************/ /* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ /* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ /* "Software"), to deal in the Software without restriction, including */ /* without limitation the rights to use, copy, modify, merge, publish, */ /* distribute, sublicense, and/or sell copies of the Software, and to */ /* permit persons to whom the Software is furnished to do so, subject to */ /* the following conditions: */ /* */ /* The above copyright notice and this permission notice shall be */ /* included in all copies or substantial portions of the Software. */ /* */ /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ /* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ /* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ /* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ /* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ /* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /**************************************************************************/ #include "editor_file_system.h" #include "core/config/project_settings.h" #include "core/extension/gdextension_manager.h" #include "core/io/dir_access.h" #include "core/io/file_access.h" #include "core/io/resource_saver.h" #include "core/object/worker_thread_pool.h" #include "core/os/os.h" #include "core/variant/variant_parser.h" #include "editor/editor_help.h" #include "editor/editor_node.h" #include "editor/editor_paths.h" #include "editor/editor_resource_preview.h" #include "editor/editor_settings.h" #include "editor/plugins/script_editor_plugin.h" #include "editor/project_settings_editor.h" #include "scene/resources/packed_scene.h" EditorFileSystem *EditorFileSystem::singleton = nullptr; int EditorFileSystem::nb_files_total = 0; EditorFileSystem::ScannedDirectory *EditorFileSystem::first_scan_root_dir = nullptr; //the name is the version, to keep compatibility with different versions of Godot #define CACHE_FILE_NAME "filesystem_cache10" int EditorFileSystemDirectory::find_file_index(const String &p_file) const { for (int i = 0; i < files.size(); i++) { if (files[i]->file == p_file) { return i; } } return -1; } int EditorFileSystemDirectory::find_dir_index(const String &p_dir) const { for (int i = 0; i < subdirs.size(); i++) { if (subdirs[i]->name == p_dir) { return i; } } return -1; } void EditorFileSystemDirectory::force_update() { // We set modified_time to 0 to force `EditorFileSystem::_scan_fs_changes` to search changes in the directory modified_time = 0; } int EditorFileSystemDirectory::get_subdir_count() const { return subdirs.size(); } EditorFileSystemDirectory *EditorFileSystemDirectory::get_subdir(int p_idx) { ERR_FAIL_INDEX_V(p_idx, subdirs.size(), nullptr); return subdirs[p_idx]; } int EditorFileSystemDirectory::get_file_count() const { return files.size(); } String EditorFileSystemDirectory::get_file(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), ""); return files[p_idx]->file; } String EditorFileSystemDirectory::get_path() const { int parents = 0; const EditorFileSystemDirectory *efd = this; // Determine the level of nesting. while (efd->parent) { parents++; efd = efd->parent; } if (parents == 0) { return "res://"; } // Using PackedStringArray, because the path is built in reverse order. PackedStringArray path_bits; // Allocate an array based on nesting. It will store path bits. path_bits.resize(parents + 2); // Last String is empty, so paths end with /. String *path_write = path_bits.ptrw(); path_write[0] = "res:/"; efd = this; for (int i = parents; i > 0; i--) { path_write[i] = efd->name; efd = efd->parent; } return String("/").join(path_bits); } String EditorFileSystemDirectory::get_file_path(int p_idx) const { return get_path().path_join(get_file(p_idx)); } Vector<String> EditorFileSystemDirectory::get_file_deps(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), Vector<String>()); Vector<String> deps; for (int i = 0; i < files[p_idx]->deps.size(); i++) { String dep = files[p_idx]->deps[i]; int sep_idx = dep.find("::"); //may contain type information, unwanted if (sep_idx != -1) { dep = dep.substr(0, sep_idx); } ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(dep); if (uid != ResourceUID::INVALID_ID) { //return proper dependency resource from uid if (ResourceUID::get_singleton()->has_id(uid)) { dep = ResourceUID::get_singleton()->get_id_path(uid); } else { continue; } } deps.push_back(dep); } return deps; } bool EditorFileSystemDirectory::get_file_import_is_valid(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), false); return files[p_idx]->import_valid; } uint64_t EditorFileSystemDirectory::get_file_modified_time(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), 0); return files[p_idx]->modified_time; } uint64_t EditorFileSystemDirectory::get_file_import_modified_time(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), 0); return files[p_idx]->import_modified_time; } String EditorFileSystemDirectory::get_file_script_class_name(int p_idx) const { return files[p_idx]->class_info.name; } String EditorFileSystemDirectory::get_file_script_class_extends(int p_idx) const { return files[p_idx]->class_info.extends; } String EditorFileSystemDirectory::get_file_script_class_icon_path(int p_idx) const { return files[p_idx]->class_info.icon_path; } String EditorFileSystemDirectory::get_file_icon_path(int p_idx) const { return files[p_idx]->class_info.icon_path; } StringName EditorFileSystemDirectory::get_file_type(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), ""); return files[p_idx]->type; } StringName EditorFileSystemDirectory::get_file_resource_script_class(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), ""); return files[p_idx]->resource_script_class; } String EditorFileSystemDirectory::get_name() { return name; } EditorFileSystemDirectory *EditorFileSystemDirectory::get_parent() { return parent; } void EditorFileSystemDirectory::_bind_methods() { ClassDB::bind_method(D_METHOD("get_subdir_count"), &EditorFileSystemDirectory::get_subdir_count); ClassDB::bind_method(D_METHOD("get_subdir", "idx"), &EditorFileSystemDirectory::get_subdir); ClassDB::bind_method(D_METHOD("get_file_count"), &EditorFileSystemDirectory::get_file_count); ClassDB::bind_method(D_METHOD("get_file", "idx"), &EditorFileSystemDirectory::get_file); ClassDB::bind_method(D_METHOD("get_file_path", "idx"), &EditorFileSystemDirectory::get_file_path); ClassDB::bind_method(D_METHOD("get_file_type", "idx"), &EditorFileSystemDirectory::get_file_type); ClassDB::bind_method(D_METHOD("get_file_script_class_name", "idx"), &EditorFileSystemDirectory::get_file_script_class_name); ClassDB::bind_method(D_METHOD("get_file_script_class_extends", "idx"), &EditorFileSystemDirectory::get_file_script_class_extends); ClassDB::bind_method(D_METHOD("get_file_import_is_valid", "idx"), &EditorFileSystemDirectory::get_file_import_is_valid); ClassDB::bind_method(D_METHOD("get_name"), &EditorFileSystemDirectory::get_name); ClassDB::bind_method(D_METHOD("get_path"), &EditorFileSystemDirectory::get_path); ClassDB::bind_method(D_METHOD("get_parent"), &EditorFileSystemDirectory::get_parent); ClassDB::bind_method(D_METHOD("find_file_index", "name"), &EditorFileSystemDirectory::find_file_index); ClassDB::bind_method(D_METHOD("find_dir_index", "name"), &EditorFileSystemDirectory::find_dir_index); } EditorFileSystemDirectory::EditorFileSystemDirectory() { modified_time = 0; parent = nullptr; } EditorFileSystemDirectory::~EditorFileSystemDirectory() { for (EditorFileSystemDirectory::FileInfo *fi : files) { memdelete(fi); } for (EditorFileSystemDirectory *dir : subdirs) { memdelete(dir); } } EditorFileSystem::ScannedDirectory::~ScannedDirectory() { for (ScannedDirectory *dir : subdirs) { memdelete(dir); } } void EditorFileSystem::_load_first_scan_root_dir() { Ref<DirAccess> d = DirAccess::create(DirAccess::ACCESS_RESOURCES); first_scan_root_dir = memnew(ScannedDirectory); first_scan_root_dir->full_path = "res://"; nb_files_total = _scan_new_dir(first_scan_root_dir, d); } void EditorFileSystem::scan_for_uid() { // Load file structure into memory. _load_first_scan_root_dir(); // Load extensions for which an .import should exists. List<String> extensionsl; HashSet<String> import_extensions; ResourceFormatImporter::get_singleton()->get_recognized_extensions(&extensionsl); for (const String &E : extensionsl) { import_extensions.insert(E); } // Scan the file system to load uid. _scan_for_uid_directory(first_scan_root_dir, import_extensions); // It's done, resetting the callback method to prevent a second scan. ResourceUID::scan_for_uid_on_startup = nullptr; } void EditorFileSystem::_scan_for_uid_directory(const ScannedDirectory *p_scan_dir, const HashSet<String> &p_import_extensions) { for (ScannedDirectory *scan_sub_dir : p_scan_dir->subdirs) { _scan_for_uid_directory(scan_sub_dir, p_import_extensions); } for (const String &scan_file : p_scan_dir->files) { const String ext = scan_file.get_extension().to_lower(); if (ext == "uid" || ext == "import") { continue; } const String path = p_scan_dir->full_path.path_join(scan_file); ResourceUID::ID uid = ResourceUID::INVALID_ID; if (p_import_extensions.has(ext)) { if (FileAccess::exists(path + ".import")) { uid = ResourceFormatImporter::get_singleton()->get_resource_uid(path); } } else { uid = ResourceLoader::get_resource_uid(path); } if (uid != ResourceUID::INVALID_ID) { if (!ResourceUID::get_singleton()->has_id(uid)) { ResourceUID::get_singleton()->add_id(uid, path); } } } } void EditorFileSystem::_first_scan_filesystem() { EditorProgress ep = EditorProgress("first_scan_filesystem", TTR("Project initialization"), 5); HashSet<String> existing_class_names; HashSet<String> extensions; if (!first_scan_root_dir) { ep.step(TTR("Scanning file structure..."), 0, true); _load_first_scan_root_dir(); } // Preloading GDExtensions file extensions to prevent looping on all the resource loaders // for each files in _first_scan_process_scripts. List<String> gdextension_extensions; ResourceLoader::get_recognized_extensions_for_type("GDExtension", &gdextension_extensions); // This loads the global class names from the scripts and ensures that even if the // global_script_class_cache.cfg was missing or invalid, the global class names are valid in ScriptServer. // At the same time, to prevent looping multiple times in all files, it looks for extensions. ep.step(TTR("Loading global class names..."), 1, true); _first_scan_process_scripts(first_scan_root_dir, gdextension_extensions, existing_class_names, extensions); // Removing invalid global class to prevent having invalid paths in ScriptServer. bool save_scripts = _remove_invalid_global_class_names(existing_class_names); // If a global class is found or removed, we sync global_script_class_cache.cfg with the ScriptServer if (!existing_class_names.is_empty() || save_scripts) { EditorNode::get_editor_data().script_class_save_global_classes(); } // Processing extensions to add new extensions or remove invalid ones. // Important to do it in the first scan so custom types, new class names, custom importers, etc... // from extensions are ready to go before plugins, autoloads and resources validation/importation. // At this point, a restart of the editor should not be needed so we don't use the return value. ep.step(TTR("Verifying GDExtensions..."), 2, true); GDExtensionManager::get_singleton()->ensure_extensions_loaded(extensions); // Now that all the global class names should be loaded, create autoloads and plugins. // This is done after loading the global class names because autoloads and plugins can use // global class names. ep.step(TTR("Creating autoload scripts..."), 3, true); ProjectSettingsEditor::get_singleton()->init_autoloads(); ep.step(TTR("Initializing plugins..."), 4, true); EditorNode::get_singleton()->init_plugins(); ep.step(TTR("Starting file scan..."), 5, true); } void EditorFileSystem::_first_scan_process_scripts(const ScannedDirectory *p_scan_dir, List<String> &p_gdextension_extensions, HashSet<String> &p_existing_class_names, HashSet<String> &p_extensions) { for (ScannedDirectory *scan_sub_dir : p_scan_dir->subdirs) { _first_scan_process_scripts(scan_sub_dir, p_gdextension_extensions, p_existing_class_names, p_extensions); } for (const String &scan_file : p_scan_dir->files) { // Optimization to skip the ResourceLoader::get_resource_type for files // that are not scripts. Some loader get_resource_type methods read the file // which can be very slow on large projects. const String ext = scan_file.get_extension().to_lower(); bool is_script = false; for (int i = 0; i < ScriptServer::get_language_count(); i++) { if (ScriptServer::get_language(i)->get_extension() == ext) { is_script = true; break; } } if (is_script) { const String path = p_scan_dir->full_path.path_join(scan_file); const String type = ResourceLoader::get_resource_type(path); if (ClassDB::is_parent_class(type, SNAME("Script"))) { const ScriptClassInfo &info = _get_global_script_class(type, path); ScriptClassInfoUpdate update(info); update.type = type; _register_global_class_script(path, path, update); if (!info.name.is_empty()) { p_existing_class_names.insert(info.name); } } } // Check for GDExtensions. if (p_gdextension_extensions.find(ext)) { const String path = p_scan_dir->full_path.path_join(scan_file); const String type = ResourceLoader::get_resource_type(path); if (type == SNAME("GDExtension")) { p_extensions.insert(path); } } } } void EditorFileSystem::_scan_filesystem() { // On the first scan, the first_scan_root_dir is created in _first_scan_filesystem. ERR_FAIL_COND(!scanning || new_filesystem || (first_scan && !first_scan_root_dir)); //read .fscache String cpath; sources_changed.clear(); file_cache.clear(); String project = ProjectSettings::get_singleton()->get_resource_path(); String fscache = EditorPaths::get_singleton()->get_project_settings_dir().path_join(CACHE_FILE_NAME); { Ref<FileAccess> f = FileAccess::open(fscache, FileAccess::READ); bool first = true; if (f.is_valid()) { //read the disk cache while (!f->eof_reached()) { String l = f->get_line().strip_edges(); if (first) { if (first_scan) { // only use this on first scan, afterwards it gets ignored // this is so on first reimport we synchronize versions, then // we don't care until editor restart. This is for usability mainly so // your workflow is not killed after changing a setting by forceful reimporting // everything there is. filesystem_settings_version_for_import = l.strip_edges(); if (filesystem_settings_version_for_import != ResourceFormatImporter::get_singleton()->get_import_settings_hash()) { revalidate_import_files = true; } } first = false; continue; } if (l.is_empty()) { continue; } if (l.begins_with("::")) { Vector<String> split = l.split("::"); ERR_CONTINUE(split.size() != 3); const String &name = split[1]; cpath = name; } else { // The last section (deps) may contain the same splitter, so limit the maxsplit to 8 to get the complete deps. Vector<String> split = l.split("::", true, 8); ERR_CONTINUE(split.size() < 9); String name = split[0]; String file; file = name; name = cpath.path_join(name); FileCache fc; fc.type = split[1].get_slice("/", 0); fc.resource_script_class = split[1].get_slice("/", 1); fc.uid = split[2].to_int(); fc.modification_time = split[3].to_int(); fc.import_modification_time = split[4].to_int(); fc.import_valid = split[5].to_int() != 0; fc.import_group_file = split[6].strip_edges(); { const Vector<String> &slices = split[7].split("<>"); ERR_CONTINUE(slices.size() < 7); fc.class_info.name = slices[0]; fc.class_info.extends = slices[1]; fc.class_info.icon_path = slices[2]; fc.class_info.is_abstract = slices[3].to_int(); fc.class_info.is_tool = slices[4].to_int(); fc.import_md5 = slices[5]; fc.import_dest_paths = slices[6].split("<*>"); } fc.deps = split[8].strip_edges().split("<>"); file_cache[name] = fc; } } } } const String update_cache = EditorPaths::get_singleton()->get_project_settings_dir().path_join("filesystem_update4"); if (first_scan && FileAccess::exists(update_cache)) { { Ref<FileAccess> f2 = FileAccess::open(update_cache, FileAccess::READ); String l = f2->get_line().strip_edges(); while (!l.is_empty()) { dep_update_list.insert(l); file_cache.erase(l); // Erase cache for this, so it gets updated. l = f2->get_line().strip_edges(); } } Ref<DirAccess> d = DirAccess::create(DirAccess::ACCESS_RESOURCES); d->remove(update_cache); // Bye bye update cache. } EditorProgressBG scan_progress("efs", "ScanFS", 1000); ScanProgress sp; sp.hi = nb_files_total; sp.progress = &scan_progress; new_filesystem = memnew(EditorFileSystemDirectory); new_filesystem->parent = nullptr; ScannedDirectory *sd; HashSet<String> *processed_files = nullptr; // On the first scan, the first_scan_root_dir is created in _first_scan_filesystem. if (first_scan) { sd = first_scan_root_dir; // Will be updated on scan. ResourceUID::get_singleton()->clear(); ResourceUID::scan_for_uid_on_startup = nullptr; processed_files = memnew(HashSet<String>()); } else { Ref<DirAccess> d = DirAccess::create(DirAccess::ACCESS_RESOURCES); sd = memnew(ScannedDirectory); sd->full_path = "res://"; nb_files_total = _scan_new_dir(sd, d); } _process_file_system(sd, new_filesystem, sp, processed_files); if (first_scan) { _process_removed_files(*processed_files); } dep_update_list.clear(); file_cache.clear(); //clear caches, no longer needed if (first_scan) { memdelete(first_scan_root_dir); first_scan_root_dir = nullptr; memdelete(processed_files); } else { //on the first scan this is done from the main thread after re-importing _save_filesystem_cache(); } scanning = false; } void EditorFileSystem::_save_filesystem_cache() { group_file_cache.clear(); String fscache = EditorPaths::get_singleton()->get_project_settings_dir().path_join(CACHE_FILE_NAME); Ref<FileAccess> f = FileAccess::open(fscache, FileAccess::WRITE); ERR_FAIL_COND_MSG(f.is_null(), "Cannot create file '" + fscache + "'. Check user write permissions."); f->store_line(filesystem_settings_version_for_import); _save_filesystem_cache(filesystem, f); } void EditorFileSystem::_thread_func(void *_userdata) { EditorFileSystem *sd = (EditorFileSystem *)_userdata; sd->_scan_filesystem(); } bool EditorFileSystem::_is_test_for_reimport_needed(const String &p_path, uint64_t p_last_modification_time, uint64_t p_modification_time, uint64_t p_last_import_modification_time, uint64_t p_import_modification_time, const Vector<String> &p_import_dest_paths) { // The idea here is to trust the cache. If the last modification times in the cache correspond // to the last modification times of the files on disk, it means the files have not changed since // the last import, and the files in .godot/imported (p_import_dest_paths) should all be valid. if (p_last_modification_time != p_modification_time) { return true; } if (p_last_import_modification_time != p_import_modification_time) { return true; } if (reimport_on_missing_imported_files) { for (const String &path : p_import_dest_paths) { if (!FileAccess::exists(path)) { return true; } } } return false; } bool EditorFileSystem::_test_for_reimport(const String &p_path, const String &p_expected_import_md5) { if (p_expected_import_md5.is_empty()) { // Marked as reimportation needed. return true; } String new_md5 = FileAccess::get_md5(p_path + ".import"); if (p_expected_import_md5 != new_md5) { return true; } Error err; Ref<FileAccess> f = FileAccess::open(p_path + ".import", FileAccess::READ, &err); if (f.is_null()) { // No import file, reimport. return true; } VariantParser::StreamFile stream; stream.f = f; String assign; Variant value; VariantParser::Tag next_tag; int lines = 0; String error_text; Vector<String> to_check; String importer_name; String source_file = ""; String source_md5 = ""; Vector<String> dest_files; String dest_md5 = ""; int version = 0; bool found_uid = false; Variant meta; while (true) { assign = Variant(); next_tag.fields.clear(); next_tag.name = String(); err = VariantParser::parse_tag_assign_eof(&stream, lines, error_text, next_tag, assign, value, nullptr, true); if (err == ERR_FILE_EOF) { break; } else if (err != OK) { ERR_PRINT("ResourceFormatImporter::load - '" + p_path + ".import:" + itos(lines) + "' error '" + error_text + "'."); // Parse error, skip and let user attempt manual reimport to avoid reimport loop. return false; } if (!assign.is_empty()) { if (assign == "valid" && value.operator bool() == false) { // Invalid import (failed previous import), skip and let user attempt manual reimport to avoid reimport loop. return false; } if (assign.begins_with("path")) { to_check.push_back(value); } else if (assign == "files") { Array fa = value; for (const Variant &check_path : fa) { to_check.push_back(check_path); } } else if (assign == "importer_version") { version = value; } else if (assign == "importer") { importer_name = value; } else if (assign == "uid") { found_uid = true; } else if (assign == "source_file") { source_file = value; } else if (assign == "dest_files") { dest_files = value; } else if (assign == "metadata") { meta = value; } } else if (next_tag.name != "remap" && next_tag.name != "deps") { break; } } if (importer_name == "keep" || importer_name == "skip") { return false; // Keep mode, do not reimport. } if (!found_uid) { return true; // UID not found, old format, reimport. } // Imported files are gone, reimport. for (const String &E : to_check) { if (!FileAccess::exists(E)) { return true; } } Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name); if (importer.is_null()) { return true; // The importer has possibly changed, try to reimport. } if (importer->get_format_version() > version) { return true; // Version changed, reimport. } if (!importer->are_import_settings_valid(p_path, meta)) { // Reimport settings are out of sync with project settings, reimport. return true; } // Read the md5's from a separate file (so the import parameters aren't dependent on the file version). String base_path = ResourceFormatImporter::get_singleton()->get_import_base_path(p_path); Ref<FileAccess> md5s = FileAccess::open(base_path + ".md5", FileAccess::READ, &err); if (md5s.is_null()) { // No md5's stored for this resource. return true; } VariantParser::StreamFile md5_stream; md5_stream.f = md5s; while (true) { assign = Variant(); next_tag.fields.clear(); next_tag.name = String(); err = VariantParser::parse_tag_assign_eof(&md5_stream, lines, error_text, next_tag, assign, value, nullptr, true); if (err == ERR_FILE_EOF) { break; } else if (err != OK) { ERR_PRINT("ResourceFormatImporter::load - '" + p_path + ".import.md5:" + itos(lines) + "' error '" + error_text + "'."); return false; // Parse error. } if (!assign.is_empty()) { if (assign == "source_md5") { source_md5 = value; } else if (assign == "dest_md5") { dest_md5 = value; } } } // Check source md5 matching. if (!source_file.is_empty() && source_file != p_path) { return true; // File was moved, reimport. } if (source_md5.is_empty()) { return true; // Lacks md5, so just reimport. } String md5 = FileAccess::get_md5(p_path); if (md5 != source_md5) { return true; } if (!dest_files.is_empty() && !dest_md5.is_empty()) { md5 = FileAccess::get_multiple_md5(dest_files); if (md5 != dest_md5) { return true; } } return false; // Nothing changed. } Vector<String> EditorFileSystem::_get_import_dest_paths(const String &p_path) { Error err; Ref<FileAccess> f = FileAccess::open(p_path + ".import", FileAccess::READ, &err); if (f.is_null()) { // No import file, reimport. return Vector<String>(); } VariantParser::StreamFile stream; stream.f = f; String assign; Variant value; VariantParser::Tag next_tag; int lines = 0; String error_text; Vector<String> dest_paths; String importer_name; while (true) { assign = Variant(); next_tag.fields.clear(); next_tag.name = String(); err = VariantParser::parse_tag_assign_eof(&stream, lines, error_text, next_tag, assign, value, nullptr, true); if (err == ERR_FILE_EOF) { break; } else if (err != OK) { ERR_PRINT("ResourceFormatImporter::load - '" + p_path + ".import:" + itos(lines) + "' error '" + error_text + "'."); // Parse error, skip and let user attempt manual reimport to avoid reimport loop. return Vector<String>(); } if (!assign.is_empty()) { if (assign == "valid" && value.operator bool() == false) { // Invalid import (failed previous import), skip and let user attempt manual reimport to avoid reimport loop. return Vector<String>(); } if (assign.begins_with("path")) { dest_paths.push_back(value); } else if (assign == "files") { Array fa = value; for (const Variant &dest_path : fa) { dest_paths.push_back(dest_path); } } else if (assign == "importer") { importer_name = value; } } else if (next_tag.name != "remap" && next_tag.name != "deps") { break; } } if (importer_name == "keep" || importer_name == "skip") { return Vector<String>(); } return dest_paths; } bool EditorFileSystem::_scan_import_support(const Vector<String> &reimports) { if (import_support_queries.size() == 0) { return false; } HashMap<String, int> import_support_test; Vector<bool> import_support_tested; import_support_tested.resize(import_support_queries.size()); for (int i = 0; i < import_support_queries.size(); i++) { import_support_tested.write[i] = false; if (import_support_queries[i]->is_active()) { Vector<String> extensions = import_support_queries[i]->get_file_extensions(); for (int j = 0; j < extensions.size(); j++) { import_support_test.insert(extensions[j], i); } } } if (import_support_test.size() == 0) { return false; //well nothing to do } for (int i = 0; i < reimports.size(); i++) { HashMap<String, int>::Iterator E = import_support_test.find(reimports[i].get_extension().to_lower()); if (E) { import_support_tested.write[E->value] = true; } } for (int i = 0; i < import_support_tested.size(); i++) { if (import_support_tested[i]) { if (import_support_queries.write[i]->query()) { return true; } } } return false; } bool EditorFileSystem::_update_scan_actions() { sources_changed.clear(); // We need to update the script global class names before the reimports to be sure that // all the importer classes that depends on class names will work. _update_script_classes(); bool fs_changed = false; Vector<String> reimports; Vector<String> reloads; EditorProgress *ep = nullptr; if (scan_actions.size() > 1) { ep = memnew(EditorProgress("_update_scan_actions", TTR("Scanning actions..."), scan_actions.size())); } int step_count = 0; for (const ItemAction &ia : scan_actions) { switch (ia.action) { case ItemAction::ACTION_NONE: { } break; case ItemAction::ACTION_DIR_ADD: { int idx = 0; for (int i = 0; i < ia.dir->subdirs.size(); i++) { if (ia.new_dir->name.filenocasecmp_to(ia.dir->subdirs[i]->name) < 0) { break; } idx++; } if (idx == ia.dir->subdirs.size()) { ia.dir->subdirs.push_back(ia.new_dir); } else { ia.dir->subdirs.insert(idx, ia.new_dir); } fs_changed = true; } break; case ItemAction::ACTION_DIR_REMOVE: { ERR_CONTINUE(!ia.dir->parent); ia.dir->parent->subdirs.erase(ia.dir); memdelete(ia.dir); fs_changed = true; } break; case ItemAction::ACTION_FILE_ADD: { int idx = 0; for (int i = 0; i < ia.dir->files.size(); i++) { if (ia.new_file->file.filenocasecmp_to(ia.dir->files[i]->file) < 0) { break; } idx++; } if (idx == ia.dir->files.size()) { ia.dir->files.push_back(ia.new_file); } else { ia.dir->files.insert(idx, ia.new_file); } fs_changed = true; const String new_file_path = ia.dir->get_file_path(idx); const ResourceUID::ID existing_id = ResourceLoader::get_resource_uid(new_file_path); if (existing_id != ResourceUID::INVALID_ID) { const String old_path = ResourceUID::get_singleton()->get_id_path(existing_id); if (old_path != new_file_path && FileAccess::exists(old_path)) { const ResourceUID::ID new_id = ResourceUID::get_singleton()->create_id(); ResourceUID::get_singleton()->add_id(new_id, new_file_path); ResourceSaver::set_uid(new_file_path, new_id); WARN_PRINT(vformat("Duplicate UID detected for Resource at \"%s\".\nOld Resource path: \"%s\". The new file UID was changed automatically.", new_file_path, old_path)); } else { // Re-assign the UID to file, just in case it was pulled from cache. ResourceSaver::set_uid(new_file_path, existing_id); } } if (ClassDB::is_parent_class(ia.new_file->type, SNAME("Script"))) { _queue_update_script_class(new_file_path, ScriptClassInfoUpdate::from_file_info(ia.new_file)); } if (ia.new_file->type == SNAME("PackedScene")) { _queue_update_scene_groups(new_file_path); } } break; case ItemAction::ACTION_FILE_REMOVE: { int idx = ia.dir->find_file_index(ia.file); ERR_CONTINUE(idx == -1); String class_name = ia.dir->files[idx]->class_info.name; if (ClassDB::is_parent_class(ia.dir->files[idx]->type, SNAME("Script"))) { _queue_update_script_class(ia.dir->get_file_path(idx), ScriptClassInfoUpdate()); } if (ia.dir->files[idx]->type == SNAME("PackedScene")) { _queue_update_scene_groups(ia.dir->get_file_path(idx)); } _delete_internal_files(ia.dir->files[idx]->file); memdelete(ia.dir->files[idx]); ia.dir->files.remove_at(idx); // Restore another script with the same global class name if it exists. if (!class_name.is_empty()) { EditorFileSystemDirectory::FileInfo *old_fi = nullptr; String old_file = _get_file_by_class_name(filesystem, class_name, old_fi); if (!old_file.is_empty() && old_fi) { _queue_update_script_class(old_file, ScriptClassInfoUpdate::from_file_info(old_fi)); } } fs_changed = true; } break; case ItemAction::ACTION_FILE_TEST_REIMPORT: { int idx = ia.dir->find_file_index(ia.file); ERR_CONTINUE(idx == -1); String full_path = ia.dir->get_file_path(idx); bool need_reimport = _test_for_reimport(full_path, ia.dir->files[idx]->import_md5); if (need_reimport) { // Must reimport. reimports.push_back(full_path); Vector<String> dependencies = _get_dependencies(full_path); for (const String &dep : dependencies) { const String &dependency_path = dep.contains("::") ? dep.get_slice("::", 0) : dep; if (import_extensions.has(dep.get_extension())) { reimports.push_back(dependency_path); } } } else { // Must not reimport, all was good. // Update modified times, md5 and destination paths, to avoid reimport. ia.dir->files[idx]->modified_time = FileAccess::get_modified_time(full_path); ia.dir->files[idx]->import_modified_time = FileAccess::get_modified_time(full_path + ".import"); if (ia.dir->files[idx]->import_md5.is_empty()) { ia.dir->files[idx]->import_md5 = FileAccess::get_md5(full_path + ".import"); } ia.dir->files[idx]->import_dest_paths = _get_import_dest_paths(full_path); } fs_changed = true; } break; case ItemAction::ACTION_FILE_RELOAD: { int idx = ia.dir->find_file_index(ia.file); ERR_CONTINUE(idx == -1); // Only reloads the resources that are already loaded. if (ResourceCache::has(ia.dir->get_file_path(idx))) { reloads.push_back(ia.dir->get_file_path(idx)); } } break; } if (ep) { ep->step(ia.file, step_count++, false); } } memdelete_notnull(ep); if (_scan_extensions()) { //needs editor restart //extensions also may provide filetypes to be imported, so they must run before importing if (EditorNode::immediate_confirmation_dialog(TTR("Some extensions need the editor to restart to take effect."), first_scan ? TTR("Restart") : TTR("Save & Restart"), TTR("Continue"))) { if (!first_scan) { EditorNode::get_singleton()->save_all_scenes(); } EditorNode::get_singleton()->restart_editor(); //do not import return true; } } if (!reimports.is_empty()) { if (_scan_import_support(reimports)) { return true; } reimport_files(reimports); } else { //reimport files will update the uid cache file so if nothing was reimported, update it manually ResourceUID::get_singleton()->update_cache(); } if (!reloads.is_empty()) { // Update global class names, dependencies, etc... update_files(reloads); } if (first_scan) { //only on first scan this is valid and updated, then settings changed. revalidate_import_files = false; filesystem_settings_version_for_import = ResourceFormatImporter::get_singleton()->get_import_settings_hash(); _save_filesystem_cache(); } // Moving the processing of pending updates before the resources_reload event to be sure all global class names // are updated. Script.cpp listens on resources_reload and reloads updated scripts. _process_update_pending(); if (reloads.size()) { emit_signal(SNAME("resources_reload"), reloads); } scan_actions.clear(); return fs_changed; } void EditorFileSystem::scan() { if (false /*&& bool(Globals::get_singleton()->get("debug/disable_scan"))*/) { return; } if (scanning || scanning_changes || thread.is_started()) { return; } // The first scan must be on the main thread because, after the first scan and update // of global class names, we load the plugins and autoloads. These need to // be added on the main thread because they are nodes, and we need to wait for them // to be loaded to continue the scan and reimportations. if (first_scan) { _first_scan_filesystem(); } _update_extensions(); if (!use_threads) { scanning = true; scan_total = 0; _scan_filesystem(); if (filesystem) { memdelete(filesystem); } //file_type_cache.clear(); filesystem = new_filesystem; new_filesystem = nullptr; _update_scan_actions(); // Update all icons so they are loaded for the FileSystemDock. _update_files_icon_path(); scanning = false; // Set first_scan to false before the signals so the function doing_first_scan can return false // in editor_node to start the export if needed. first_scan = false; ResourceImporter::load_on_startup = nullptr; emit_signal(SNAME("filesystem_changed")); emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); } else { ERR_FAIL_COND(thread.is_started()); set_process(true); Thread::Settings s; scanning = true; scan_total = 0; s.priority = Thread::PRIORITY_LOW; thread.start(_thread_func, this, s); } } void EditorFileSystem::ScanProgress::increment() { current++; float ratio = current / MAX(hi, 1.0f); if (progress) { progress->step(ratio * 1000.0f); } EditorFileSystem::singleton->scan_total = ratio; } int EditorFileSystem::_scan_new_dir(ScannedDirectory *p_dir, Ref<DirAccess> &da) { List<String> dirs; List<String> files; String cd = da->get_current_dir(); da->list_dir_begin(); while (true) { String f = da->get_next(); if (f.is_empty()) { break; } if (da->current_is_hidden()) { continue; } if (da->current_is_dir()) { if (f.begins_with(".")) { // Ignore special and . / .. continue; } if (_should_skip_directory(cd.path_join(f))) { continue; } dirs.push_back(f); } else { files.push_back(f); } } da->list_dir_end(); dirs.sort_custom<FileNoCaseComparator>(); files.sort_custom<FileNoCaseComparator>(); int nb_files_total_scan = 0; for (List<String>::Element *E = dirs.front(); E; E = E->next()) { if (da->change_dir(E->get()) == OK) { String d = da->get_current_dir(); if (d == cd || !d.begins_with(cd)) { da->change_dir(cd); //avoid recursion } else { ScannedDirectory *sd = memnew(ScannedDirectory); sd->name = E->get(); sd->full_path = p_dir->full_path.path_join(sd->name); nb_files_total_scan += _scan_new_dir(sd, da); p_dir->subdirs.push_back(sd); da->change_dir(".."); } } else { ERR_PRINT("Cannot go into subdir '" + E->get() + "'."); } } p_dir->files = files; nb_files_total_scan += files.size(); return nb_files_total_scan; } void EditorFileSystem::_process_file_system(const ScannedDirectory *p_scan_dir, EditorFileSystemDirectory *p_dir, ScanProgress &p_progress, HashSet<String> *r_processed_files) { p_dir->modified_time = FileAccess::get_modified_time(p_scan_dir->full_path); for (ScannedDirectory *scan_sub_dir : p_scan_dir->subdirs) { EditorFileSystemDirectory *sub_dir = memnew(EditorFileSystemDirectory); sub_dir->parent = p_dir; sub_dir->name = scan_sub_dir->name; p_dir->subdirs.push_back(sub_dir); _process_file_system(scan_sub_dir, sub_dir, p_progress, r_processed_files); } for (const String &scan_file : p_scan_dir->files) { String ext = scan_file.get_extension().to_lower(); if (!valid_extensions.has(ext)) { p_progress.increment(); continue; //invalid } String path = p_scan_dir->full_path.path_join(scan_file); EditorFileSystemDirectory::FileInfo *fi = memnew(EditorFileSystemDirectory::FileInfo); fi->file = scan_file; p_dir->files.push_back(fi); if (r_processed_files) { r_processed_files->insert(path); } FileCache *fc = file_cache.getptr(path); uint64_t mt = FileAccess::get_modified_time(path); if (import_extensions.has(ext)) { //is imported uint64_t import_mt = FileAccess::get_modified_time(path + ".import"); if (fc) { fi->type = fc->type; fi->resource_script_class = fc->resource_script_class; fi->uid = fc->uid; fi->deps = fc->deps; fi->modified_time = mt; fi->import_modified_time = import_mt; fi->import_md5 = fc->import_md5; fi->import_dest_paths = fc->import_dest_paths; fi->import_valid = fc->import_valid; fi->import_group_file = fc->import_group_file; fi->class_info = fc->class_info; // Ensures backward compatibility when the project is loaded for the first time with the added import_md5 // and import_dest_paths properties in the file cache. if (fc->import_md5.is_empty()) { fi->import_md5 = FileAccess::get_md5(path + ".import"); fi->import_dest_paths = _get_import_dest_paths(path); } // The method _is_test_for_reimport_needed checks if the files were modified and ensures that // all the destination files still exist without reading the .import file. // If something is different, we will queue a test for reimportation that will check // the md5 of all files and import settings and, if necessary, execute a reimportation. if (_is_test_for_reimport_needed(path, fc->modification_time, mt, fc->import_modification_time, import_mt, fi->import_dest_paths) || (revalidate_import_files && !ResourceFormatImporter::get_singleton()->are_import_settings_valid(path))) { ItemAction ia; ia.action = ItemAction::ACTION_FILE_TEST_REIMPORT; ia.dir = p_dir; ia.file = fi->file; scan_actions.push_back(ia); } if (fc->type.is_empty()) { fi->type = ResourceLoader::get_resource_type(path); fi->resource_script_class = ResourceLoader::get_resource_script_class(path); fi->import_group_file = ResourceLoader::get_import_group_file(path); //there is also the chance that file type changed due to reimport, must probably check this somehow here (or kind of note it for next time in another file?) //note: I think this should not happen any longer.. } if (fc->uid == ResourceUID::INVALID_ID) { // imported files should always have a UID, so attempt to fetch it. fi->uid = ResourceLoader::get_resource_uid(path); } } else { // Using get_resource_import_info() to prevent calling 3 times ResourceFormatImporter::_get_path_and_type. ResourceFormatImporter::get_singleton()->get_resource_import_info(path, fi->type, fi->uid, fi->import_group_file); fi->class_info = _get_global_script_class(fi->type, path); fi->modified_time = 0; fi->import_modified_time = 0; fi->import_md5 = ""; fi->import_dest_paths = Vector<String>(); fi->import_valid = (fi->type == "TextFile" || fi->type == "OtherFile") ? true : ResourceLoader::is_import_valid(path); ItemAction ia; ia.action = ItemAction::ACTION_FILE_TEST_REIMPORT; ia.dir = p_dir; ia.file = fi->file; scan_actions.push_back(ia); } } else { if (fc && fc->modification_time == mt) { //not imported, so just update type if changed fi->type = fc->type; fi->resource_script_class = fc->resource_script_class; fi->uid = fc->uid; fi->modified_time = mt; fi->deps = fc->deps; fi->import_modified_time = 0; fi->import_md5 = ""; fi->import_dest_paths = Vector<String>(); fi->import_valid = true; fi->class_info = fc->class_info; if (first_scan && ClassDB::is_parent_class(fi->type, SNAME("Script"))) { bool update_script = false; String old_class_name = fi->class_info.name; fi->class_info = _get_global_script_class(fi->type, path); if (old_class_name != fi->class_info.name) { update_script = true; } else if (!fi->class_info.name.is_empty() && (!ScriptServer::is_global_class(fi->class_info.name) || ScriptServer::get_global_class_path(fi->class_info.name) != path)) { // This script has a class name but is not in the global class names or the path of the class has changed. update_script = true; } if (update_script) { _queue_update_script_class(path, ScriptClassInfoUpdate::from_file_info(fi)); } } } else { //new or modified time fi->type = ResourceLoader::get_resource_type(path); fi->resource_script_class = ResourceLoader::get_resource_script_class(path); if (fi->type == "" && textfile_extensions.has(ext)) { fi->type = "TextFile"; } if (fi->type == "" && other_file_extensions.has(ext)) { fi->type = "OtherFile"; } fi->uid = ResourceLoader::get_resource_uid(path); fi->class_info = _get_global_script_class(fi->type, path); fi->deps = _get_dependencies(path); fi->modified_time = mt; fi->import_modified_time = 0; fi->import_md5 = ""; fi->import_dest_paths = Vector<String>(); fi->import_valid = true; // Files in dep_update_list are forced for rescan to update dependencies. They don't need other updates. if (!dep_update_list.has(path)) { if (ClassDB::is_parent_class(fi->type, SNAME("Script"))) { _queue_update_script_class(path, ScriptClassInfoUpdate::from_file_info(fi)); } if (fi->type == SNAME("PackedScene")) { _queue_update_scene_groups(path); } } } if (ResourceLoader::should_create_uid_file(path)) { // Create a UID file and new UID, if it's invalid. Ref<FileAccess> f = FileAccess::open(path + ".uid", FileAccess::WRITE); if (f.is_valid()) { if (fi->uid == ResourceUID::INVALID_ID) { fi->uid = ResourceUID::get_singleton()->create_id(); } else { WARN_PRINT(vformat("Missing .uid file for path \"%s\". The file was re-created from cache.", path)); } f->store_line(ResourceUID::get_singleton()->id_to_text(fi->uid)); } } } if (fi->uid != ResourceUID::INVALID_ID) { if (ResourceUID::get_singleton()->has_id(fi->uid)) { // Restrict UID dupe warning to first-scan since we know there are no file moves going on yet. if (first_scan) { // Warn if we detect files with duplicate UIDs. const String other_path = ResourceUID::get_singleton()->get_id_path(fi->uid); if (other_path != path) { WARN_PRINT(vformat("UID duplicate detected between %s and %s.", path, other_path)); } } ResourceUID::get_singleton()->set_id(fi->uid, path); } else { ResourceUID::get_singleton()->add_id(fi->uid, path); } } p_progress.increment(); } } void EditorFileSystem::_process_removed_files(const HashSet<String> &p_processed_files) { for (const KeyValue<String, EditorFileSystem::FileCache> &kv : file_cache) { if (!p_processed_files.has(kv.key)) { if (ClassDB::is_parent_class(kv.value.type, SNAME("Script"))) { // A script has been removed from disk since the last startup. The documentation needs to be updated. // There's no need to add the path in update_script_paths since that is exclusively for updating global class names, // which is handled in _first_scan_filesystem before the full scan to ensure plugins and autoloads can be created. MutexLock update_script_lock(update_script_mutex); update_script_paths_documentation.insert(kv.key); } } } } void EditorFileSystem::_scan_fs_changes(EditorFileSystemDirectory *p_dir, ScanProgress &p_progress, bool p_recursive) { uint64_t current_mtime = FileAccess::get_modified_time(p_dir->get_path()); bool updated_dir = false; String cd = p_dir->get_path(); int diff_nb_files = 0; if (current_mtime != p_dir->modified_time || using_fat32_or_exfat) { updated_dir = true; p_dir->modified_time = current_mtime; //ooooops, dir changed, see what's going on //first mark everything as verified for (int i = 0; i < p_dir->files.size(); i++) { p_dir->files[i]->verified = false; } for (int i = 0; i < p_dir->subdirs.size(); i++) { p_dir->get_subdir(i)->verified = false; } diff_nb_files -= p_dir->files.size(); //then scan files and directories and check what's different Ref<DirAccess> da = DirAccess::create(DirAccess::ACCESS_RESOURCES); Error ret = da->change_dir(cd); ERR_FAIL_COND_MSG(ret != OK, "Cannot change to '" + cd + "' folder."); da->list_dir_begin(); while (true) { String f = da->get_next(); if (f.is_empty()) { break; } if (da->current_is_hidden()) { continue; } if (da->current_is_dir()) { if (f.begins_with(".")) { // Ignore special and . / .. continue; } int idx = p_dir->find_dir_index(f); if (idx == -1) { String dir_path = cd.path_join(f); if (_should_skip_directory(dir_path)) { continue; } ScannedDirectory sd; sd.name = f; sd.full_path = dir_path; EditorFileSystemDirectory *efd = memnew(EditorFileSystemDirectory); efd->parent = p_dir; efd->name = f; Ref<DirAccess> d = DirAccess::create(DirAccess::ACCESS_RESOURCES); d->change_dir(dir_path); int nb_files_dir = _scan_new_dir(&sd, d); p_progress.hi += nb_files_dir; diff_nb_files += nb_files_dir; _process_file_system(&sd, efd, p_progress, nullptr); ItemAction ia; ia.action = ItemAction::ACTION_DIR_ADD; ia.dir = p_dir; ia.file = f; ia.new_dir = efd; scan_actions.push_back(ia); } else { p_dir->subdirs[idx]->verified = true; } } else { String ext = f.get_extension().to_lower(); if (!valid_extensions.has(ext)) { continue; //invalid } int idx = p_dir->find_file_index(f); if (idx == -1) { //never seen this file, add actition to add it EditorFileSystemDirectory::FileInfo *fi = memnew(EditorFileSystemDirectory::FileInfo); fi->file = f; String path = cd.path_join(fi->file); fi->modified_time = FileAccess::get_modified_time(path); fi->import_modified_time = 0; fi->import_md5 = ""; fi->import_dest_paths = Vector<String>(); fi->type = ResourceLoader::get_resource_type(path); fi->resource_script_class = ResourceLoader::get_resource_script_class(path); if (fi->type == "" && textfile_extensions.has(ext)) { fi->type = "TextFile"; } if (fi->type == "" && other_file_extensions.has(ext)) { fi->type = "OtherFile"; } fi->class_info = _get_global_script_class(fi->type, path); fi->import_valid = (fi->type == "TextFile" || fi->type == "OtherFile") ? true : ResourceLoader::is_import_valid(path); fi->import_group_file = ResourceLoader::get_import_group_file(path); { ItemAction ia; ia.action = ItemAction::ACTION_FILE_ADD; ia.dir = p_dir; ia.file = f; ia.new_file = fi; scan_actions.push_back(ia); } if (import_extensions.has(ext)) { //if it can be imported, and it was added, it needs to be reimported ItemAction ia; ia.action = ItemAction::ACTION_FILE_TEST_REIMPORT; ia.dir = p_dir; ia.file = f; scan_actions.push_back(ia); } diff_nb_files++; } else { p_dir->files[idx]->verified = true; } } } da->list_dir_end(); } for (int i = 0; i < p_dir->files.size(); i++) { if (updated_dir && !p_dir->files[i]->verified) { //this file was removed, add action to remove it ItemAction ia; ia.action = ItemAction::ACTION_FILE_REMOVE; ia.dir = p_dir; ia.file = p_dir->files[i]->file; scan_actions.push_back(ia); diff_nb_files--; continue; } String path = cd.path_join(p_dir->files[i]->file); if (import_extensions.has(p_dir->files[i]->file.get_extension().to_lower())) { // Check here if file must be imported or not. // Same logic as in _process_file_system, the last modifications dates // needs to be trusted to prevent reading all the .import files and the md5 // each time the user switch back to Godot. uint64_t mt = FileAccess::get_modified_time(path); uint64_t import_mt = FileAccess::get_modified_time(path + ".import"); if (_is_test_for_reimport_needed(path, p_dir->files[i]->modified_time, mt, p_dir->files[i]->import_modified_time, import_mt, p_dir->files[i]->import_dest_paths)) { ItemAction ia; ia.action = ItemAction::ACTION_FILE_TEST_REIMPORT; ia.dir = p_dir; ia.file = p_dir->files[i]->file; scan_actions.push_back(ia); } } else { uint64_t mt = FileAccess::get_modified_time(path); if (mt != p_dir->files[i]->modified_time) { p_dir->files[i]->modified_time = mt; //save new time, but test for reload ItemAction ia; ia.action = ItemAction::ACTION_FILE_RELOAD; ia.dir = p_dir; ia.file = p_dir->files[i]->file; scan_actions.push_back(ia); } } p_progress.increment(); } for (int i = 0; i < p_dir->subdirs.size(); i++) { if ((updated_dir && !p_dir->subdirs[i]->verified) || _should_skip_directory(p_dir->subdirs[i]->get_path())) { // Add all the files of the folder to be sure _update_scan_actions process the removed files // for global class names. diff_nb_files += _insert_actions_delete_files_directory(p_dir->subdirs[i]); //this directory was removed or ignored, add action to remove it ItemAction ia; ia.action = ItemAction::ACTION_DIR_REMOVE; ia.dir = p_dir->subdirs[i]; scan_actions.push_back(ia); continue; } if (p_recursive) { _scan_fs_changes(p_dir->get_subdir(i), p_progress); } } nb_files_total = MAX(nb_files_total + diff_nb_files, 0); } void EditorFileSystem::_delete_internal_files(const String &p_file) { if (FileAccess::exists(p_file + ".import")) { List<String> paths; ResourceFormatImporter::get_singleton()->get_internal_resource_path_list(p_file, &paths); Ref<DirAccess> da = DirAccess::create(DirAccess::ACCESS_RESOURCES); for (const String &E : paths) { da->remove(E); } da->remove(p_file + ".import"); } if (FileAccess::exists(p_file + ".uid")) { DirAccess::remove_absolute(p_file + ".uid"); } } int EditorFileSystem::_insert_actions_delete_files_directory(EditorFileSystemDirectory *p_dir) { int nb_files = 0; for (EditorFileSystemDirectory::FileInfo *fi : p_dir->files) { ItemAction ia; ia.action = ItemAction::ACTION_FILE_REMOVE; ia.dir = p_dir; ia.file = fi->file; scan_actions.push_back(ia); nb_files++; } for (EditorFileSystemDirectory *sub_dir : p_dir->subdirs) { nb_files += _insert_actions_delete_files_directory(sub_dir); } return nb_files; } void EditorFileSystem::_thread_func_sources(void *_userdata) { EditorFileSystem *efs = (EditorFileSystem *)_userdata; if (efs->filesystem) { EditorProgressBG pr("sources", TTR("ScanSources"), 1000); ScanProgress sp; sp.progress = ≺ sp.hi = efs->nb_files_total; efs->_scan_fs_changes(efs->filesystem, sp); } efs->scanning_changes_done.set(); } bool EditorFileSystem::_remove_invalid_global_class_names(const HashSet<String> &p_existing_class_names) { List<StringName> global_classes; bool must_save = false; ScriptServer::get_global_class_list(&global_classes); for (const StringName &class_name : global_classes) { if (!p_existing_class_names.has(class_name)) { ScriptServer::remove_global_class(class_name); must_save = true; } } return must_save; } String EditorFileSystem::_get_file_by_class_name(EditorFileSystemDirectory *p_dir, const String &p_class_name, EditorFileSystemDirectory::FileInfo *&r_file_info) { for (EditorFileSystemDirectory::FileInfo *fi : p_dir->files) { if (fi->class_info.name == p_class_name) { r_file_info = fi; return p_dir->get_path().path_join(fi->file); } } for (EditorFileSystemDirectory *sub_dir : p_dir->subdirs) { String file = _get_file_by_class_name(sub_dir, p_class_name, r_file_info); if (!file.is_empty()) { return file; } } r_file_info = nullptr; return ""; } void EditorFileSystem::scan_changes() { if (first_scan || // Prevent a premature changes scan from inhibiting the first full scan scanning || scanning_changes || thread.is_started()) { scan_changes_pending = true; set_process(true); return; } _update_extensions(); sources_changed.clear(); scanning_changes = true; scanning_changes_done.clear(); if (!use_threads) { if (filesystem) { EditorProgressBG pr("sources", TTR("ScanSources"), 1000); ScanProgress sp; sp.progress = ≺ sp.hi = nb_files_total; scan_total = 0; _scan_fs_changes(filesystem, sp); if (_update_scan_actions()) { emit_signal(SNAME("filesystem_changed")); } } scanning_changes = false; scanning_changes_done.set(); emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); } else { ERR_FAIL_COND(thread_sources.is_started()); set_process(true); scan_total = 0; Thread::Settings s; s.priority = Thread::PRIORITY_LOW; thread_sources.start(_thread_func_sources, this, s); } } void EditorFileSystem::_notification(int p_what) { switch (p_what) { case NOTIFICATION_EXIT_TREE: { Thread &active_thread = thread.is_started() ? thread : thread_sources; if (use_threads && active_thread.is_started()) { while (scanning) { OS::get_singleton()->delay_usec(1000); } active_thread.wait_to_finish(); WARN_PRINT("Scan thread aborted..."); set_process(false); } if (filesystem) { memdelete(filesystem); } if (new_filesystem) { memdelete(new_filesystem); } filesystem = nullptr; new_filesystem = nullptr; } break; case NOTIFICATION_PROCESS: { if (use_threads) { /** This hack exists because of the EditorProgress nature * of processing events recursively. This needs to be rewritten * at some point entirely, but in the meantime the following * hack prevents deadlock on import. */ static bool prevent_recursive_process_hack = false; if (prevent_recursive_process_hack) { break; } prevent_recursive_process_hack = true; bool done_importing = false; if (scanning_changes) { if (scanning_changes_done.is_set()) { set_process(false); if (thread_sources.is_started()) { thread_sources.wait_to_finish(); } bool changed = _update_scan_actions(); // Set first_scan to false before the signals so the function doing_first_scan can return false // in editor_node to start the export if needed. first_scan = false; ResourceImporter::load_on_startup = nullptr; if (changed) { emit_signal(SNAME("filesystem_changed")); } emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); scanning_changes = false; // Changed to false here to prevent recursive triggering of scan thread. done_importing = true; } } else if (!scanning && thread.is_started()) { set_process(false); if (filesystem) { memdelete(filesystem); } filesystem = new_filesystem; new_filesystem = nullptr; thread.wait_to_finish(); _update_scan_actions(); // Update all icons so they are loaded for the FileSystemDock. _update_files_icon_path(); // Set first_scan to false before the signals so the function doing_first_scan can return false // in editor_node to start the export if needed. first_scan = false; ResourceImporter::load_on_startup = nullptr; emit_signal(SNAME("filesystem_changed")); emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); } if (done_importing && scan_changes_pending) { scan_changes_pending = false; scan_changes(); } prevent_recursive_process_hack = false; } } break; } } bool EditorFileSystem::is_scanning() const { return scanning || scanning_changes || first_scan; } float EditorFileSystem::get_scanning_progress() const { return scan_total; } EditorFileSystemDirectory *EditorFileSystem::get_filesystem() { return filesystem; } void EditorFileSystem::_save_filesystem_cache(EditorFileSystemDirectory *p_dir, Ref<FileAccess> p_file) { if (!p_dir) { return; //none } p_file->store_line("::" + p_dir->get_path() + "::" + String::num_int64(p_dir->modified_time)); for (int i = 0; i < p_dir->files.size(); i++) { const EditorFileSystemDirectory::FileInfo *file_info = p_dir->files[i]; if (!file_info->import_group_file.is_empty()) { group_file_cache.insert(file_info->import_group_file); } String type = file_info->type; if (file_info->resource_script_class) { type += "/" + String(file_info->resource_script_class); } PackedStringArray cache_string; cache_string.append(file_info->file); cache_string.append(type); cache_string.append(itos(file_info->uid)); cache_string.append(itos(file_info->modified_time)); cache_string.append(itos(file_info->import_modified_time)); cache_string.append(itos(file_info->import_valid)); cache_string.append(file_info->import_group_file); cache_string.append(String("<>").join({ file_info->class_info.name, file_info->class_info.extends, file_info->class_info.icon_path, itos(file_info->class_info.is_abstract), itos(file_info->class_info.is_tool), file_info->import_md5, String("<*>").join(file_info->import_dest_paths) })); cache_string.append(String("<>").join(file_info->deps)); p_file->store_line(String("::").join(cache_string)); } for (int i = 0; i < p_dir->subdirs.size(); i++) { _save_filesystem_cache(p_dir->subdirs[i], p_file); } } bool EditorFileSystem::_find_file(const String &p_file, EditorFileSystemDirectory **r_d, int &r_file_pos) const { //todo make faster if (!filesystem || scanning) { return false; } String f = ProjectSettings::get_singleton()->localize_path(p_file); // Note: Only checks if base directory is case sensitive. Ref<DirAccess> dir = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); bool fs_case_sensitive = dir->is_case_sensitive("res://"); if (!f.begins_with("res://")) { return false; } f = f.substr(6, f.length()); f = f.replace("\\", "/"); Vector<String> path = f.split("/"); if (path.size() == 0) { return false; } String file = path[path.size() - 1]; path.resize(path.size() - 1); EditorFileSystemDirectory *fs = filesystem; for (int i = 0; i < path.size(); i++) { if (path[i].begins_with(".")) { return false; } int idx = -1; for (int j = 0; j < fs->get_subdir_count(); j++) { if (fs_case_sensitive) { if (fs->get_subdir(j)->get_name() == path[i]) { idx = j; break; } } else { if (fs->get_subdir(j)->get_name().to_lower() == path[i].to_lower()) { idx = j; break; } } } if (idx == -1) { // Only create a missing directory in memory when it exists on disk. if (!dir->dir_exists(fs->get_path().path_join(path[i]))) { return false; } EditorFileSystemDirectory *efsd = memnew(EditorFileSystemDirectory); efsd->name = path[i]; efsd->parent = fs; int idx2 = 0; for (int j = 0; j < fs->get_subdir_count(); j++) { if (efsd->name.filenocasecmp_to(fs->get_subdir(j)->get_name()) < 0) { break; } idx2++; } if (idx2 == fs->get_subdir_count()) { fs->subdirs.push_back(efsd); } else { fs->subdirs.insert(idx2, efsd); } fs = efsd; } else { fs = fs->get_subdir(idx); } } int cpos = -1; for (int i = 0; i < fs->files.size(); i++) { if (fs->files[i]->file == file) { cpos = i; break; } } r_file_pos = cpos; *r_d = fs; return cpos != -1; } String EditorFileSystem::get_file_type(const String &p_file) const { EditorFileSystemDirectory *fs = nullptr; int cpos = -1; if (!_find_file(p_file, &fs, cpos)) { return ""; } return fs->files[cpos]->type; } EditorFileSystemDirectory *EditorFileSystem::find_file(const String &p_file, int *r_index) const { if (!filesystem || scanning) { return nullptr; } EditorFileSystemDirectory *fs = nullptr; int cpos = -1; if (!_find_file(p_file, &fs, cpos)) { return nullptr; } if (r_index) { *r_index = cpos; } return fs; } ResourceUID::ID EditorFileSystem::get_file_uid(const String &p_path) const { int file_idx; EditorFileSystemDirectory *directory = find_file(p_path, &file_idx); if (!directory) { return ResourceUID::INVALID_ID; } return directory->files[file_idx]->uid; } EditorFileSystemDirectory *EditorFileSystem::get_filesystem_path(const String &p_path) { if (!filesystem || scanning) { return nullptr; } String f = ProjectSettings::get_singleton()->localize_path(p_path); if (!f.begins_with("res://")) { return nullptr; } f = f.substr(6, f.length()); f = f.replace("\\", "/"); if (f.is_empty()) { return filesystem; } if (f.ends_with("/")) { f = f.substr(0, f.length() - 1); } Vector<String> path = f.split("/"); if (path.size() == 0) { return nullptr; } EditorFileSystemDirectory *fs = filesystem; for (int i = 0; i < path.size(); i++) { int idx = -1; for (int j = 0; j < fs->get_subdir_count(); j++) { if (fs->get_subdir(j)->get_name() == path[i]) { idx = j; break; } } if (idx == -1) { return nullptr; } else { fs = fs->get_subdir(idx); } } return fs; } void EditorFileSystem::_save_late_updated_files() { //files that already existed, and were modified, need re-scanning for dependencies upon project restart. This is done via saving this special file String fscache = EditorPaths::get_singleton()->get_project_settings_dir().path_join("filesystem_update4"); Ref<FileAccess> f = FileAccess::open(fscache, FileAccess::WRITE); ERR_FAIL_COND_MSG(f.is_null(), "Cannot create file '" + fscache + "'. Check user write permissions."); for (const String &E : late_update_files) { f->store_line(E); } } Vector<String> EditorFileSystem::_get_dependencies(const String &p_path) { // Avoid error spam on first opening of a not yet imported project by treating the following situation // as a benign one, not letting the file open error happen: the resource is of an importable type but // it has not been imported yet. if (ResourceFormatImporter::get_singleton()->recognize_path(p_path)) { const String &internal_path = ResourceFormatImporter::get_singleton()->get_internal_resource_path(p_path); if (!internal_path.is_empty() && !FileAccess::exists(internal_path)) { // If path is empty (error), keep the code flow to the error. return Vector<String>(); } } List<String> deps; ResourceLoader::get_dependencies(p_path, &deps); Vector<String> ret; for (const String &E : deps) { ret.push_back(E); } return ret; } EditorFileSystem::ScriptClassInfo EditorFileSystem::_get_global_script_class(const String &p_type, const String &p_path) const { ScriptClassInfo info; for (int i = 0; i < ScriptServer::get_language_count(); i++) { if (ScriptServer::get_language(i)->handles_global_class_type(p_type)) { info.name = ScriptServer::get_language(i)->get_global_class_name(p_path, &info.extends, &info.icon_path, &info.is_abstract, &info.is_tool); break; } } return info; } void EditorFileSystem::_update_file_icon_path(EditorFileSystemDirectory::FileInfo *file_info) { String icon_path; if (file_info->resource_script_class != StringName()) { icon_path = EditorNode::get_editor_data().script_class_get_icon_path(file_info->resource_script_class); } else if (file_info->class_info.icon_path.is_empty() && !file_info->deps.is_empty()) { const String &script_dep = file_info->deps[0]; // Assuming the first dependency is a script. const String &script_path = script_dep.contains("::") ? script_dep.get_slice("::", 2) : script_dep; if (!script_path.is_empty()) { String *cached = file_icon_cache.getptr(script_path); if (cached) { icon_path = *cached; } else { if (ClassDB::is_parent_class(ResourceLoader::get_resource_type(script_path), SNAME("Script"))) { int script_file; EditorFileSystemDirectory *efsd = find_file(script_path, &script_file); if (efsd) { icon_path = efsd->files[script_file]->class_info.icon_path; } } file_icon_cache.insert(script_path, icon_path); } } } if (icon_path.is_empty() && !file_info->type.is_empty()) { Ref<Texture2D> icon = EditorNode::get_singleton()->get_class_icon(file_info->type); if (icon.is_valid()) { icon_path = icon->get_path(); } } file_info->class_info.icon_path = icon_path; } void EditorFileSystem::_update_files_icon_path(EditorFileSystemDirectory *edp) { if (!edp) { edp = filesystem; file_icon_cache.clear(); } for (EditorFileSystemDirectory *sub_dir : edp->subdirs) { _update_files_icon_path(sub_dir); } for (EditorFileSystemDirectory::FileInfo *fi : edp->files) { _update_file_icon_path(fi); } } void EditorFileSystem::_update_script_classes() { if (update_script_paths.is_empty()) { // Ensure the global class file is always present; it's essential for exports to work. if (!FileAccess::exists(ProjectSettings::get_singleton()->get_global_class_list_path())) { ScriptServer::save_global_classes(); } return; } { MutexLock update_script_lock(update_script_mutex); EditorProgress *ep = nullptr; if (update_script_paths.size() > 1) { if (MessageQueue::get_singleton()->is_flushing()) { // Use background progress when message queue is flushing. ep = memnew(EditorProgress("update_scripts_classes", TTR("Registering global classes..."), update_script_paths.size(), false, true)); } else { ep = memnew(EditorProgress("update_scripts_classes", TTR("Registering global classes..."), update_script_paths.size())); } } int step_count = 0; for (const KeyValue<String, ScriptClassInfoUpdate> &E : update_script_paths) { _register_global_class_script(E.key, E.key, E.value); if (ep) { ep->step(E.value.name, step_count++, false); } } memdelete_notnull(ep); update_script_paths.clear(); } EditorNode::get_editor_data().script_class_save_global_classes(); emit_signal("script_classes_updated"); // Rescan custom loaders and savers. // Doing the following here because the `filesystem_changed` signal fires multiple times and isn't always followed by script classes update. // So I thought it's better to do this when script classes really get updated ResourceLoader::remove_custom_loaders(); ResourceLoader::add_custom_loaders(); ResourceSaver::remove_custom_savers(); ResourceSaver::add_custom_savers(); } void EditorFileSystem::_update_script_documentation() { if (update_script_paths_documentation.is_empty()) { return; } MutexLock update_script_lock(update_script_mutex); EditorProgress *ep = nullptr; if (update_script_paths_documentation.size() > 1) { if (MessageQueue::get_singleton()->is_flushing()) { // Use background progress when message queue is flushing. ep = memnew(EditorProgress("update_script_paths_documentation", TTR("Updating scripts documentation"), update_script_paths_documentation.size(), false, true)); } else { ep = memnew(EditorProgress("update_script_paths_documentation", TTR("Updating scripts documentation"), update_script_paths_documentation.size())); } } int step_count = 0; for (const String &path : update_script_paths_documentation) { int index = -1; EditorFileSystemDirectory *efd = find_file(path, &index); if (!efd || index < 0) { // The file was removed continue; } for (int i = 0; i < ScriptServer::get_language_count(); i++) { ScriptLanguage *lang = ScriptServer::get_language(i); if (lang->supports_documentation() && efd->files[index]->type == lang->get_type()) { bool should_reload_script = _should_reload_script(path); Ref<Script> scr = ResourceLoader::load(path); if (scr.is_null()) { continue; } if (should_reload_script) { // Reloading the script from disk. Otherwise, the ResourceLoader::load will // return the last loaded version of the script (without the modifications). scr->reload_from_file(); } for (const DocData::ClassDoc &cd : scr->get_documentation()) { EditorHelp::get_doc_data()->add_doc(cd); if (!first_scan) { // Update the documentation in the Script Editor if it is open. ScriptEditor::get_singleton()->update_doc(cd.name); } } } } if (ep) { ep->step(efd->files[index]->file, step_count++, false); } } memdelete_notnull(ep); update_script_paths_documentation.clear(); } bool EditorFileSystem::_should_reload_script(const String &p_path) { if (first_scan) { return false; } Ref<Script> scr = ResourceCache::get_ref(p_path); if (scr.is_null()) { // Not a script or not already loaded. return false; } // Scripts are reloaded via the script editor if they are currently opened. if (ScriptEditor::get_singleton()->get_open_scripts().has(scr)) { return false; } return true; } void EditorFileSystem::_process_update_pending() { _update_script_classes(); // Parse documentation second, as it requires the class names to be loaded // because _update_script_documentation loads the scripts completely. _update_script_documentation(); _update_pending_scene_groups(); } void EditorFileSystem::_queue_update_script_class(const String &p_path, const ScriptClassInfoUpdate &p_script_update) { MutexLock update_script_lock(update_script_mutex); update_script_paths.insert(p_path, p_script_update); update_script_paths_documentation.insert(p_path); } void EditorFileSystem::_update_scene_groups() { if (update_scene_paths.is_empty()) { return; } EditorProgress *ep = nullptr; if (update_scene_paths.size() > 20) { ep = memnew(EditorProgress("update_scene_groups", TTR("Updating Scene Groups"), update_scene_paths.size())); } int step_count = 0; { MutexLock update_scene_lock(update_scene_mutex); for (const String &path : update_scene_paths) { ProjectSettings::get_singleton()->remove_scene_groups_cache(path); int index = -1; EditorFileSystemDirectory *efd = find_file(path, &index); if (!efd || index < 0) { // The file was removed. continue; } const HashSet<StringName> scene_groups = PackedScene::get_scene_groups(path); if (!scene_groups.is_empty()) { ProjectSettings::get_singleton()->add_scene_groups_cache(path, scene_groups); } if (ep) { ep->step(efd->files[index]->file, step_count++, false); } } memdelete_notnull(ep); update_scene_paths.clear(); } ProjectSettings::get_singleton()->save_scene_groups_cache(); } void EditorFileSystem::_update_pending_scene_groups() { if (!FileAccess::exists(ProjectSettings::get_singleton()->get_scene_groups_cache_path())) { _get_all_scenes(get_filesystem(), update_scene_paths); _update_scene_groups(); } else if (!update_scene_paths.is_empty()) { _update_scene_groups(); } } void EditorFileSystem::_queue_update_scene_groups(const String &p_path) { MutexLock update_scene_lock(update_scene_mutex); update_scene_paths.insert(p_path); } void EditorFileSystem::_get_all_scenes(EditorFileSystemDirectory *p_dir, HashSet<String> &r_list) { for (int i = 0; i < p_dir->get_file_count(); i++) { if (p_dir->get_file_type(i) == SNAME("PackedScene")) { r_list.insert(p_dir->get_file_path(i)); } } for (int i = 0; i < p_dir->get_subdir_count(); i++) { _get_all_scenes(p_dir->get_subdir(i), r_list); } } void EditorFileSystem::update_file(const String &p_file) { ERR_FAIL_COND(p_file.is_empty()); update_files({ p_file }); } void EditorFileSystem::update_files(const Vector<String> &p_script_paths) { bool updated = false; bool update_files_icon_cache = false; Vector<EditorFileSystemDirectory::FileInfo *> files_to_update_icon_path; for (const String &file : p_script_paths) { ERR_CONTINUE(file.is_empty()); EditorFileSystemDirectory *fs = nullptr; int cpos = -1; if (!_find_file(file, &fs, cpos)) { if (!fs) { continue; } } if (!FileAccess::exists(file)) { //was removed _delete_internal_files(file); if (cpos != -1) { // Might've never been part of the editor file system (*.* files deleted in Open dialog). if (fs->files[cpos]->uid != ResourceUID::INVALID_ID) { if (ResourceUID::get_singleton()->has_id(fs->files[cpos]->uid)) { ResourceUID::get_singleton()->remove_id(fs->files[cpos]->uid); } } if (ClassDB::is_parent_class(fs->files[cpos]->type, SNAME("Script"))) { ScriptClassInfoUpdate update; update.type = fs->files[cpos]->type; _queue_update_script_class(file, update); if (!fs->files[cpos]->class_info.icon_path.is_empty()) { update_files_icon_cache = true; } } if (fs->files[cpos]->type == SNAME("PackedScene")) { _queue_update_scene_groups(file); } memdelete(fs->files[cpos]); fs->files.remove_at(cpos); updated = true; } } else { String type = ResourceLoader::get_resource_type(file); if (type.is_empty() && textfile_extensions.has(file.get_extension())) { type = "TextFile"; } if (type.is_empty() && other_file_extensions.has(file.get_extension())) { type = "OtherFile"; } String script_class = ResourceLoader::get_resource_script_class(file); ResourceUID::ID uid = ResourceLoader::get_resource_uid(file); if (cpos == -1) { // The file did not exist, it was added. int idx = 0; String file_name = file.get_file(); for (int i = 0; i < fs->files.size(); i++) { if (file.filenocasecmp_to(fs->files[i]->file) < 0) { break; } idx++; } EditorFileSystemDirectory::FileInfo *fi = memnew(EditorFileSystemDirectory::FileInfo); fi->file = file_name; fi->import_modified_time = 0; fi->import_valid = (type == "TextFile" || type == "OtherFile") ? true : ResourceLoader::is_import_valid(file); fi->import_md5 = ""; fi->import_dest_paths = Vector<String>(); if (idx == fs->files.size()) { fs->files.push_back(fi); } else { fs->files.insert(idx, fi); } cpos = idx; } else { //the file exists and it was updated, and was not added in this step. //this means we must force upon next restart to scan it again, to get proper type and dependencies late_update_files.insert(file); _save_late_updated_files(); //files need to be updated in the re-scan } EditorFileSystemDirectory::FileInfo *fi = fs->files[cpos]; const String old_script_class_icon_path = fi->class_info.icon_path; const String old_class_name = fi->class_info.name; fi->type = type; fi->resource_script_class = script_class; fi->uid = uid; fi->class_info = _get_global_script_class(type, file); fi->import_group_file = ResourceLoader::get_import_group_file(file); fi->modified_time = FileAccess::get_modified_time(file); fi->deps = _get_dependencies(file); fi->import_valid = (type == "TextFile" || type == "OtherFile") ? true : ResourceLoader::is_import_valid(file); if (uid != ResourceUID::INVALID_ID) { if (ResourceUID::get_singleton()->has_id(uid)) { ResourceUID::get_singleton()->set_id(uid, file); } else { ResourceUID::get_singleton()->add_id(uid, file); } ResourceUID::get_singleton()->update_cache(); } else { if (ResourceLoader::should_create_uid_file(file)) { Ref<FileAccess> f = FileAccess::open(file + ".uid", FileAccess::WRITE); if (f.is_valid()) { const ResourceUID::ID id = ResourceUID::get_singleton()->create_id(); ResourceUID::get_singleton()->add_id(id, file); f->store_line(ResourceUID::get_singleton()->id_to_text(id)); fi->uid = id; } } } // Update preview EditorResourcePreview::get_singleton()->check_for_invalidation(file); if (ClassDB::is_parent_class(fi->type, SNAME("Script"))) { _queue_update_script_class(file, ScriptClassInfoUpdate::from_file_info(fi)); } if (fi->type == SNAME("PackedScene")) { _queue_update_scene_groups(file); } if (ClassDB::is_parent_class(fi->type, SNAME("Resource"))) { files_to_update_icon_path.push_back(fi); } else if (old_script_class_icon_path != fi->class_info.icon_path) { update_files_icon_cache = true; } // Restore another script as the global class name if multiple scripts had the same old class name. if (!old_class_name.is_empty() && fi->class_info.name != old_class_name && ClassDB::is_parent_class(type, SNAME("Script"))) { EditorFileSystemDirectory::FileInfo *old_fi = nullptr; String old_file = _get_file_by_class_name(filesystem, old_class_name, old_fi); if (!old_file.is_empty() && old_fi) { _queue_update_script_class(old_file, ScriptClassInfoUpdate::from_file_info(old_fi)); } } updated = true; } } if (updated) { if (update_files_icon_cache) { _update_files_icon_path(); } else { for (EditorFileSystemDirectory::FileInfo *fi : files_to_update_icon_path) { _update_file_icon_path(fi); } } if (!is_scanning()) { _process_update_pending(); } if (!filesystem_changed_queued) { filesystem_changed_queued = true; callable_mp(this, &EditorFileSystem::_notify_filesystem_changed).call_deferred(); } } } void EditorFileSystem::_notify_filesystem_changed() { emit_signal("filesystem_changed"); filesystem_changed_queued = false; } HashSet<String> EditorFileSystem::get_valid_extensions() const { return valid_extensions; } void EditorFileSystem::_register_global_class_script(const String &p_search_path, const String &p_target_path, const ScriptClassInfoUpdate &p_script_update) { ScriptServer::remove_global_class_by_path(p_search_path); // First remove, just in case it changed if (p_script_update.name.is_empty()) { return; } String lang; for (int j = 0; j < ScriptServer::get_language_count(); j++) { if (ScriptServer::get_language(j)->handles_global_class_type(p_script_update.type)) { lang = ScriptServer::get_language(j)->get_name(); break; } } if (lang.is_empty()) { return; // No lang found that can handle this global class } ScriptServer::add_global_class(p_script_update.name, p_script_update.extends, lang, p_target_path, p_script_update.is_abstract, p_script_update.is_tool); EditorNode::get_editor_data().script_class_set_icon_path(p_script_update.name, p_script_update.icon_path); EditorNode::get_editor_data().script_class_set_name(p_target_path, p_script_update.name); } void EditorFileSystem::register_global_class_script(const String &p_search_path, const String &p_target_path) { int index_file; EditorFileSystemDirectory *efsd = find_file(p_search_path, &index_file); if (efsd) { const EditorFileSystemDirectory::FileInfo *fi = efsd->files[index_file]; EditorFileSystem::get_singleton()->_register_global_class_script(p_search_path, p_target_path, ScriptClassInfoUpdate::from_file_info(fi)); } else { ScriptServer::remove_global_class_by_path(p_search_path); } } Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector<String> &p_files) { String importer_name; HashMap<String, HashMap<StringName, Variant>> source_file_options; HashMap<String, ResourceUID::ID> uids; HashMap<String, String> base_paths; for (int i = 0; i < p_files.size(); i++) { Ref<ConfigFile> config; config.instantiate(); Error err = config->load(p_files[i] + ".import"); ERR_CONTINUE(err != OK); ERR_CONTINUE(!config->has_section_key("remap", "importer")); String file_importer_name = config->get_value("remap", "importer"); ERR_CONTINUE(file_importer_name.is_empty()); if (!importer_name.is_empty() && importer_name != file_importer_name) { EditorNode::get_singleton()->show_warning(vformat(TTR("There are multiple importers for different types pointing to file %s, import aborted"), p_group_file)); ERR_FAIL_V(ERR_FILE_CORRUPT); } ResourceUID::ID uid = ResourceUID::INVALID_ID; if (config->has_section_key("remap", "uid")) { String uidt = config->get_value("remap", "uid"); uid = ResourceUID::get_singleton()->text_to_id(uidt); } uids[p_files[i]] = uid; source_file_options[p_files[i]] = HashMap<StringName, Variant>(); importer_name = file_importer_name; if (importer_name == "keep" || importer_name == "skip") { continue; //do nothing } Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name); ERR_FAIL_COND_V(importer.is_null(), ERR_FILE_CORRUPT); List<ResourceImporter::ImportOption> options; importer->get_import_options(p_files[i], &options); //set default values for (const ResourceImporter::ImportOption &E : options) { source_file_options[p_files[i]][E.option.name] = E.default_value; } if (config->has_section("params")) { List<String> sk; config->get_section_keys("params", &sk); for (const String ¶m : sk) { Variant value = config->get_value("params", param); //override with whatever is in file source_file_options[p_files[i]][param] = value; } } base_paths[p_files[i]] = ResourceFormatImporter::get_singleton()->get_import_base_path(p_files[i]); } if (importer_name == "keep" || importer_name == "skip") { return OK; // (do nothing) } ERR_FAIL_COND_V(importer_name.is_empty(), ERR_UNCONFIGURED); Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name); Error err = importer->import_group_file(p_group_file, source_file_options, base_paths); //all went well, overwrite config files with proper remaps and md5s for (const KeyValue<String, HashMap<StringName, Variant>> &E : source_file_options) { const String &file = E.key; String base_path = ResourceFormatImporter::get_singleton()->get_import_base_path(file); Vector<String> dest_paths; ResourceUID::ID uid = uids[file]; { Ref<FileAccess> f = FileAccess::open(file + ".import", FileAccess::WRITE); ERR_FAIL_COND_V_MSG(f.is_null(), ERR_FILE_CANT_OPEN, "Cannot open import file '" + file + ".import'."); //write manually, as order matters ([remap] has to go first for performance). f->store_line("[remap]"); f->store_line(""); f->store_line("importer=\"" + importer->get_importer_name() + "\""); int version = importer->get_format_version(); if (version > 0) { f->store_line("importer_version=" + itos(version)); } if (!importer->get_resource_type().is_empty()) { f->store_line("type=\"" + importer->get_resource_type() + "\""); } if (uid == ResourceUID::INVALID_ID) { uid = ResourceUID::get_singleton()->create_id(); } f->store_line("uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\""); // Store in readable format. if (err == OK) { String path = base_path + "." + importer->get_save_extension(); f->store_line("path=\"" + path + "\""); dest_paths.push_back(path); } f->store_line("group_file=" + Variant(p_group_file).get_construct_string()); if (err == OK) { f->store_line("valid=true"); } else { f->store_line("valid=false"); } f->store_line("[deps]\n"); f->store_line(""); f->store_line("source_file=" + Variant(file).get_construct_string()); if (dest_paths.size()) { Array dp; for (int i = 0; i < dest_paths.size(); i++) { dp.push_back(dest_paths[i]); } f->store_line("dest_files=" + Variant(dp).get_construct_string() + "\n"); } f->store_line("[params]"); f->store_line(""); //store options in provided order, to avoid file changing. Order is also important because first match is accepted first. List<ResourceImporter::ImportOption> options; importer->get_import_options(file, &options); //set default values for (const ResourceImporter::ImportOption &F : options) { String base = F.option.name; Variant v = F.default_value; if (source_file_options[file].has(base)) { v = source_file_options[file][base]; } String value; VariantWriter::write_to_string(v, value); f->store_line(base + "=" + value); } } // Store the md5's of the various files. These are stored separately so that the .import files can be version controlled. { Ref<FileAccess> md5s = FileAccess::open(base_path + ".md5", FileAccess::WRITE); ERR_FAIL_COND_V_MSG(md5s.is_null(), ERR_FILE_CANT_OPEN, "Cannot open MD5 file '" + base_path + ".md5'."); md5s->store_line("source_md5=\"" + FileAccess::get_md5(file) + "\""); if (dest_paths.size()) { md5s->store_line("dest_md5=\"" + FileAccess::get_multiple_md5(dest_paths) + "\"\n"); } } EditorFileSystemDirectory *fs = nullptr; int cpos = -1; bool found = _find_file(file, &fs, cpos); ERR_FAIL_COND_V_MSG(!found, ERR_UNCONFIGURED, vformat("Can't find file '%s' during group reimport.", file)); //update modified times, to avoid reimport fs->files[cpos]->modified_time = FileAccess::get_modified_time(file); fs->files[cpos]->import_modified_time = FileAccess::get_modified_time(file + ".import"); fs->files[cpos]->import_md5 = FileAccess::get_md5(file + ".import"); fs->files[cpos]->import_dest_paths = dest_paths; fs->files[cpos]->deps = _get_dependencies(file); fs->files[cpos]->uid = uid; fs->files[cpos]->type = importer->get_resource_type(); if (fs->files[cpos]->type == "" && textfile_extensions.has(file.get_extension())) { fs->files[cpos]->type = "TextFile"; } if (fs->files[cpos]->type == "" && other_file_extensions.has(file.get_extension())) { fs->files[cpos]->type = "OtherFile"; } fs->files[cpos]->import_valid = err == OK; if (ResourceUID::get_singleton()->has_id(uid)) { ResourceUID::get_singleton()->set_id(uid, file); } else { ResourceUID::get_singleton()->add_id(uid, file); } //if file is currently up, maybe the source it was loaded from changed, so import math must be updated for it //to reload properly Ref<Resource> r = ResourceCache::get_ref(file); if (r.is_valid()) { if (!r->get_import_path().is_empty()) { String dst_path = ResourceFormatImporter::get_singleton()->get_internal_resource_path(file); r->set_import_path(dst_path); r->set_import_last_modified_time(0); } } EditorResourcePreview::get_singleton()->check_for_invalidation(file); } return err; } Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<StringName, Variant> &p_custom_options, const String &p_custom_importer, Variant *p_generator_parameters, bool p_update_file_system) { print_verbose(vformat("EditorFileSystem: Importing file: %s", p_file)); uint64_t start_time = OS::get_singleton()->get_ticks_msec(); EditorFileSystemDirectory *fs = nullptr; int cpos = -1; if (p_update_file_system) { bool found = _find_file(p_file, &fs, cpos); ERR_FAIL_COND_V_MSG(!found, ERR_FILE_NOT_FOUND, vformat("Can't find file '%s' during file reimport.", p_file)); } //try to obtain existing params HashMap<StringName, Variant> params = p_custom_options; String importer_name; //empty by default though if (!p_custom_importer.is_empty()) { importer_name = p_custom_importer; } ResourceUID::ID uid = ResourceUID::INVALID_ID; Variant generator_parameters; if (p_generator_parameters) { generator_parameters = *p_generator_parameters; } if (FileAccess::exists(p_file + ".import")) { //use existing Ref<ConfigFile> cf; cf.instantiate(); Error err = cf->load(p_file + ".import"); if (err == OK) { if (cf->has_section("params")) { List<String> sk; cf->get_section_keys("params", &sk); for (const String &E : sk) { if (!params.has(E)) { params[E] = cf->get_value("params", E); } } } if (cf->has_section("remap")) { if (p_custom_importer.is_empty()) { importer_name = cf->get_value("remap", "importer"); } if (cf->has_section_key("remap", "uid")) { String uidt = cf->get_value("remap", "uid"); uid = ResourceUID::get_singleton()->text_to_id(uidt); } if (!p_generator_parameters) { if (cf->has_section_key("remap", "generator_parameters")) { generator_parameters = cf->get_value("remap", "generator_parameters"); } } } } } if (importer_name == "keep" || importer_name == "skip") { //keep files, do nothing. if (p_update_file_system) { fs->files[cpos]->modified_time = FileAccess::get_modified_time(p_file); fs->files[cpos]->import_modified_time = FileAccess::get_modified_time(p_file + ".import"); fs->files[cpos]->import_md5 = FileAccess::get_md5(p_file + ".import"); fs->files[cpos]->import_dest_paths = Vector<String>(); fs->files[cpos]->deps.clear(); fs->files[cpos]->type = ""; fs->files[cpos]->import_valid = false; EditorResourcePreview::get_singleton()->check_for_invalidation(p_file); } return OK; } Ref<ResourceImporter> importer; bool load_default = false; //find the importer if (!importer_name.is_empty()) { importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name); } if (importer.is_null()) { //not found by name, find by extension importer = ResourceFormatImporter::get_singleton()->get_importer_by_extension(p_file.get_extension()); load_default = true; if (importer.is_null()) { ERR_FAIL_V_MSG(ERR_FILE_CANT_OPEN, "BUG: File queued for import, but can't be imported, importer for type '" + importer_name + "' not found."); } } if (FileAccess::exists(p_file + ".import")) { // We only want to handle compat for existing files, not new ones. importer->handle_compatibility_options(params); } //mix with default params, in case a parameter is missing List<ResourceImporter::ImportOption> opts; importer->get_import_options(p_file, &opts); for (const ResourceImporter::ImportOption &E : opts) { if (!params.has(E.option.name)) { //this one is not present params[E.option.name] = E.default_value; } } if (load_default && ProjectSettings::get_singleton()->has_setting("importer_defaults/" + importer->get_importer_name())) { //use defaults if exist Dictionary d = GLOBAL_GET("importer_defaults/" + importer->get_importer_name()); List<Variant> v; d.get_key_list(&v); for (const Variant &E : v) { params[E] = d[E]; } } if (uid == ResourceUID::INVALID_ID) { uid = ResourceUID::get_singleton()->create_id(); } //finally, perform import!! String base_path = ResourceFormatImporter::get_singleton()->get_import_base_path(p_file); List<String> import_variants; List<String> gen_files; Variant meta; Error err = importer->import(uid, p_file, base_path, params, &import_variants, &gen_files, &meta); // As import is complete, save the .import file. Vector<String> dest_paths; { Ref<FileAccess> f = FileAccess::open(p_file + ".import", FileAccess::WRITE); ERR_FAIL_COND_V_MSG(f.is_null(), ERR_FILE_CANT_OPEN, "Cannot open file from path '" + p_file + ".import'."); // Write manually, as order matters ([remap] has to go first for performance). f->store_line("[remap]"); f->store_line(""); f->store_line("importer=\"" + importer->get_importer_name() + "\""); int version = importer->get_format_version(); if (version > 0) { f->store_line("importer_version=" + itos(version)); } if (!importer->get_resource_type().is_empty()) { f->store_line("type=\"" + importer->get_resource_type() + "\""); } f->store_line("uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\""); // Store in readable format. if (err == OK) { if (importer->get_save_extension().is_empty()) { //no path } else if (import_variants.size()) { //import with variants for (const String &E : import_variants) { String path = base_path.c_escape() + "." + E + "." + importer->get_save_extension(); f->store_line("path." + E + "=\"" + path + "\""); dest_paths.push_back(path); } } else { String path = base_path + "." + importer->get_save_extension(); f->store_line("path=\"" + path + "\""); dest_paths.push_back(path); } } else { f->store_line("valid=false"); } if (meta != Variant()) { f->store_line("metadata=" + meta.get_construct_string()); } if (generator_parameters != Variant()) { f->store_line("generator_parameters=" + generator_parameters.get_construct_string()); } f->store_line(""); f->store_line("[deps]\n"); if (gen_files.size()) { Array genf; for (const String &E : gen_files) { genf.push_back(E); dest_paths.push_back(E); } String value; VariantWriter::write_to_string(genf, value); f->store_line("files=" + value); f->store_line(""); } f->store_line("source_file=" + Variant(p_file).get_construct_string()); if (dest_paths.size()) { Array dp; for (int i = 0; i < dest_paths.size(); i++) { dp.push_back(dest_paths[i]); } f->store_line("dest_files=" + Variant(dp).get_construct_string()); } f->store_line(""); f->store_line("[params]"); f->store_line(""); // Store options in provided order, to avoid file changing. Order is also important because first match is accepted first. for (const ResourceImporter::ImportOption &E : opts) { String base = E.option.name; String value; VariantWriter::write_to_string(params[base], value); f->store_line(base + "=" + value); } } // Store the md5's of the various files. These are stored separately so that the .import files can be version controlled. { Ref<FileAccess> md5s = FileAccess::open(base_path + ".md5", FileAccess::WRITE); ERR_FAIL_COND_V_MSG(md5s.is_null(), ERR_FILE_CANT_OPEN, "Cannot open MD5 file '" + base_path + ".md5'."); md5s->store_line("source_md5=\"" + FileAccess::get_md5(p_file) + "\""); if (dest_paths.size()) { md5s->store_line("dest_md5=\"" + FileAccess::get_multiple_md5(dest_paths) + "\"\n"); } } if (p_update_file_system) { // Update cpos, newly created files could've changed the index of the reimported p_file. _find_file(p_file, &fs, cpos); // Update modified times, to avoid reimport. fs->files[cpos]->modified_time = FileAccess::get_modified_time(p_file); fs->files[cpos]->import_modified_time = FileAccess::get_modified_time(p_file + ".import"); fs->files[cpos]->import_md5 = FileAccess::get_md5(p_file + ".import"); fs->files[cpos]->import_dest_paths = dest_paths; fs->files[cpos]->deps = _get_dependencies(p_file); fs->files[cpos]->type = importer->get_resource_type(); fs->files[cpos]->uid = uid; fs->files[cpos]->import_valid = fs->files[cpos]->type == "TextFile" ? true : ResourceLoader::is_import_valid(p_file); } if (ResourceUID::get_singleton()->has_id(uid)) { ResourceUID::get_singleton()->set_id(uid, p_file); } else { ResourceUID::get_singleton()->add_id(uid, p_file); } // If file is currently up, maybe the source it was loaded from changed, so import math must be updated for it // to reload properly. Ref<Resource> r = ResourceCache::get_ref(p_file); if (r.is_valid()) { if (!r->get_import_path().is_empty()) { String dst_path = ResourceFormatImporter::get_singleton()->get_internal_resource_path(p_file); r->set_import_path(dst_path); r->set_import_last_modified_time(0); } } EditorResourcePreview::get_singleton()->check_for_invalidation(p_file); print_verbose(vformat("EditorFileSystem: \"%s\" import took %d ms.", p_file, OS::get_singleton()->get_ticks_msec() - start_time)); ERR_FAIL_COND_V_MSG(err != OK, ERR_FILE_UNRECOGNIZED, "Error importing '" + p_file + "'."); return OK; } void EditorFileSystem::_find_group_files(EditorFileSystemDirectory *efd, HashMap<String, Vector<String>> &group_files, HashSet<String> &groups_to_reimport) { int fc = efd->files.size(); const EditorFileSystemDirectory::FileInfo *const *files = efd->files.ptr(); for (int i = 0; i < fc; i++) { if (groups_to_reimport.has(files[i]->import_group_file)) { if (!group_files.has(files[i]->import_group_file)) { group_files[files[i]->import_group_file] = Vector<String>(); } group_files[files[i]->import_group_file].push_back(efd->get_file_path(i)); } } for (int i = 0; i < efd->get_subdir_count(); i++) { _find_group_files(efd->get_subdir(i), group_files, groups_to_reimport); } } void EditorFileSystem::reimport_file_with_custom_parameters(const String &p_file, const String &p_importer, const HashMap<StringName, Variant> &p_custom_params) { Vector<String> reloads; reloads.append(p_file); // Emit the resource_reimporting signal for the single file before the actual importation. emit_signal(SNAME("resources_reimporting"), reloads); _reimport_file(p_file, p_custom_params, p_importer); // Emit the resource_reimported signal for the single file we just reimported. emit_signal(SNAME("resources_reimported"), reloads); } Error EditorFileSystem::_copy_file(const String &p_from, const String &p_to) { Ref<DirAccess> da = DirAccess::create(DirAccess::ACCESS_RESOURCES); if (FileAccess::exists(p_from + ".import")) { Error err = da->copy(p_from, p_to); if (err != OK) { return err; } // Remove uid from .import file to avoid conflict. Ref<ConfigFile> cfg; cfg.instantiate(); cfg->load(p_from + ".import"); cfg->erase_section_key("remap", "uid"); err = cfg->save(p_to + ".import"); if (err != OK) { return err; } } else if (ResourceLoader::get_resource_uid(p_from) == ResourceUID::INVALID_ID) { // Files which do not use an uid can just be copied. Error err = da->copy(p_from, p_to); if (err != OK) { return err; } } else { // Load the resource and save it again in the new location (this generates a new UID). Error err; Ref<Resource> res = ResourceLoader::load(p_from, "", ResourceFormatLoader::CACHE_MODE_REUSE, &err); if (err == OK && res.is_valid()) { err = ResourceSaver::save(res, p_to, ResourceSaver::FLAG_COMPRESS); if (err != OK) { return err; } } else if (err != OK) { // When loading files like text files the error is OK but the resource is still null. // We can ignore such files. return err; } } return OK; } bool EditorFileSystem::_copy_directory(const String &p_from, const String &p_to, List<CopiedFile> *p_files) { Ref<DirAccess> old_dir = DirAccess::open(p_from); ERR_FAIL_COND_V(old_dir.is_null(), false); Error err = make_dir_recursive(p_to); if (err != OK && err != ERR_ALREADY_EXISTS) { return false; } bool success = true; old_dir->set_include_navigational(false); old_dir->list_dir_begin(); for (String F = old_dir->_get_next(); !F.is_empty(); F = old_dir->_get_next()) { if (old_dir->current_is_dir()) { success = _copy_directory(p_from.path_join(F), p_to.path_join(F), p_files) && success; } else if (F.get_extension() != "import" && F.get_extension() != "uid") { CopiedFile copy; copy.from = p_from.path_join(F); copy.to = p_to.path_join(F); p_files->push_back(copy); } } return success; } void EditorFileSystem::_queue_refresh_filesystem() { if (refresh_queued) { return; } refresh_queued = true; get_tree()->connect(SNAME("process_frame"), callable_mp(this, &EditorFileSystem::_refresh_filesystem), CONNECT_ONE_SHOT); } void EditorFileSystem::_refresh_filesystem() { for (const ObjectID &id : folders_to_sort) { EditorFileSystemDirectory *dir = Object::cast_to<EditorFileSystemDirectory>(ObjectDB::get_instance(id)); if (dir) { dir->subdirs.sort_custom<DirectoryComparator>(); } } folders_to_sort.clear(); _update_scan_actions(); emit_signal(SNAME("filesystem_changed")); refresh_queued = false; } void EditorFileSystem::_reimport_thread(uint32_t p_index, ImportThreadData *p_import_data) { int file_idx = p_import_data->reimport_from + int(p_index); _reimport_file(p_import_data->reimport_files[file_idx].path); p_import_data->imported_sem->post(); } void EditorFileSystem::reimport_files(const Vector<String> &p_files) { ERR_FAIL_COND_MSG(importing, "Attempted to call reimport_files() recursively, this is not allowed."); importing = true; Vector<String> reloads; EditorProgress *ep = memnew(EditorProgress("reimport", TTR("(Re)Importing Assets"), p_files.size())); // The method reimport_files runs on the main thread, and if VSync is enabled // or Update Continuously is disabled, Main::Iteration takes longer each frame. // Each EditorProgress::step can trigger a redraw, and when there are many files to import, // this could lead to a slow import process, especially when the editor is unfocused. // Temporarily disabling VSync and low_processor_usage_mode while reimporting fixes this. const bool old_low_processor_usage_mode = OS::get_singleton()->is_in_low_processor_usage_mode(); const DisplayServer::VSyncMode old_vsync_mode = DisplayServer::get_singleton()->window_get_vsync_mode(DisplayServer::MAIN_WINDOW_ID); OS::get_singleton()->set_low_processor_usage_mode(false); DisplayServer::get_singleton()->window_set_vsync_mode(DisplayServer::VSyncMode::VSYNC_DISABLED); Vector<ImportFile> reimport_files; HashSet<String> groups_to_reimport; for (int i = 0; i < p_files.size(); i++) { ep->step(TTR("Preparing files to reimport..."), i, false); String file = p_files[i]; ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(file); if (uid != ResourceUID::INVALID_ID && ResourceUID::get_singleton()->has_id(uid)) { file = ResourceUID::get_singleton()->get_id_path(uid); } String group_file = ResourceFormatImporter::get_singleton()->get_import_group_file(file); if (group_file_cache.has(file)) { // Maybe the file itself is a group! groups_to_reimport.insert(file); // Groups do not belong to groups. group_file = String(); } else if (groups_to_reimport.has(file)) { // Groups do not belong to groups. group_file = String(); } else if (!group_file.is_empty()) { // It's a group file, add group to import and skip this file. groups_to_reimport.insert(group_file); } else { // It's a regular file. ImportFile ifile; ifile.path = file; ResourceFormatImporter::get_singleton()->get_import_order_threads_and_importer(file, ifile.order, ifile.threaded, ifile.importer); reloads.push_back(file); reimport_files.push_back(ifile); } // Group may have changed, so also update group reference. EditorFileSystemDirectory *fs = nullptr; int cpos = -1; if (_find_file(file, &fs, cpos)) { fs->files.write[cpos]->import_group_file = group_file; } } reimport_files.sort(); ep->step(TTR("Executing pre-reimport operations..."), 0, true); // Emit the resource_reimporting signal for the single file before the actual importation. emit_signal(SNAME("resources_reimporting"), reloads); #ifdef THREADS_ENABLED bool use_multiple_threads = GLOBAL_GET("editor/import/use_multiple_threads"); #else bool use_multiple_threads = false; #endif int from = 0; Semaphore imported_sem; for (int i = 0; i < reimport_files.size(); i++) { if (groups_to_reimport.has(reimport_files[i].path)) { from = i + 1; continue; } if (use_multiple_threads && reimport_files[i].threaded) { if (i + 1 == reimport_files.size() || reimport_files[i + 1].importer != reimport_files[from].importer || groups_to_reimport.has(reimport_files[i + 1].path)) { if (from - i == 0) { // Single file, do not use threads. ep->step(reimport_files[i].path.get_file(), i, false); _reimport_file(reimport_files[i].path); } else { Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(reimport_files[from].importer); if (importer.is_null()) { ERR_PRINT(vformat("Invalid importer for \"%s\".", reimport_files[from].importer)); from = i + 1; continue; } importer->import_threaded_begin(); ImportThreadData tdata; tdata.reimport_from = from; tdata.reimport_files = reimport_files.ptr(); tdata.imported_sem = &imported_sem; int item_count = i - from + 1; WorkerThreadPool::GroupID group_task = WorkerThreadPool::get_singleton()->add_template_group_task(this, &EditorFileSystem::_reimport_thread, &tdata, item_count, -1, false, vformat(TTR("Import resources of type: %s"), reimport_files[from].importer)); int imported_count = 0; while (true) { ep->step(reimport_files[imported_count].path.get_file(), from + imported_count, false); imported_sem.wait(); do { imported_count++; } while (imported_sem.try_wait()); if (imported_count == item_count) { break; } } WorkerThreadPool::get_singleton()->wait_for_group_task_completion(group_task); DEV_ASSERT(!imported_sem.try_wait()); importer->import_threaded_end(); } from = i + 1; } } else { ep->step(reimport_files[i].path.get_file(), i, false); _reimport_file(reimport_files[i].path); // We need to increment the counter, maybe the next file is multithreaded // and doesn't have the same importer. from = i + 1; } } // Reimport groups. from = reimport_files.size(); if (groups_to_reimport.size()) { HashMap<String, Vector<String>> group_files; _find_group_files(filesystem, group_files, groups_to_reimport); for (const KeyValue<String, Vector<String>> &E : group_files) { ep->step(E.key.get_file(), from++, false); Error err = _reimport_group(E.key, E.value); reloads.push_back(E.key); reloads.append_array(E.value); if (err == OK) { _reimport_file(E.key); } } } ep->step(TTR("Finalizing Asset Import..."), p_files.size()); ResourceUID::get_singleton()->update_cache(); // After reimporting, update the cache. _save_filesystem_cache(); memdelete_notnull(ep); _process_update_pending(); // Revert to previous values to restore editor settings for VSync and Update Continuously. OS::get_singleton()->set_low_processor_usage_mode(old_low_processor_usage_mode); DisplayServer::get_singleton()->window_set_vsync_mode(old_vsync_mode); importing = false; ep = memnew(EditorProgress("reimport", TTR("(Re)Importing Assets"), p_files.size())); ep->step(TTR("Executing post-reimport operations..."), 0, true); if (!is_scanning()) { emit_signal(SNAME("filesystem_changed")); } emit_signal(SNAME("resources_reimported"), reloads); memdelete_notnull(ep); } Error EditorFileSystem::reimport_append(const String &p_file, const HashMap<StringName, Variant> &p_custom_options, const String &p_custom_importer, Variant p_generator_parameters) { Vector<String> reloads; reloads.append(p_file); // Emit the resource_reimporting signal for the single file before the actual importation. emit_signal(SNAME("resources_reimporting"), reloads); Error ret = _reimport_file(p_file, p_custom_options, p_custom_importer, &p_generator_parameters); // Emit the resource_reimported signal for the single file we just reimported. emit_signal(SNAME("resources_reimported"), reloads); return ret; } Error EditorFileSystem::_resource_import(const String &p_path) { Vector<String> files; files.push_back(p_path); singleton->update_file(p_path); singleton->reimport_files(files); return OK; } Ref<Resource> EditorFileSystem::_load_resource_on_startup(ResourceFormatImporter *p_importer, const String &p_path, Error *r_error, bool p_use_sub_threads, float *r_progress, ResourceFormatLoader::CacheMode p_cache_mode) { ERR_FAIL_NULL_V(p_importer, Ref<Resource>()); if (!FileAccess::exists(p_path)) { ERR_FAIL_V_MSG(Ref<Resource>(), vformat("Failed loading resource: %s. The file doesn't seem to exist.", p_path)); } Ref<Resource> res; bool can_retry = true; bool retry = true; while (retry) { retry = false; res = p_importer->load_internal(p_path, r_error, p_use_sub_threads, r_progress, p_cache_mode, can_retry); if (res.is_null() && can_retry) { can_retry = false; Error err = singleton->_reimport_file(p_path, HashMap<StringName, Variant>(), "", nullptr, false); if (err == OK) { retry = true; } } } return res; } bool EditorFileSystem::_should_skip_directory(const String &p_path) { String project_data_path = ProjectSettings::get_singleton()->get_project_data_path(); if (p_path == project_data_path || p_path.begins_with(project_data_path + "/")) { return true; } if (FileAccess::exists(p_path.path_join("project.godot"))) { // Skip if another project inside this. if (EditorFileSystem::get_singleton()->first_scan) { WARN_PRINT_ONCE(vformat("Detected another project.godot at %s. The folder will be ignored.", p_path)); } return true; } if (FileAccess::exists(p_path.path_join(".gdignore"))) { // Skip if a `.gdignore` file is inside this. return true; } return false; } bool EditorFileSystem::is_group_file(const String &p_path) const { return group_file_cache.has(p_path); } void EditorFileSystem::_move_group_files(EditorFileSystemDirectory *efd, const String &p_group_file, const String &p_new_location) { int fc = efd->files.size(); EditorFileSystemDirectory::FileInfo *const *files = efd->files.ptrw(); for (int i = 0; i < fc; i++) { if (files[i]->import_group_file == p_group_file) { files[i]->import_group_file = p_new_location; Ref<ConfigFile> config; config.instantiate(); String path = efd->get_file_path(i) + ".import"; Error err = config->load(path); if (err != OK) { continue; } if (config->has_section_key("remap", "group_file")) { config->set_value("remap", "group_file", p_new_location); } List<String> sk; config->get_section_keys("params", &sk); for (const String ¶m : sk) { //not very clean, but should work String value = config->get_value("params", param); if (value == p_group_file) { config->set_value("params", param, p_new_location); } } config->save(path); } } for (int i = 0; i < efd->get_subdir_count(); i++) { _move_group_files(efd->get_subdir(i), p_group_file, p_new_location); } } void EditorFileSystem::move_group_file(const String &p_path, const String &p_new_path) { if (get_filesystem()) { _move_group_files(get_filesystem(), p_path, p_new_path); if (group_file_cache.has(p_path)) { group_file_cache.erase(p_path); group_file_cache.insert(p_new_path); } } } Error EditorFileSystem::make_dir_recursive(const String &p_path, const String &p_base_path) { Error err; Ref<DirAccess> da = DirAccess::create(DirAccess::ACCESS_RESOURCES); if (!p_base_path.is_empty()) { err = da->change_dir(p_base_path); ERR_FAIL_COND_V_MSG(err != OK, err, "Cannot open base directory '" + p_base_path + "'."); } if (da->dir_exists(p_path)) { return ERR_ALREADY_EXISTS; } err = da->make_dir_recursive(p_path); if (err != OK) { return err; } const String path = da->get_current_dir(); EditorFileSystemDirectory *parent = get_filesystem_path(path); ERR_FAIL_NULL_V(parent, ERR_FILE_NOT_FOUND); folders_to_sort.insert(parent->get_instance_id()); const PackedStringArray folders = p_path.trim_prefix(path).trim_suffix("/").split("/"); for (const String &folder : folders) { const int current = parent->find_dir_index(folder); if (current > -1) { parent = parent->get_subdir(current); continue; } EditorFileSystemDirectory *efd = memnew(EditorFileSystemDirectory); efd->parent = parent; efd->name = folder; parent->subdirs.push_back(efd); parent = efd; } _queue_refresh_filesystem(); return OK; } Error EditorFileSystem::copy_file(const String &p_from, const String &p_to) { _copy_file(p_from, p_to); EditorFileSystemDirectory *parent = get_filesystem_path(p_to.get_base_dir()); ERR_FAIL_NULL_V(parent, ERR_FILE_NOT_FOUND); ScanProgress sp; _scan_fs_changes(parent, sp, false); _queue_refresh_filesystem(); return OK; } Error EditorFileSystem::copy_directory(const String &p_from, const String &p_to) { List<CopiedFile> files; bool success = _copy_directory(p_from, p_to, &files); EditorProgress *ep = nullptr; if (files.size() > 10) { ep = memnew(EditorProgress("_copy_files", TTR("Copying files..."), files.size())); } int i = 0; for (const CopiedFile &F : files) { if (_copy_file(F.from, F.to) != OK) { success = false; } if (ep) { ep->step(F.from.get_file(), i++, false); } } memdelete_notnull(ep); EditorFileSystemDirectory *efd = get_filesystem_path(p_to); ERR_FAIL_NULL_V(efd, FAILED); ERR_FAIL_NULL_V(efd->get_parent(), FAILED); folders_to_sort.insert(efd->get_parent()->get_instance_id()); ScanProgress sp; _scan_fs_changes(efd, sp); _queue_refresh_filesystem(); return success ? OK : FAILED; } ResourceUID::ID EditorFileSystem::_resource_saver_get_resource_id_for_path(const String &p_path, bool p_generate) { if (!p_path.is_resource_file() || p_path.begins_with(ProjectSettings::get_singleton()->get_project_data_path())) { // Saved externally (configuration file) or internal file, do not assign an ID. return ResourceUID::INVALID_ID; } EditorFileSystemDirectory *fs = nullptr; int cpos = -1; if (!singleton->_find_file(p_path, &fs, cpos)) { // Fallback to ResourceLoader if filesystem cache fails (can happen during scanning etc.). ResourceUID::ID fallback = ResourceLoader::get_resource_uid(p_path); if (fallback != ResourceUID::INVALID_ID) { return fallback; } if (p_generate) { return ResourceUID::get_singleton()->create_id(); // Just create a new one, we will be notified of save anyway and fetch the right UID at that time, to keep things simple. } else { return ResourceUID::INVALID_ID; } } else if (fs->files[cpos]->uid != ResourceUID::INVALID_ID) { return fs->files[cpos]->uid; } else if (p_generate) { return ResourceUID::get_singleton()->create_id(); // Just create a new one, we will be notified of save anyway and fetch the right UID at that time, to keep things simple. } else { return ResourceUID::INVALID_ID; } } static void _scan_extensions_dir(EditorFileSystemDirectory *d, HashSet<String> &extensions) { int fc = d->get_file_count(); for (int i = 0; i < fc; i++) { if (d->get_file_type(i) == SNAME("GDExtension")) { extensions.insert(d->get_file_path(i)); } } int dc = d->get_subdir_count(); for (int i = 0; i < dc; i++) { _scan_extensions_dir(d->get_subdir(i), extensions); } } bool EditorFileSystem::_scan_extensions() { EditorFileSystemDirectory *d = get_filesystem(); HashSet<String> extensions; _scan_extensions_dir(d, extensions); return GDExtensionManager::get_singleton()->ensure_extensions_loaded(extensions); } void EditorFileSystem::_bind_methods() { ClassDB::bind_method(D_METHOD("get_filesystem"), &EditorFileSystem::get_filesystem); ClassDB::bind_method(D_METHOD("is_scanning"), &EditorFileSystem::is_scanning); ClassDB::bind_method(D_METHOD("get_scanning_progress"), &EditorFileSystem::get_scanning_progress); ClassDB::bind_method(D_METHOD("scan"), &EditorFileSystem::scan); ClassDB::bind_method(D_METHOD("scan_sources"), &EditorFileSystem::scan_changes); ClassDB::bind_method(D_METHOD("update_file", "path"), &EditorFileSystem::update_file); ClassDB::bind_method(D_METHOD("get_filesystem_path", "path"), &EditorFileSystem::get_filesystem_path); ClassDB::bind_method(D_METHOD("get_file_type", "path"), &EditorFileSystem::get_file_type); ClassDB::bind_method(D_METHOD("reimport_files", "files"), &EditorFileSystem::reimport_files); ADD_SIGNAL(MethodInfo("filesystem_changed")); ADD_SIGNAL(MethodInfo("script_classes_updated")); ADD_SIGNAL(MethodInfo("sources_changed", PropertyInfo(Variant::BOOL, "exist"))); ADD_SIGNAL(MethodInfo("resources_reimporting", PropertyInfo(Variant::PACKED_STRING_ARRAY, "resources"))); ADD_SIGNAL(MethodInfo("resources_reimported", PropertyInfo(Variant::PACKED_STRING_ARRAY, "resources"))); ADD_SIGNAL(MethodInfo("resources_reload", PropertyInfo(Variant::PACKED_STRING_ARRAY, "resources"))); } void EditorFileSystem::_update_extensions() { valid_extensions.clear(); import_extensions.clear(); textfile_extensions.clear(); other_file_extensions.clear(); List<String> extensionsl; ResourceLoader::get_recognized_extensions_for_type("", &extensionsl); for (const String &E : extensionsl) { valid_extensions.insert(E); } const Vector<String> textfile_ext = ((String)(EDITOR_GET("docks/filesystem/textfile_extensions"))).split(",", false); for (const String &E : textfile_ext) { if (valid_extensions.has(E)) { continue; } valid_extensions.insert(E); textfile_extensions.insert(E); } const Vector<String> other_file_ext = ((String)(EDITOR_GET("docks/filesystem/other_file_extensions"))).split(",", false); for (const String &E : other_file_ext) { if (valid_extensions.has(E)) { continue; } valid_extensions.insert(E); other_file_extensions.insert(E); } extensionsl.clear(); ResourceFormatImporter::get_singleton()->get_recognized_extensions(&extensionsl); for (const String &E : extensionsl) { import_extensions.insert(E); } } void EditorFileSystem::add_import_format_support_query(Ref<EditorFileSystemImportFormatSupportQuery> p_query) { ERR_FAIL_COND(import_support_queries.has(p_query)); import_support_queries.push_back(p_query); } void EditorFileSystem::remove_import_format_support_query(Ref<EditorFileSystemImportFormatSupportQuery> p_query) { import_support_queries.erase(p_query); } EditorFileSystem::EditorFileSystem() { #ifdef THREADS_ENABLED use_threads = true; #endif ResourceLoader::import = _resource_import; reimport_on_missing_imported_files = GLOBAL_GET("editor/import/reimport_missing_imported_files"); singleton = this; filesystem = memnew(EditorFileSystemDirectory); //like, empty filesystem->parent = nullptr; new_filesystem = nullptr; // This should probably also work on Unix and use the string it returns for FAT32 or exFAT Ref<DirAccess> da = DirAccess::create(DirAccess::ACCESS_RESOURCES); using_fat32_or_exfat = (da->get_filesystem_type() == "FAT32" || da->get_filesystem_type() == "exFAT"); scan_total = 0; ResourceSaver::set_get_resource_id_for_path(_resource_saver_get_resource_id_for_path); // Set the callback method that the ResourceFormatImporter will use // if resources are loaded during the first scan. ResourceImporter::load_on_startup = _load_resource_on_startup; } EditorFileSystem::~EditorFileSystem() { if (filesystem) { memdelete(filesystem); } filesystem = nullptr; ResourceSaver::set_get_resource_id_for_path(nullptr); }