diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt index 110ff1152d0..9a5b894c48f 100644 --- a/Core/CMakeLists.txt +++ b/Core/CMakeLists.txt @@ -14,6 +14,7 @@ target_include_directories(corei_main INTERFACE "Main") target_sources(corei_libraries_include PRIVATE Libraries/Include/Lib/BaseType.h Libraries/Include/Lib/BaseTypeCore.h + Libraries/Include/Lib/PathUtil.h Libraries/Include/Lib/trig.h Libraries/Include/rts/debug.h Libraries/Include/rts/profile.h diff --git a/Core/GameEngine/CMakeLists.txt b/Core/GameEngine/CMakeLists.txt index 8af78e0e16c..e9a25715d09 100644 --- a/Core/GameEngine/CMakeLists.txt +++ b/Core/GameEngine/CMakeLists.txt @@ -594,6 +594,7 @@ set(GAMEENGINE_SRC Source/Common/INI/INIDamageFX.cpp Source/Common/INI/INIDrawGroupInfo.cpp Source/Common/INI/INIGameData.cpp + Source/Common/INI/INIFileSystem.cpp Source/Common/INI/INIMapCache.cpp Source/Common/INI/INIMapData.cpp Source/Common/INI/INIMappedImage.cpp diff --git a/Core/GameEngine/Include/Common/ArchiveFile.h b/Core/GameEngine/Include/Common/ArchiveFile.h index 63987f9c9cb..5900b1fa158 100644 --- a/Core/GameEngine/Include/Common/ArchiveFile.h +++ b/Core/GameEngine/Include/Common/ArchiveFile.h @@ -46,6 +46,8 @@ class File; class ArchiveFile { + typedef std::hash_map ArchivedFileInfoPtrHashMap; // Archived file name to archived file info ptr + public: ArchiveFile(); virtual ~ArchiveFile(); @@ -60,13 +62,22 @@ class ArchiveFile void attachFile(File *file); void getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; - void getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; void addFile(const AsciiString& path, const ArchivedFileInfo *fileInfo); ///< add this file to our directory tree. + Bool ignoreFile(const AsciiString& filename, Bool ignore); + Bool ignoreDirectory(const AsciiString& directory, Bool ignore); protected: + void getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; const ArchivedFileInfo * getArchivedFileInfo(const AsciiString& filename) const; ///< return the ArchivedFileInfo from the directory tree. +private: + void ignoreDirectory(DetailedArchivedDirectoryInfo *dirInfo, Bool ignore); + +protected: File *m_file; ///< file pointer to the archive file on disk. Kept open so we don't have to continuously open and close the file all the time. + +private: DetailedArchivedDirectoryInfo m_rootDirectory; + ArchivedFileInfoPtrHashMap m_absFilenameToFileInfo; }; diff --git a/Core/GameEngine/Include/Common/ArchiveFileSystem.h b/Core/GameEngine/Include/Common/ArchiveFileSystem.h index af2321d4e25..78c2f1497b5 100644 --- a/Core/GameEngine/Include/Common/ArchiveFileSystem.h +++ b/Core/GameEngine/Include/Common/ArchiveFileSystem.h @@ -102,6 +102,12 @@ class DetailedArchivedDirectoryInfo AsciiString m_directoryName; DetailedArchivedDirectoryInfoMap m_directories; ArchivedFileInfoMap m_files; + Bool m_ignore; + + DetailedArchivedDirectoryInfo() + : m_ignore(false) + { + } }; class ArchivedFileInfo @@ -111,10 +117,12 @@ class ArchivedFileInfo AsciiString m_archiveFilename; UnsignedInt m_offset; UnsignedInt m_size; + Bool m_ignore; ArchivedFileInfo() : m_offset(0) , m_size(0) + , m_ignore(false) { } }; @@ -143,11 +151,11 @@ class ArchiveFileSystem : public SubsystemInterface virtual Bool loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0; - // Unprotected this for copy-protection routines - ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const; - void loadMods(); + Bool ignoreFile(const AsciiString& filename, Bool ignore = true); ///< Ignore all file instances with this name in the Archive System. + Bool ignoreDirectory(const AsciiString& directory, Bool ignore = true); ///< Ignore this directory and all its contents in the Archive System. + ArchivedDirectoryInfo* friend_getArchivedDirectoryInfo(const Char* directory); protected: @@ -160,6 +168,7 @@ class ArchiveFileSystem : public SubsystemInterface AsciiString lastToken; ///< Synonymous for file name if the search directory was a file path }; + ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const; ArchivedDirectoryInfoResult getArchivedDirectoryInfo(const Char* directory); virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool overwrite = FALSE); ///< load the archive file's header information and apply it to the global archive directory tree. diff --git a/Core/GameEngine/Include/Common/FileSystem.h b/Core/GameEngine/Include/Common/FileSystem.h index 2aaa30a61f1..01b1b22dae8 100644 --- a/Core/GameEngine/Include/Common/FileSystem.h +++ b/Core/GameEngine/Include/Common/FileSystem.h @@ -63,7 +63,7 @@ // Type Defines //---------------------------------------------------------------------------- -typedef std::set/**/> FilenameList; +typedef std::set FilenameList; typedef FilenameList::iterator FilenameListIter; typedef UnsignedByte FileInstance; @@ -136,6 +136,10 @@ struct FileInfo { // from multiple threads. // // TheSuperHackers @feature Mauller 24/04/2026 Add extension removal functions +// +// TheSuperHackers @feature xezon 26/07/2026 Implements ignore functionality for files and directories in the local +// and archive file systems. Can be called at any time during engine runtime. Can be controlled by data with +// FileSystemIgnore definitions in Data/INI/FileSystem.ini . //=============================== class FileSystem : public SubsystemInterface { @@ -157,6 +161,9 @@ class FileSystem : public SubsystemInterface Bool createDirectory(AsciiString directory); ///< create a directory of the given name. + Bool ignoreFile(const AsciiString& filename, Bool ignore = true); ///< Ignore this file. + Bool ignoreDirectory(const AsciiString& directory, Bool ignore = true); ///< Ignore this directory and all its contents. + static AsciiString normalizePath(const AsciiString& path); ///< normalizes a file path. The path can refer to a directory. File path must be absolute, but does not need to exist. Returns an empty string on failure. static Bool isPathInDirectory(const AsciiString& testPath, const AsciiString& basePath); ///< determines if a file path is within a base path. Both paths must be absolute, but do not need to exist. @@ -173,8 +180,8 @@ class FileSystem : public SubsystemInterface }; typedef std::hash_map< rts::string_key, FileExistData, - rts::string_key_hash, - rts::string_key_equal/**/> FileExistMap; + rts::string_key_hash_path, + rts::string_key_equal_to_path> FileExistMap; mutable FileExistMap m_fileExist; mutable FastCriticalSectionClass m_fileExistMutex; diff --git a/Core/GameEngine/Include/Common/GameCommon.h b/Core/GameEngine/Include/Common/GameCommon.h index e220b91e8c2..25f3ce47256 100644 --- a/Core/GameEngine/Include/Common/GameCommon.h +++ b/Core/GameEngine/Include/Common/GameCommon.h @@ -51,6 +51,7 @@ // ---------------------------------------------------------------------------------------------- #include "Lib/BaseType.h" +#include "Lib/PathUtil.h" #include "WWLib/WWCommon.h" #include "Common/GameDefines.h" diff --git a/Core/GameEngine/Include/Common/INI.h b/Core/GameEngine/Include/Common/INI.h index e53b8b9bc6c..31c48c2a94a 100644 --- a/Core/GameEngine/Include/Common/INI.h +++ b/Core/GameEngine/Include/Common/INI.h @@ -164,17 +164,29 @@ class INI INI& operator=(const INI&) FUNCTION_DELETE; public: + + typedef UnsignedInt LoadFlags; + enum LoadFlags_ CPP_11( : UnsignedInt) + { + LoadFlags_SearchSubDirs = 1 << 0, + LoadFlags_ExpectFileFound = 1 << 1, + }; + +public: + INI(); // TheSuperHackers @feature xezon 19/08/2025 // Load a specific INI file by name and/or INI files from a directory (and its subdirectories). // For example "Data\INI\Armor" loads "Data\INI\Armor.ini" and all *.ini files in "Data\INI\Armor". // Throws if not a single INI file is found or one is not read correctly. - UnsignedInt loadFileDirectory( AsciiString fileDirName, INILoadType loadType, Xfer *pXfer, Bool subdirs = TRUE ); + UnsignedInt loadFileDirectory( AsciiString fileDirName, INILoadType loadType, Xfer *pXfer, + LoadFlags loadFlags = LoadFlags_SearchSubDirs | LoadFlags_ExpectFileFound ); // Load INI files from a directory (and its subdirectories). // Throws if one INI file is not read correctly. - UnsignedInt loadDirectory( AsciiString dirName, INILoadType loadType, Xfer *pXfer, Bool subdirs = TRUE ); + UnsignedInt loadDirectory( AsciiString dirName, INILoadType loadType, Xfer *pXfer, + LoadFlags loadFlags = LoadFlags_SearchSubDirs ); // Load one specific INI file by name. // Throws if the INI file is not found or is not read correctly. @@ -213,6 +225,7 @@ class INI static void parseTerrainRoadDefinition( INI *ini ); static void parseTerrainBridgeDefinition( INI *ini ); static void parseMetaMapDefinition( INI *ini ); + static void parseFileSystemIgnoreDefinition( INI* ini ); static void parseFXListDefinition( INI *ini ); static void parseObjectCreationListDefinition( INI* ini ); static void parseMultiplayerSettingsDefinition( INI* ini ); diff --git a/Core/GameEngine/Include/Common/LocalFileSystem.h b/Core/GameEngine/Include/Common/LocalFileSystem.h index c20887c3f50..5f0b43551cc 100644 --- a/Core/GameEngine/Include/Common/LocalFileSystem.h +++ b/Core/GameEngine/Include/Common/LocalFileSystem.h @@ -33,6 +33,22 @@ class LocalFileSystem : public SubsystemInterface { + struct IgnoreFileData + { + }; + + typedef std::hash_map< + rts::string_key, IgnoreFileData, + rts::string_key_hash_path, + rts::string_key_equal_to_path> IgnoreFileHashMap; + +protected: + typedef Int IgnoreFileTestFlags; + enum IgnoreFileTestFlags_ + { + IgnoreFileTestFlags_SkipParentDirectories = 1 << 0, + }; + public: virtual ~LocalFileSystem() override {} @@ -43,7 +59,27 @@ class LocalFileSystem : public SubsystemInterface virtual Bool createDirectory(AsciiString directory) = 0; ///< see FileSystem.h virtual AsciiString normalizePath(const AsciiString& filePath) const = 0; ///< see FileSystem.h + Bool ignoreFile(const AsciiString& filename, Bool ignore = true); ///< Ignore this file on the disk. + Bool ignoreDirectory(const AsciiString& directory, Bool ignore = true); ///< Ignore this directory and all its contents on the disk. + protected: + Bool isFileIgnored(const Char* filename, IgnoreFileTestFlags flags = 0) const; ///< Whether the given file or its parent directories are ignored. + Bool isDirectoryIgnored(const Char* directory, IgnoreFileTestFlags flags = 0) const; ///< Whether the given directory or its parent directories are ignored. + +private: + Bool isDirectoryIgnoredRecursive(AsciiString& directory, IgnoreFileTestFlags flags) const; + Bool isFileIgnoredInternal(const Char* filename) const; + Bool isDirectoryIgnoredInternal(const Char* directory) const; + Bool hasIgnoredFile() const; + Bool hasIgnoredDirectory() const; + + static void trimLastPathSegmentInplace(AsciiString& path); ///< Removes trailing path segment until and including the last slash. + static void trimTrailingSlash(AsciiString& path); ///< Remove trailing forward or backward slashes. + + IgnoreFileHashMap m_ignoreFileHashMap; + IgnoreFileHashMap m_ignoreDirectoryHashMap; + mutable FastCriticalSectionClass m_ignoreFileMutex; + mutable FastCriticalSectionClass m_ignoreDirectoryMutex; }; extern LocalFileSystem *TheLocalFileSystem; diff --git a/Core/GameEngine/Include/Common/STLTypedefs.h b/Core/GameEngine/Include/Common/STLTypedefs.h index 7276a459c3f..9be4711177a 100644 --- a/Core/GameEngine/Include/Common/STLTypedefs.h +++ b/Core/GameEngine/Include/Common/STLTypedefs.h @@ -114,7 +114,7 @@ namespace rts // specific types. template struct hash { - size_t operator()(const T& __t) const + size_t operator()(const T& __t) const noexcept { std::hash tmp; return tmp(__t); @@ -126,7 +126,7 @@ namespace rts // the case of pointers.) template struct equal_to { - Bool operator()(const T& __t1, const T& __t2) const + Bool operator()(const T& __t1, const T& __t2) const noexcept { return (__t1 == __t2); } @@ -137,7 +137,7 @@ namespace rts // the case of pointers, or strings.) template struct less_than_nocase { - bool operator()(const T& __t1, const T& __t2) const + bool operator()(const T& __t1, const T& __t2) const noexcept { return (__t1 < __t2); } @@ -146,7 +146,7 @@ namespace rts #ifdef USING_STLPORT template<> struct hash { - size_t operator()(NameKeyType nkt) const + size_t operator()(NameKeyType nkt) const noexcept { std::hash tmp; return tmp((UnsignedInt)nkt); @@ -155,7 +155,7 @@ namespace rts template<> struct hash { - size_t operator()(DrawableID nkt) const + size_t operator()(DrawableID nkt) const noexcept { std::hash tmp; return tmp((UnsignedInt)nkt); @@ -164,7 +164,7 @@ namespace rts template<> struct hash { - size_t operator()(ObjectID nkt) const + size_t operator()(ObjectID nkt) const noexcept { std::hash tmp; return tmp((UnsignedInt)nkt); @@ -173,7 +173,7 @@ namespace rts template<> struct hash { - size_t operator()(ParticleSystemID nkt) const + size_t operator()(ParticleSystemID nkt) const noexcept { std::hash tmp; return tmp((UnsignedInt)nkt); @@ -183,7 +183,7 @@ namespace rts template<> struct hash { - size_t operator()(const Char* s) const + size_t operator()(const Char* s) const noexcept { #ifdef USING_STLPORT std::hash hasher; @@ -201,7 +201,7 @@ namespace rts // they are to be used in lots of places.) template<> struct equal_to { - Bool operator()(const char* s1, const char* s2) const + Bool operator()(const char* s1, const char* s2) const noexcept { return strcmp(s1, s2) == 0; } @@ -209,13 +209,12 @@ namespace rts template<> struct hash { - size_t operator()(const AsciiString& ast) const + size_t operator()(const AsciiString& ast) const noexcept { #ifdef USING_STLPORT std::hash tmp; return tmp((const char *) ast.str()); #else - // TheSuperHackers @bugfix xezon 16/03/2024 Re-implements hash function that works with non-STLPort. std::hash hasher; return hasher(std::string_view(ast.str(), ast.getLength())); #endif @@ -224,7 +223,7 @@ namespace rts template<> struct equal_to { - Bool operator()(const AsciiString& __t1, const AsciiString& __t2) const + Bool operator()(const AsciiString& __t1, const AsciiString& __t2) const noexcept { return (__t1 == __t2); } @@ -232,7 +231,7 @@ namespace rts template<> struct less_than_nocase { - bool operator()(const AsciiString& __t1, const AsciiString& __t2) const + bool operator()(const AsciiString& __t1, const AsciiString& __t2) const noexcept { return (__t1.compareNoCase(__t2) < 0); } @@ -240,12 +239,54 @@ namespace rts template<> struct less_than_nocase { - bool operator()(const UnicodeString& __t1, const UnicodeString& __t2) const + bool operator()(const UnicodeString& __t1, const UnicodeString& __t2) const noexcept { return (__t1.compareNoCase(__t2) < 0); } }; + // TheSuperHackers @info Functor to compare equal paths with. Ignores case, type of slash and trailing slash. + struct equal_to_path + { + bool operator()(const char* __t1, const char* __t2) const noexcept + { + return comparePath(__t1, __t2) == 0; + } + + bool operator()(const AsciiString& __t1, const AsciiString& __t2) const noexcept + { + return comparePath(__t1.str(), __t2.str()) == 0; + } + }; + + // TheSuperHackers @info Functor to compare lesser paths with. Ignores case, type of slash and trailing slash. + struct less_than_path + { + bool operator()(const char* __t1, const char* __t2) const noexcept + { + return comparePath(__t1, __t2) < 0; + } + + bool operator()(const AsciiString& __t1, const AsciiString& __t2) const noexcept + { + return comparePath(__t1.str(), __t2.str()) < 0; + } + }; + + // TheSuperHackers @info Functor to hash paths with. Ignores case, type of slash and trailing slash. + struct hash_path + { + size_t operator()(const char* str) const noexcept + { + return hashPath(str); + } + + size_t operator()(const AsciiString& str) const noexcept + { + return hashPath(str.str()); + } + }; + // TheSuperHackers @info Structs to help create maps that can use C strings for // lookups without the need to allocate a string. template @@ -282,23 +323,42 @@ namespace rts const_pointer cstr; }; - template struct string_key_hash { - typedef typename String::const_pointer const_pointer; - size_t operator()(const string_key& key) const + template + size_t operator()(const string_key& key) const noexcept { + typedef typename String::const_pointer const_pointer; return hash()(key.c_str()); } }; - template struct string_key_equal { - bool operator()(const string_key& a, const string_key& b) const + template + bool operator()(const string_key& a, const string_key& b) const noexcept { return strcmp(a.c_str(), b.c_str()) == 0; } }; + + struct string_key_hash_path + { + template + size_t operator()(const string_key& key) const noexcept + { + return hashPath(key.c_str()); + } + }; + + struct string_key_equal_to_path + { + template + bool operator()(const string_key& a, const string_key& b) const noexcept + { + return comparePath(a.c_str(), b.c_str()) == 0; + } + }; + } // namespace rts diff --git a/Core/GameEngine/Include/Common/UnicodeString.h b/Core/GameEngine/Include/Common/UnicodeString.h index 0f3c2333c44..de55a12da4c 100644 --- a/Core/GameEngine/Include/Common/UnicodeString.h +++ b/Core/GameEngine/Include/Common/UnicodeString.h @@ -348,7 +348,7 @@ class UnicodeString token was found. (note that this modifies 'this' as well, stripping the token off!) */ - Bool nextToken(UnicodeString* token, UnicodeString delimiters = UnicodeString::TheEmptyString); + Bool nextToken(UnicodeString* token, const WideChar* separators = nullptr); // // You might think it would be a good idea to overload the * operator diff --git a/Core/GameEngine/Source/Common/INI/INI.cpp b/Core/GameEngine/Source/Common/INI/INI.cpp index b3a26a5f4c1..7e47745fc38 100644 --- a/Core/GameEngine/Source/Common/INI/INI.cpp +++ b/Core/GameEngine/Source/Common/INI/INI.cpp @@ -110,6 +110,7 @@ static const BlockParse theTypeTable[] = { "DrawGroupInfo", INI::parseDrawGroupNumberDefinition }, { "DynamicGameLOD", INI::parseDynamicGameLODDefinition }, { "EvaEvent", INI::parseEvaEvent }, + { "FileSystemIgnore", INI::parseFileSystemIgnoreDefinition }, { "FXList", INI::parseFXListDefinition }, { "GameData", INI::parseGameDataDefinition }, { "HeaderTemplate", INI::parseHeaderTemplateDefinition }, @@ -189,7 +190,7 @@ INI::INI() } //------------------------------------------------------------------------------------------------- -UnsignedInt INI::loadFileDirectory( AsciiString fileDirName, INILoadType loadType, Xfer *pXfer, Bool subdirs ) +UnsignedInt INI::loadFileDirectory( AsciiString fileDirName, INILoadType loadType, Xfer *pXfer, LoadFlags loadFlags ) { UnsignedInt filesRead = 0; @@ -214,10 +215,11 @@ UnsignedInt INI::loadFileDirectory( AsciiString fileDirName, INILoadType loadTyp } // Load any additional ini files from a "filename" directory and its subdirectories. - filesRead += loadDirectory(iniDir, loadType, pXfer, subdirs); + filesRead += loadDirectory(iniDir, loadType, pXfer, loadFlags & ~LoadFlags_ExpectFileFound); // Expect to open and load at least one file. - if (filesRead == 0) + const Bool expectFileFound = (loadFlags & LoadFlags_ExpectFileFound) != 0; + if (expectFileFound && filesRead == 0) { throw INI_CANT_OPEN_FILE; } @@ -230,7 +232,7 @@ UnsignedInt INI::loadFileDirectory( AsciiString fileDirName, INILoadType loadTyp * If we are to load subdirectories, we will load them *after* we load all the * files in the current directory */ //------------------------------------------------------------------------------------------------- -UnsignedInt INI::loadDirectory( AsciiString dirName, INILoadType loadType, Xfer *pXfer, Bool subdirs ) +UnsignedInt INI::loadDirectory( AsciiString dirName, INILoadType loadType, Xfer *pXfer, LoadFlags loadFlags ) { UnsignedInt filesRead = 0; @@ -238,6 +240,7 @@ UnsignedInt INI::loadDirectory( AsciiString dirName, INILoadType loadType, Xfer if( dirName.isEmpty() ) throw INI_INVALID_DIRECTORY; + const Bool subdirs = (loadFlags & LoadFlags_SearchSubDirs) != 0; FilenameList filenameList; dirName.concat('\\'); TheFileSystem->getFileListInDirectory(dirName, "*.ini", filenameList, subdirs); @@ -268,6 +271,13 @@ UnsignedInt INI::loadDirectory( AsciiString dirName, INILoadType loadType, Xfer ++it; } + // Expect to open and load at least one file. + const Bool expectFileFound = (loadFlags & LoadFlags_ExpectFileFound) != 0; + if (expectFileFound && filesRead == 0) + { + throw INI_CANT_OPEN_FILE; + } + return filesRead; } diff --git a/Core/GameEngine/Source/Common/INI/INIFileSystem.cpp b/Core/GameEngine/Source/Common/INI/INIFileSystem.cpp new file mode 100644 index 00000000000..e31b51d066e --- /dev/null +++ b/Core/GameEngine/Source/Common/INI/INIFileSystem.cpp @@ -0,0 +1,86 @@ +/* +** Command & Conquer Generals Zero Hour(tm) +** Copyright 2026 TheSuperHackers +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +#include "PreRTS.h" // This must go first in EVERY cpp file in the GameEngine + +#include "Common/FileSystem.h" +#include "Common/INI.h" + +struct FileSystemIgnore +{ + enum Type + { + Auto, + File, + Dir, + }; + + FileSystemIgnore() : m_type(Auto) {} + + AsciiString m_name; + Type m_type; + + static const FieldParse s_fieldParseTable[]; + static const char *const s_typeNames[]; +}; + +const char *const FileSystemIgnore::s_typeNames[] = +{ + "AUTO", + "FILE", + "DIR", + nullptr +}; + +const FieldParse FileSystemIgnore::s_fieldParseTable[] = +{ + { "Name", INI::parseQuotedAsciiString, nullptr, offsetof(FileSystemIgnore, m_name) }, + { "Type", INI::parseIndexList, s_typeNames, offsetof(FileSystemIgnore, m_type) }, + { nullptr, nullptr, nullptr, 0 } +}; + +void INI::parseFileSystemIgnoreDefinition(INI* ini) +{ + DEBUG_ASSERTCRASH(TheFileSystem != nullptr, ("TheFileSystem is null")); + + FileSystemIgnore fsIgnore; + ini->initFromINI(&fsIgnore, fsIgnore.s_fieldParseTable); + + if (fsIgnore.m_type == FileSystemIgnore::Auto) + { + if (isFilePath(fsIgnore.m_name.str())) + fsIgnore.m_type = FileSystemIgnore::File; + else + fsIgnore.m_type = FileSystemIgnore::Dir; + } + + switch (fsIgnore.m_type) + { + case FileSystemIgnore::File: + TheFileSystem->ignoreFile(fsIgnore.m_name); + break; + + case FileSystemIgnore::Dir: + TheFileSystem->ignoreDirectory(fsIgnore.m_name); + break; + + default: + DEBUG_CRASH(("Unhandled case in parseFileSystemIgnoreDefinition")); + break; + } +} diff --git a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp index 1f982b65a1d..d6cd6fd267d 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp @@ -106,20 +106,79 @@ void ArchiveFile::addFile(const AsciiString& path, const ArchivedFileInfo *fileI while (!token.isEmpty()) { DetailedArchivedDirectoryInfoMap::iterator tempiter = dirInfo->m_directories.find(token); - if (tempiter == dirInfo->m_directories.end()) - { + if (tempiter == dirInfo->m_directories.end()) { dirInfo = &(dirInfo->m_directories[token]); dirInfo->m_directoryName = token; } - else - { + else { dirInfo = &tempiter->second; } tokenizer.nextToken(&token, "\\/"); } - dirInfo->m_files[fileInfo->m_filename] = *fileInfo; + typedef std::pair Result; + const Result result = dirInfo->m_files.insert(std::make_pair(fileInfo->m_filename, *fileInfo)); + if (result.second) { + AsciiString filename = path; + filename.concat(fileInfo->m_filename); + m_absFilenameToFileInfo[filename] = &result.first->second; + + static_assert((std::is_same_vm_files), std::map>), "Hash map expects stable references"); + } +} + +Bool ArchiveFile::ignoreFile(const AsciiString& filename, Bool ignore) +{ + ArchivedFileInfoPtrHashMap::iterator it = m_absFilenameToFileInfo.find(filename); + if (it == m_absFilenameToFileInfo.end()) { + return false; + } + + it->second->m_ignore = ignore; + return true; +} + +Bool ArchiveFile::ignoreDirectory(const AsciiString& directory, Bool ignore) +{ + DetailedArchivedDirectoryInfo *dirInfo = &m_rootDirectory; + + AsciiString token; + AsciiString tokenizer = directory; + tokenizer.toLower(); + tokenizer.nextToken(&token, "\\/"); + + while (!token.isEmpty()) { + DetailedArchivedDirectoryInfoMap::iterator it = dirInfo->m_directories.find(token); + if (it == dirInfo->m_directories.end()) + return false; + dirInfo = &it->second; + tokenizer.nextToken(&token, "\\/"); + } + + ignoreDirectory(dirInfo, ignore); + return true; +} + +void ArchiveFile::ignoreDirectory(DetailedArchivedDirectoryInfo *dirInfo, Bool ignore) +{ + // Ignore this directory + dirInfo->m_ignore = ignore; + + // Ignore all files in this directory + ArchivedFileInfoMap::iterator fileIt = dirInfo->m_files.begin(); + for (; fileIt != dirInfo->m_files.end(); ++fileIt) { + ArchivedFileInfo &fileInfo = fileIt->second; + fileInfo.m_ignore = ignore; + } + + // Ignore subdirectories and its files + DetailedArchivedDirectoryInfoMap::iterator dirIt = dirInfo->m_directories.begin(); + for (; dirIt != dirInfo->m_directories.end(); ++dirIt) { + DetailedArchivedDirectoryInfo *tempDirInfo = &(dirIt->second); + tempDirInfo->m_ignore = ignore; + ignoreDirectory(tempDirInfo, ignore); + } } void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const @@ -132,17 +191,14 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co tokenizer.nextToken(&token, "\\/"); while (!token.isEmpty()) { - DetailedArchivedDirectoryInfoMap::const_iterator it = dirInfo->m_directories.find(token); - if (it != dirInfo->m_directories.end()) - { - dirInfo = &it->second; - } - else - { - // if the directory doesn't exist, then there aren't any files to be had. + // if the directory doesn't exist, then there aren't any files to be had. + if (it == dirInfo->m_directories.end()) + return; + + dirInfo = &it->second; + if (dirInfo->m_ignore) return; - } tokenizer.nextToken(&token, "\\/"); } @@ -153,33 +209,35 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co void ArchiveFile::getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const { DetailedArchivedDirectoryInfoMap::const_iterator diriter = dirInfo->m_directories.begin(); - while (diriter != dirInfo->m_directories.end()) { + for (; diriter != dirInfo->m_directories.end(); ++diriter) { const DetailedArchivedDirectoryInfo *tempDirInfo = &(diriter->second); - AsciiString tempdirname; - tempdirname = currentDirectory; - if ((!tempdirname.isEmpty()) && (!tempdirname.endsWith("\\"))) { - tempdirname.concat('\\'); + if (tempDirInfo->m_ignore) + continue; + AsciiString tempDirName = currentDirectory; + if (!tempDirName.isEmpty() && !tempDirName.endsWith("\\")) { + tempDirName.concat('\\'); } - tempdirname.concat(tempDirInfo->m_directoryName); - getFileListInDirectory(tempDirInfo, tempdirname, searchName, filenameList, searchSubdirectories); - diriter++; + tempDirName.concat(tempDirInfo->m_directoryName); + getFileListInDirectory(tempDirInfo, tempDirName, searchName, filenameList, searchSubdirectories); } ArchivedFileInfoMap::const_iterator fileiter = dirInfo->m_files.begin(); - while (fileiter != dirInfo->m_files.end()) { - if (SearchStringMatches(fileiter->second.m_filename, searchName)) { + for (; fileiter != dirInfo->m_files.end(); ++fileiter) { + const ArchivedFileInfo &fileInfo = fileiter->second; + if (fileInfo.m_ignore) + continue; + if (SearchStringMatches(fileInfo.m_filename, searchName)) { AsciiString tempfilename; tempfilename = currentDirectory; - if ((!tempfilename.isEmpty()) && (!tempfilename.endsWith("\\"))) { + if (!tempfilename.isEmpty() && !tempfilename.endsWith("\\")) { tempfilename.concat('\\'); } - tempfilename.concat(fileiter->second.m_filename); + tempfilename.concat(fileInfo.m_filename); if (filenameList.find(tempfilename) == filenameList.end()) { // only insert into the list if its not already in there. filenameList.insert(tempfilename); } } - fileiter++; } } @@ -194,36 +252,12 @@ void ArchiveFile::attachFile(File *file) const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& filename) const { - const DetailedArchivedDirectoryInfo *dirInfo = &m_rootDirectory; - - AsciiString token; - AsciiString tokenizer = filename; - tokenizer.toLower(); - tokenizer.nextToken(&token, "\\/"); - - while (!token.find('.') || tokenizer.find('.')) - { - DetailedArchivedDirectoryInfoMap::const_iterator it = dirInfo->m_directories.find(token); - if (it != dirInfo->m_directories.end()) - { - dirInfo = &it->second; - } - else - { - return nullptr; - } - - tokenizer.nextToken(&token, "\\/"); - } + ArchivedFileInfoPtrHashMap::const_iterator it = m_absFilenameToFileInfo.find(filename); + if (it == m_absFilenameToFileInfo.end()) + return nullptr; - ArchivedFileInfoMap::const_iterator it = dirInfo->m_files.find(token); - if (it != dirInfo->m_files.end()) - { - return &it->second; - } - else - { + if (it->second->m_ignore) return nullptr; - } + return it->second; } diff --git a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp index f118b34bad0..49f3fc4f731 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp @@ -236,6 +236,32 @@ void ArchiveFileSystem::loadMods() } } +Bool ArchiveFileSystem::ignoreFile( const AsciiString& filename, Bool ignore ) +{ + Bool wasIgnored = false; + + if (!filename.isEmpty()){ + ArchiveFileMap::const_iterator it = m_archiveFileMap.begin(); + for (; it != m_archiveFileMap.end(); ++it) { + wasIgnored |= it->second->ignoreFile(filename, ignore); + } + } + return wasIgnored; +} + +Bool ArchiveFileSystem::ignoreDirectory( const AsciiString& directory, Bool ignore ) +{ + Bool wasIgnored = false; + + if (!directory.isEmpty()) { + ArchiveFileMap::const_iterator it = m_archiveFileMap.begin(); + for (; it != m_archiveFileMap.end(); ++it) { + wasIgnored |= it->second->ignoreDirectory(directory, ignore); + } + } + return wasIgnored; +} + Bool ArchiveFileSystem::doesFileExist(const Char *filename, FileInstance instance) const { ArchivedDirectoryInfoResult result = const_cast(this)->getArchivedDirectoryInfo(filename); @@ -245,7 +271,14 @@ Bool ArchiveFileSystem::doesFileExist(const Char *filename, FileInstance instanc stl::const_range range = stl::get_range(result.dirInfo->m_files, result.lastToken, instance); - return range.valid(); + if (!range.valid()) + return false; + + FileInfo fileInfo; + ArchiveFile *archiveFile = range.get()->second; + + // Returns null if the file in the archive is ignored. + return archiveFile->getFileInfo(filename, &fileInfo); } ArchivedDirectoryInfo* ArchiveFileSystem::friend_getArchivedDirectoryInfo(const Char* directory) diff --git a/Core/GameEngine/Source/Common/System/AsciiString.cpp b/Core/GameEngine/Source/Common/System/AsciiString.cpp index 2602b2ed67b..881d3f0dd5d 100644 --- a/Core/GameEngine/Source/Common/System/AsciiString.cpp +++ b/Core/GameEngine/Source/Common/System/AsciiString.cpp @@ -572,8 +572,13 @@ Bool AsciiString::isNone() const //----------------------------------------------------------------------------- Bool AsciiString::nextToken(AsciiString* tok, const char* seps) { - if (this->isEmpty() || tok == this) + DEBUG_ASSERTCRASH(tok != this, ("Tokenizer and Token cannot be the same object")); + + if (this->isEmpty()) + { + tok->clear(); return false; + } if (seps == nullptr) seps = " \n\r\t"; diff --git a/Core/GameEngine/Source/Common/System/FileSystem.cpp b/Core/GameEngine/Source/Common/System/FileSystem.cpp index b8e4c4695b6..e75c5e97b84 100644 --- a/Core/GameEngine/Source/Common/System/FileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/FileSystem.cpp @@ -54,8 +54,6 @@ #include "Common/LocalFileSystem.h" #include "Common/PerfTimer.h" -#include "Lib/PathUtil.h" - DECLARE_PERF_TIMER(FileSystem) @@ -331,6 +329,52 @@ Bool FileSystem::createDirectory(AsciiString directory) return FALSE; } +//============================================================================ +Bool FileSystem::ignoreFile(const AsciiString& filename, Bool ignore) +{ + Bool wasIgnored = FALSE; + wasIgnored |= TheLocalFileSystem->ignoreFile(filename, ignore); + wasIgnored |= TheArchiveFileSystem->ignoreFile(filename, ignore); + +#if ENABLE_FILESYSTEM_EXISTENCE_CACHE + if (wasIgnored) + { + // Remove this file from the existence cache. + FastCriticalSectionClass::LockClass lock(m_fileExistMutex); + m_fileExist.erase(filename); + } +#endif + + return wasIgnored; +} + +//============================================================================ +Bool FileSystem::ignoreDirectory(const AsciiString& directory, Bool ignore) +{ + Bool wasIgnored = FALSE; + wasIgnored |= TheLocalFileSystem->ignoreDirectory(directory, ignore); + wasIgnored |= TheArchiveFileSystem->ignoreDirectory(directory, ignore); + +#if ENABLE_FILESYSTEM_EXISTENCE_CACHE + if (wasIgnored) + { + // Remove all relevant files from the existence cache. + FastCriticalSectionClass::LockClass lock(m_fileExistMutex); + FileExistMap::const_iterator it = m_fileExist.begin(); + while (it != m_fileExist.end()) + { + FileExistMap::const_iterator curr = it++; + if (startsWithPath(curr->first.c_str(), directory.str())) + { + m_fileExist.erase(curr->first); + } + } + } +#endif + + return wasIgnored; +} + //============================================================================ // FileSystem::normalizePath //============================================================================ diff --git a/Core/GameEngine/Source/Common/System/LocalFileSystem.cpp b/Core/GameEngine/Source/Common/System/LocalFileSystem.cpp index 0d45847ce78..3f89b6cdde5 100644 --- a/Core/GameEngine/Source/Common/System/LocalFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/LocalFileSystem.cpp @@ -87,8 +87,167 @@ LocalFileSystem *TheLocalFileSystem = nullptr; // Private Functions //---------------------------------------------------------------------------- - +void LocalFileSystem::trimLastPathSegmentInplace(AsciiString& path) +{ + if (const char* end = maxPtr(path.reverseFind('\\'), path.reverseFind('/'))) + { + path.truncateTo(end - path.str()); + } + else + { + path.clear(); + } +} + +void LocalFileSystem::trimTrailingSlash(AsciiString& path) +{ + path.trimEnd('\\'); + path.trimEnd('/'); +} //---------------------------------------------------------------------------- // Public Functions //---------------------------------------------------------------------------- + +Bool LocalFileSystem::ignoreFile(const AsciiString& filename, Bool ignore) +{ + if (filename.isEmpty()) + return false; + + if (ignore) + { + FastCriticalSectionClass::LockClass lock(m_ignoreFileMutex); + m_ignoreFileHashMap.insert(std::make_pair(filename, IgnoreFileData())); + } + else if (!m_ignoreFileHashMap.empty()) + { + FastCriticalSectionClass::LockClass lock(m_ignoreFileMutex); + m_ignoreFileHashMap.erase(filename); + } + return true; +} + +Bool LocalFileSystem::ignoreDirectory(const AsciiString& directory, Bool ignore) +{ + if (directory.isEmpty()) + return false; + + if (ignore) + { + FastCriticalSectionClass::LockClass lock(m_ignoreDirectoryMutex); + m_ignoreDirectoryHashMap[directory]; + } + else + { + // Lift ignore on this directory and subdirectories + { + FastCriticalSectionClass::LockClass lock(m_ignoreDirectoryMutex); + IgnoreFileHashMap::const_iterator it = m_ignoreDirectoryHashMap.begin(); + while (it != m_ignoreDirectoryHashMap.end()) + { + IgnoreFileHashMap::const_iterator curr = it++; + if (startsWithPath(curr->first.c_str(), directory.str())) + { + m_ignoreDirectoryHashMap.erase(curr->first); + } + } + } + + // Also lift ignore on files in the directory + { + FastCriticalSectionClass::LockClass lock(m_ignoreFileMutex); + IgnoreFileHashMap::const_iterator it = m_ignoreFileHashMap.begin(); + while (it != m_ignoreFileHashMap.end()) + { + IgnoreFileHashMap::const_iterator curr = it++; + if (startsWithPath(curr->first.c_str(), directory.str())) + { + m_ignoreFileHashMap.erase(curr->first); + } + } + } + } + return true; +} + +Bool LocalFileSystem::isFileIgnored(const Char* filename, IgnoreFileTestFlags flags) const +{ + if (*filename == '\0') + return false; + + // Early out if nothing is ignored. + if (!hasIgnoredFile() && !hasIgnoredDirectory()) + return false; + + if (isFileIgnoredInternal(filename)) + return true; + + if (flags & IgnoreFileTestFlags_SkipParentDirectories) + return false; + + // Also check parent directories. + AsciiString recurseDirectory = filename; + trimLastPathSegmentInplace(recurseDirectory); + return isDirectoryIgnoredRecursive(recurseDirectory, flags); +} + +Bool LocalFileSystem::isDirectoryIgnored(const Char* directory, IgnoreFileTestFlags flags) const +{ + if (*directory == '\0') + return false; + + // Early out if nothing is ignored. + if (!hasIgnoredDirectory()) + return false; + + if (isDirectoryIgnoredInternal(directory)) + return true; + + if (flags & IgnoreFileTestFlags_SkipParentDirectories) + return false; + + // Also check parent directories. + AsciiString recurseDirectory = directory; + trimTrailingSlash(recurseDirectory); + trimLastPathSegmentInplace(recurseDirectory); + return isDirectoryIgnoredRecursive(recurseDirectory, flags); +} + +Bool LocalFileSystem::isDirectoryIgnoredRecursive(AsciiString& directory, IgnoreFileTestFlags flags) const +{ + if (directory.isEmpty()) + return false; + + if (isDirectoryIgnoredInternal(directory.str())) + return true; + + if (flags & IgnoreFileTestFlags_SkipParentDirectories) + return false; + + trimLastPathSegmentInplace(directory); + return isDirectoryIgnoredRecursive(directory, flags); +} + +Bool LocalFileSystem::isFileIgnoredInternal(const Char* filename) const +{ + FastCriticalSectionClass::LockClass lock(m_ignoreFileMutex); + return m_ignoreFileHashMap.find(IgnoreFileHashMap::key_type::temporary(filename)) != m_ignoreFileHashMap.end(); +} + +Bool LocalFileSystem::isDirectoryIgnoredInternal(const Char* directory) const +{ + FastCriticalSectionClass::LockClass lock(m_ignoreDirectoryMutex); + return m_ignoreDirectoryHashMap.find(IgnoreFileHashMap::key_type::temporary(directory)) != m_ignoreDirectoryHashMap.end(); +} + +Bool LocalFileSystem::hasIgnoredFile() const +{ + // Not locked for speed + return !m_ignoreFileHashMap.empty(); +} + +Bool LocalFileSystem::hasIgnoredDirectory() const +{ + // Not locked for speed + return !m_ignoreDirectoryHashMap.empty(); +} diff --git a/Core/GameEngine/Source/Common/System/UnicodeString.cpp b/Core/GameEngine/Source/Common/System/UnicodeString.cpp index 386778d321b..378b4a4399d 100644 --- a/Core/GameEngine/Source/Common/System/UnicodeString.cpp +++ b/Core/GameEngine/Source/Common/System/UnicodeString.cpp @@ -431,20 +431,25 @@ Bool UnicodeString::endsWithNoCase(const WideChar* p) const } //----------------------------------------------------------------------------- -Bool UnicodeString::nextToken(UnicodeString* tok, UnicodeString delimiters) +Bool UnicodeString::nextToken(UnicodeString* tok, const WideChar* delimiters) { - if (this->isEmpty() || tok == this) + DEBUG_ASSERTCRASH(tok != this, ("Tokenizer and Token cannot be the same object")); + + if (this->isEmpty()) + { + tok->clear(); return false; + } - if (delimiters.isEmpty()) + if (delimiters == nullptr) delimiters = L" \t\n\r"; Int offset; - offset = wcsspn(peek(), delimiters.str()); + offset = wcsspn(peek(), delimiters); WideChar* start = peek() + offset; - offset = wcscspn(start, delimiters.str()); + offset = wcscspn(start, delimiters); WideChar* end = start + offset; if (end > start) diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp index c873b733524..6c5b566fc67 100644 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp +++ b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp @@ -62,7 +62,7 @@ File* StdBIGFile::openFile( const Char *filename, Int access ) { const ArchivedFileInfo *fileInfo = getArchivedFileInfo(AsciiString(filename)); - if (fileInfo == nullptr) { + if (fileInfo == nullptr || fileInfo->m_ignore) { return nullptr; } @@ -151,17 +151,17 @@ void StdBIGFile::close() Bool StdBIGFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const { - const ArchivedFileInfo *tempFileInfo = getArchivedFileInfo(filename); + const ArchivedFileInfo *archivedFileInfo = getArchivedFileInfo(filename); - if (tempFileInfo == nullptr) { + if (archivedFileInfo == nullptr) { return FALSE; } - TheLocalFileSystem->getFileInfo(AsciiString(m_file->getName()), fileInfo); - - // fill in the size info. Since the size can't be bigger than a JUNK file, the high Int will always be 0. + // fill in the size info. Since the size can't be bigger than a JUNK file, the high int will always be 0. fileInfo->sizeHigh = 0; - fileInfo->sizeLow = tempFileInfo->m_size; + fileInfo->sizeLow = archivedFileInfo->m_size; + fileInfo->timestampHigh = 0; + fileInfo->timestampLow = 0; return TRUE; } diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp index ffd0150f4c5..9327c4914a8 100644 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp +++ b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp @@ -128,7 +128,7 @@ ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { // seek to the beginning of the directory listing. fp->seek(0x10, File::START); // read in each directory listing. - ArchivedFileInfo *fileInfo = NEW ArchivedFileInfo; + ArchivedFileInfo fileInfo; for (Int i = 0; i < numLittleFiles; ++i) { Int filesize = 0; @@ -139,9 +139,9 @@ ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { filesize = betoh(filesize); fileOffset = betoh(fileOffset); - fileInfo->m_archiveFilename = archiveFileName; - fileInfo->m_offset = fileOffset; - fileInfo->m_size = filesize; + fileInfo.m_archiveFilename = archiveFileName; + fileInfo.m_offset = fileOffset; + fileInfo.m_size = filesize; // read in the path name of the file. Int pathIndex = -1; @@ -155,8 +155,8 @@ ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { --filenameIndex; } - fileInfo->m_filename = (char *)(buffer + filenameIndex + 1); - fileInfo->m_filename.toLower(); + fileInfo.m_filename = (char *)(buffer + filenameIndex + 1); + fileInfo.m_filename.toLower(); buffer[filenameIndex + 1] = 0; AsciiString path; @@ -164,17 +164,14 @@ ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { AsciiString debugpath; debugpath = path; - debugpath.concat(fileInfo->m_filename); + debugpath.concat(fileInfo.m_filename); // DEBUG_LOG(("StdBIGFileSystem::openArchiveFile - adding file %s to archive file %s, file number %d", debugpath.str(), fileInfo->m_archiveFilename.str(), i)); - archiveFile->addFile(path, fileInfo); + archiveFile->addFile(path, &fileInfo); } archiveFile->attachFile(fp); - delete fileInfo; - fileInfo = nullptr; - // leave fp open as the archive file will be using it. return archiveFile; diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp index d47e473f4d2..5eaeaaa1a7d 100644 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp +++ b/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp @@ -130,7 +130,11 @@ File * StdLocalFileSystem::openFile(const Char *filename, Int access, size_t buf //USE_PERF_TIMER(StdLocalFileSystem_openFile) // sanity check - if (strlen(filename) <= 0) { + if (*filename == '\0') { + return nullptr; + } + + if (isFileIgnored(filename)) { return nullptr; } @@ -204,16 +208,23 @@ Bool StdLocalFileSystem::doesFileExist(const Char *filename) const return FALSE; } + if (isFileIgnored(filename)) { + return FALSE; + } + std::error_code ec; return std::filesystem::exists(path, ec); } void StdLocalFileSystem::getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList & filenameList, Bool searchSubdirectories) const { + AsciiString directory = originalDirectory; + directory.concat(currentDirectory); + + if (isDirectoryIgnored(directory.str())) + return; - AsciiString asciisearch; - asciisearch = originalDirectory; - asciisearch.concat(currentDirectory); + AsciiString asciisearch = directory; auto searchExt = std::filesystem::path(searchName.str()).extension(); if (asciisearch.isEmpty()) { asciisearch = "."; @@ -226,32 +237,32 @@ void StdLocalFileSystem::getFileListInDirectory(const AsciiString& currentDirect std::replace(fixedDirectory.begin(), fixedDirectory.end(), '\\', '/'); #endif - Bool done = FALSE; std::error_code ec; auto iter = std::filesystem::directory_iterator(fixedDirectory.c_str(), ec); - // The default iterator constructor creates an end iterator - done = iter == std::filesystem::directory_iterator(); if (ec) { DEBUG_LOG(("StdLocalFileSystem::getFileListInDirectory - Error opening directory %s", fixedDirectory.c_str())); return; } - while (!done) { + // The default iterator constructor creates an end iterator + for (; iter != std::filesystem::directory_iterator(); ++iter) { std::string filenameStr = iter->path().filename().string(); if (!iter->is_directory() && iter->path().extension() == searchExt && (strcmp(filenameStr.c_str(), ".") != 0 && strcmp(filenameStr.c_str(), "..") != 0)) { // if we haven't already, add this filename to the list. // a stl set should only allow one copy of each filename AsciiString newFilename = iter->path().string().c_str(); + + // skip parent directories because they were already tested before. + if (isFileIgnored(newFilename.str(), IgnoreFileTestFlags_SkipParentDirectories)) + continue; + if (filenameList.find(newFilename) == filenameList.end()) { filenameList.insert(newFilename); } } - - iter++; - done = iter == std::filesystem::directory_iterator(); } if (searchSubdirectories) { @@ -263,9 +274,7 @@ void StdLocalFileSystem::getFileListInDirectory(const AsciiString& currentDirect } // The default iterator constructor creates an end iterator - done = iter == std::filesystem::directory_iterator(); - - while (!done) { + for (; iter != std::filesystem::directory_iterator(); ++iter) { std::string filenameStr = iter->path().filename().string(); if(iter->is_directory() && (strcmp(filenameStr.c_str(), ".") != 0 && strcmp(filenameStr.c_str(), "..") != 0)) { @@ -274,9 +283,6 @@ void StdLocalFileSystem::getFileListInDirectory(const AsciiString& currentDirect // recursively add files in subdirectories if required. getFileListInDirectory(tempsearchstr, originalDirectory, searchName, filenameList, searchSubdirectories); } - - iter++; - done = iter == std::filesystem::directory_iterator(); } } } @@ -285,10 +291,13 @@ Bool StdLocalFileSystem::getFileInfo(const AsciiString& filename, FileInfo *file { std::filesystem::path path = fixFilenameFromWindowsPath(filename.str(), 0); - if(path.empty()) { + if (path.empty()) { return FALSE; } + if (isFileIgnored(filename.str())) + return FALSE; + std::error_code ec; auto file_size = std::filesystem::file_size(path, ec); if (ec) @@ -304,10 +313,10 @@ Bool StdLocalFileSystem::getFileInfo(const AsciiString& filename, FileInfo *file // TODO: fix this to be win compatible (time since 1601) auto time = write_time.time_since_epoch().count(); + fileInfo->sizeHigh = file_size >> 32; + fileInfo->sizeLow = file_size & UINT32_MAX; fileInfo->timestampHigh = time >> 32; fileInfo->timestampLow = time & UINT32_MAX; - fileInfo->sizeHigh = file_size >> 32; - fileInfo->sizeLow = file_size & UINT32_MAX; return TRUE; } diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp index 3deee01fd82..732fcdc9555 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp +++ b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp @@ -62,7 +62,7 @@ File* Win32BIGFile::openFile( const Char *filename, Int access ) { const ArchivedFileInfo *fileInfo = getArchivedFileInfo(AsciiString(filename)); - if (fileInfo == nullptr) { + if (fileInfo == nullptr || fileInfo->m_ignore) { return nullptr; } @@ -151,17 +151,17 @@ void Win32BIGFile::close() Bool Win32BIGFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const { - const ArchivedFileInfo *tempFileInfo = getArchivedFileInfo(filename); + const ArchivedFileInfo *archivedFileInfo = getArchivedFileInfo(filename); - if (tempFileInfo == nullptr) { + if (archivedFileInfo == nullptr) { return FALSE; } - TheLocalFileSystem->getFileInfo(AsciiString(m_file->getName()), fileInfo); - - // fill in the size info. Since the size can't be bigger than a JUNK file, the high Int will always be 0. + // fill in the size info. Since the size can't be bigger than a JUNK file, the high int will always be 0. fileInfo->sizeHigh = 0; - fileInfo->sizeLow = tempFileInfo->m_size; + fileInfo->sizeLow = archivedFileInfo->m_size; + fileInfo->timestampHigh = 0; + fileInfo->timestampLow = 0; return TRUE; } diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp index 64401c47eff..f6f81cef660 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp +++ b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp @@ -127,7 +127,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { // seek to the beginning of the directory listing. fp->seek(0x10, File::START); // read in each directory listing. - ArchivedFileInfo *fileInfo = NEW ArchivedFileInfo; + ArchivedFileInfo fileInfo; // TheSuperHackers @fix Mauller 23/04/2025 Create new file handle when necessary to prevent memory leak ArchiveFile *archiveFile = NEW Win32BIGFile(filename, AsciiString::TheEmptyString); @@ -140,9 +140,9 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { filesize = betoh(filesize); fileOffset = betoh(fileOffset); - fileInfo->m_archiveFilename = archiveFileName; - fileInfo->m_offset = fileOffset; - fileInfo->m_size = filesize; + fileInfo.m_archiveFilename = archiveFileName; + fileInfo.m_offset = fileOffset; + fileInfo.m_size = filesize; // read in the path name of the file. Int pathIndex = -1; @@ -156,8 +156,8 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { --filenameIndex; } - fileInfo->m_filename = (char *)(buffer + filenameIndex + 1); - fileInfo->m_filename.toLower(); + fileInfo.m_filename = (char *)(buffer + filenameIndex + 1); + fileInfo.m_filename.toLower(); buffer[filenameIndex + 1] = 0; AsciiString path; @@ -165,17 +165,14 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { AsciiString debugpath; debugpath = path; - debugpath.concat(fileInfo->m_filename); + debugpath.concat(fileInfo.m_filename); // DEBUG_LOG(("Win32BIGFileSystem::openArchiveFile - adding file %s to archive file %s, file number %d", debugpath.str(), fileInfo->m_archiveFilename.str(), i)); - archiveFile->addFile(path, fileInfo); + archiveFile->addFile(path, &fileInfo); } archiveFile->attachFile(fp); - delete fileInfo; - fileInfo = nullptr; - // leave fp open as the archive file will be using it. return archiveFile; diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp index 43a6ca5cbdd..4178ae19de0 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp +++ b/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp @@ -47,7 +47,11 @@ File * Win32LocalFileSystem::openFile(const Char *filename, Int access, size_t b //USE_PERF_TIMER(Win32LocalFileSystem_openFile) // sanity check - if (strlen(filename) <= 0) { + if (*filename == '\0') { + return nullptr; + } + + if (isFileIgnored(filename)) { return nullptr; } @@ -116,54 +120,59 @@ void Win32LocalFileSystem::reset() Bool Win32LocalFileSystem::doesFileExist(const Char *filename) const { //USE_PERF_TIMER(Win32LocalFileSystem_doesFileExist) + + if (isFileIgnored(filename)) { + return FALSE; + } + if (_access(filename, 0) == 0) { return TRUE; } + return FALSE; } void Win32LocalFileSystem::getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList & filenameList, Bool searchSubdirectories) const { - HANDLE fileHandle = nullptr; - WIN32_FIND_DATA findData; + AsciiString directory = originalDirectory; + directory.concat(currentDirectory); - AsciiString asciisearch; - asciisearch = originalDirectory; - asciisearch.concat(currentDirectory); - asciisearch.concat(searchName); + if (isDirectoryIgnored(directory.str())) + return; - Bool done = FALSE; + AsciiString asciisearch = directory; + asciisearch.concat(searchName); - fileHandle = FindFirstFile(asciisearch.str(), &findData); - done = (fileHandle == INVALID_HANDLE_VALUE); + WIN32_FIND_DATA findData; + HANDLE fileHandle = FindFirstFile(asciisearch.str(), &findData); + Bool done = (fileHandle == INVALID_HANDLE_VALUE); - while (!done) { + for (; !done; done = (FindNextFile(fileHandle, &findData) == 0)) { if (!(findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) && (strcmp(findData.cFileName, ".") != 0 && strcmp(findData.cFileName, "..") != 0)) { - // if we haven't already, add this filename to the list. + // if we haven't already, add this filename to the list. // a stl set should only allow one copy of each filename - AsciiString newFilename; - newFilename = originalDirectory; - newFilename.concat(currentDirectory); + AsciiString newFilename = directory; newFilename.concat(findData.cFileName); + + // skip parent directories because they were already tested before. + if (isFileIgnored(newFilename.str(), IgnoreFileTestFlags_SkipParentDirectories)) + continue; + if (filenameList.find(newFilename) == filenameList.end()) { filenameList.insert(newFilename); } } - - done = (FindNextFile(fileHandle, &findData) == 0); } FindClose(fileHandle); if (searchSubdirectories) { - AsciiString subdirsearch; - subdirsearch = originalDirectory; - subdirsearch.concat(currentDirectory); + AsciiString subdirsearch = directory; subdirsearch.concat("*."); fileHandle = FindFirstFile(subdirsearch.str(), &findData); - done = fileHandle == INVALID_HANDLE_VALUE; + done = (fileHandle == INVALID_HANDLE_VALUE); - while (!done) { + for (; !done; done = (FindNextFile(fileHandle, &findData) == 0)) { if ((findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) && (strcmp(findData.cFileName, ".") != 0 && strcmp(findData.cFileName, "..") != 0)) { @@ -175,8 +184,6 @@ void Win32LocalFileSystem::getFileListInDirectory(const AsciiString& currentDire // recursively add files in subdirectories if required. getFileListInDirectory(tempsearchstr, originalDirectory, searchName, filenameList, searchSubdirectories); } - - done = (FindNextFile(fileHandle, &findData) == 0); } FindClose(fileHandle); @@ -186,6 +193,12 @@ void Win32LocalFileSystem::getFileListInDirectory(const AsciiString& currentDire Bool Win32LocalFileSystem::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const { + if (filename.isEmpty()) + return FALSE; + + if (isFileIgnored(filename.str())) + return FALSE; + WIN32_FIND_DATA findData; HANDLE findHandle = nullptr; findHandle = FindFirstFile(filename.str(), &findData); @@ -194,10 +207,10 @@ Bool Win32LocalFileSystem::getFileInfo(const AsciiString& filename, FileInfo *fi return FALSE; } - fileInfo->timestampHigh = findData.ftLastWriteTime.dwHighDateTime; - fileInfo->timestampLow = findData.ftLastWriteTime.dwLowDateTime; fileInfo->sizeHigh = findData.nFileSizeHigh; fileInfo->sizeLow = findData.nFileSizeLow; + fileInfo->timestampHigh = findData.ftLastWriteTime.dwHighDateTime; + fileInfo->timestampLow = findData.ftLastWriteTime.dwLowDateTime; FindClose(findHandle); diff --git a/Core/Libraries/Include/Lib/PathUtil.h b/Core/Libraries/Include/Lib/PathUtil.h index cf1ce769d91..655d149dd8b 100644 --- a/Core/Libraries/Include/Lib/PathUtil.h +++ b/Core/Libraries/Include/Lib/PathUtil.h @@ -23,6 +23,123 @@ #include "BaseType.h" #include +// Whether the character is a path separator +inline bool isPathSeparator(unsigned char ch) +{ + return ch == '/' || ch == '\\'; +} + +// Return a lowercase ascii character. Faster than tolower. +inline unsigned char toLowerAscii(unsigned char ch) +{ + if (ch >= 'A' && ch <= 'Z') + ch += 'a' - 'A'; + + return ch; +} + +// Compare two ascii paths and ignore case and different types of slashes. Useful for map lookups. +inline int comparePath(const char* a, const char* b) +{ + while (*a && *b) + { + unsigned char ca = static_cast(*a); + unsigned char cb = static_cast(*b); + + if (isPathSeparator(ca) && isPathSeparator(cb)) + { + ++a; + ++b; + continue; + } + + ca = toLowerAscii(ca); + cb = toLowerAscii(cb); + + if (ca != cb) + return ca - cb; + + ++a; + ++b; + } + + // Ignore trailing separators + while (isPathSeparator(*a)) + ++a; + while (isPathSeparator(*b)) + ++b; + + return static_cast(*a) - static_cast(*b); +} + +// Test if an ascii path starts with a given path and ignore case and different types of slashes. +inline bool startsWithPath(const char* path, const char* prefix) +{ + while (*path && *prefix) + { + unsigned char cp = static_cast(*path); + unsigned char cq = static_cast(*prefix); + + if (isPathSeparator(cp) && isPathSeparator(cq)) + { + ++path; + ++prefix; + continue; + } + + cp = toLowerAscii(cp); + cq = toLowerAscii(cq); + + if (cp != cq) + return false; + + ++path; + ++prefix; + } + + // Ignore trailing separators in the prefix + while (isPathSeparator(*prefix)) + ++prefix; + + return *prefix == '\0'; +} + +// Hash an ascii path by normalizing case and different types of slashes. Useful for hash maps. +inline size_t hashPath(const char* path) +{ + constexpr const UnsignedInt64 FnvOffset = UnsignedInt64(14695981039346656037); + constexpr const UnsignedInt64 FnvPrime = UnsignedInt64(1099511628211); + + // FNV-1a 64-bit + UnsignedInt64 hash = FnvOffset; + + // Find end + const char* end = path; + while (*end) + ++end; + + // Ignore trailing separators + while (end > path && isPathSeparator(end[-1])) + --end; + + // Hash normalized characters + for (const char* p = path; p != end; ++p) + { + unsigned char ch = static_cast(*p); + + if (isPathSeparator(ch)) + ch = '\\'; + else + ch = toLowerAscii(ch); + + hash ^= ch; + hash *= FnvPrime; + } + + return (size_t)hash; +} + +// Return file extension part from absolute or relative path when present. Null otherwise. inline const char* getExtension(const char* path) { const char* lastDot = strrchr(path, '.'); @@ -43,6 +160,7 @@ inline const char* getExtension(const char* path) return lastDot; } +// Return file extension part from absolute or relative path when present. Null otherwise. inline const wchar_t* getExtension(const wchar_t* path) { const wchar_t* lastDot = wcsrchr(path, L'.'); @@ -62,3 +180,17 @@ inline const wchar_t* getExtension(const wchar_t* path) return lastDot; } + +// Returns whether the given path is assumed to be a file path or directory path. +// Requirements for file path are: Must contain a dot in the trailing name after the last slash. +inline bool isFilePath(const char* path) +{ + return getExtension(path) != nullptr; +} + +// Returns whether the given path is assumed to be a file path or directory path. +// Requirements for file path are: Must contain a dot in the trailing name after the last slash. +inline bool isFilePath(const wchar_t* path) +{ + return getExtension(path) != nullptr; +} diff --git a/Core/Libraries/Source/Compression/CMakeLists.txt b/Core/Libraries/Source/Compression/CMakeLists.txt index 5a5fbc8275a..ec998c503ac 100644 --- a/Core/Libraries/Source/Compression/CMakeLists.txt +++ b/Core/Libraries/Source/Compression/CMakeLists.txt @@ -29,11 +29,9 @@ target_include_directories(core_compression INTERFACE ) target_link_libraries(core_compression PRIVATE - corei_libraries_include corei_always ) - target_link_libraries(core_compression PUBLIC liblzhl ) diff --git a/Core/Libraries/Source/WWVegas/WWLib/STLUtils.h b/Core/Libraries/Source/WWVegas/WWLib/STLUtils.h index 5dae677abe8..8d21975c561 100644 --- a/Core/Libraries/Source/WWVegas/WWLib/STLUtils.h +++ b/Core/Libraries/Source/WWVegas/WWLib/STLUtils.h @@ -123,22 +123,26 @@ Iter advance_in_range(Iter first, Iter last, ptrdiff_t n) return first; } -template -range/**/> get_range(std::multimap& mm, const Key& key, ptrdiff_t n = 0) +template +range/**/> get_range(std::multimap& mm, const Key& key, ptrdiff_t n = 0) { - typedef typename std::multimap::iterator Iter; + typedef std::multimap Multimap; + typedef typename Multimap::iterator Iter; + const std::pair pair = mm.equal_range(key); const Iter it = advance_in_range(pair.first, pair.second, n); - return range/**/>(it, pair.second); + return range(it, pair.second); } -template -const_range/**/> get_range(const std::multimap& mm, const Key& key, ptrdiff_t n = 0) +template +const_range/**/> get_range(const std::multimap& mm, const Key& key, ptrdiff_t n = 0) { - typedef typename std::multimap::const_iterator Iter; + typedef std::multimap Multimap; + typedef typename Multimap::const_iterator Iter; + const std::pair pair = mm.equal_range(key); const Iter it = advance_in_range(pair.first, pair.second, n); - return const_range/**/>(it, pair.second); + return const_range(it, pair.second); } } // namespace stl diff --git a/Core/Tools/MapCacheBuilder/Source/WinMain.cpp b/Core/Tools/MapCacheBuilder/Source/WinMain.cpp index 134f1465980..203e68a2276 100644 --- a/Core/Tools/MapCacheBuilder/Source/WinMain.cpp +++ b/Core/Tools/MapCacheBuilder/Source/WinMain.cpp @@ -251,6 +251,11 @@ Int APIENTRY WinMain( HINSTANCE hInstance, HINSTANCE hPrevInstance, initSubsystem(TheLocalFileSystem, (LocalFileSystem*)new Win32LocalFileSystem); initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new Win32BIGFileSystem); INI ini; + // TheSuperHackers @feature xezon 26/07/2026 Loads new FileSystem.ini before any other ini files + // to allow configure the file system, such as setting up ignored game files and directories. + ini.loadFileDirectory( "Data\\INI\\Default\\FileSystem", INI_LOAD_OVERWRITE, nullptr, INI::LoadFlags_SearchSubDirs ); + ini.loadFileDirectory( "Data\\INI\\FileSystem", INI_LOAD_OVERWRITE, nullptr, INI::LoadFlags_SearchSubDirs ); + initSubsystem(TheWritableGlobalData, new GlobalData(), "Data\\INI\\Default\\GameData", "Data\\INI\\GameData"); initSubsystem(TheGameText, CreateGameTextInterface()); initSubsystem(TheScienceStore, new ScienceStore(), "Data\\INI\\Default\\Science", "Data\\INI\\Science"); diff --git a/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp b/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp index 32b93d3dba7..7193244e008 100644 --- a/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp +++ b/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp @@ -442,6 +442,11 @@ void GameEngine::init() initSubsystem(TheArchiveFileSystem, "TheArchiveFileSystem", createArchiveFileSystem(), nullptr); // this MUST come after TheLocalFileSystem creation + // TheSuperHackers @feature xezon 26/07/2026 Loads new FileSystem.ini before any other ini files + // to allow configure the file system, such as setting up ignored game files and directories. + ini.loadFileDirectory( "Data\\INI\\Default\\FileSystem", INI_LOAD_OVERWRITE, nullptr, INI::LoadFlags_SearchSubDirs ); + ini.loadFileDirectory( "Data\\INI\\FileSystem", INI_LOAD_OVERWRITE, nullptr, INI::LoadFlags_SearchSubDirs ); + #ifdef DUMP_PERF_STATS/////////////////////////////////////////////////////////////////////////// GetPrecisionTimer(&endTime64);////////////////////////////////////////////////////////////////// sprintf(Buf,"----------------------------------------------------------------------------After TheArchiveFileSystem = %f seconds",((double)(endTime64-startTime64)/(double)(freq64))); diff --git a/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp b/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp index 5f19126638a..d8a63422b40 100644 --- a/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp +++ b/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp @@ -334,6 +334,11 @@ BOOL CWorldBuilderApp::InitInstance() INI ini; + // TheSuperHackers @feature xezon 26/07/2026 Loads new FileSystem.ini before any other ini files + // to allow configure the file system, such as setting up ignored game files and directories. + ini.loadFileDirectory( "Data\\INI\\Default\\FileSystem", INI_LOAD_OVERWRITE, nullptr, INI::LoadFlags_SearchSubDirs ); + ini.loadFileDirectory( "Data\\INI\\FileSystem", INI_LOAD_OVERWRITE, nullptr, INI::LoadFlags_SearchSubDirs ); + initSubsystem(TheWritableGlobalData, new GlobalData(), "Data\\INI\\Default\\GameData", "Data\\INI\\GameData"); TheFramePacer = new FramePacer();