When Move()ing, Remove()ing, or Close()ing a file opened for writing, update the cache when possible (RageFileDriverDirect) or invalidate the directory cache for the directory containing that file (everything else). Remove most of the FlushDirCache() calls as they are no longer needed. This fixes a long standing bug where files added into the cache were stat(2)ed O(n^2) times--that is, each file is stat(2)ed O(n) times.

This commit is contained in:
Steve Checkoway
2009-09-14 08:08:06 +00:00
parent b60a92e633
commit 76f0af145b
25 changed files with 140 additions and 106 deletions
+2 -2
View File
@@ -54,7 +54,7 @@ retry:
RageException::Throw( "%s", sError.c_str() );
break;
case Dialog::retry:
FlushDirCache();
FILEMAN->FlushDirCache();
goto retry;
case Dialog::ignore:
return false;
@@ -75,7 +75,7 @@ retry:
RageException::Throw( "%s", sError.c_str() );
break;
case Dialog::retry:
FlushDirCache();
FILEMAN->FlushDirCache();
goto retry;
case Dialog::ignore:
asPaths.erase( asPaths.begin()+1, asPaths.end() );
-4
View File
@@ -349,10 +349,7 @@ bool EditCourseUtil::RenameAndSave( Course *pCourse, RString sNewName )
// remove the old file if the name is changing
if( !pCourse->m_sPath.empty() && sNewFilePath != pCourse->m_sPath )
{
FILEMAN->Remove( pCourse->m_sPath ); // not fatal if this fails
FlushDirCache();
}
pCourse->m_sMainTitle = sNewName;
pCourse->m_sPath = sNewFilePath;
@@ -364,7 +361,6 @@ bool EditCourseUtil::RemoveAndDeleteFile( Course *pCourse )
if( !FILEMAN->Remove( pCourse->m_sPath ) )
return false;
FILEMAN->Remove( pCourse->GetCacheFilePath() );
FlushDirCache();
if( pCourse->IsAnEdit() )
{
PROFILEMAN->LoadMachineProfile();
-1
View File
@@ -114,7 +114,6 @@ void CryptManager::GenerateGlobalKeys()
{
LOG->Warn( "Keys missing or failed to load. Generating new keys" );
GenerateRSAKeyToFile( KEY_LENGTH, PRIVATE_KEY_PATH, PUBLIC_KEY_PATH );
FlushDirCache();
}
}
-3
View File
@@ -320,9 +320,6 @@ static bool WriteDWINotesTag( RageFile &f, const Steps &out )
if( out.GetDifficulty() == Difficulty_Edit )
return false; // not supported by DWI
/* Flush dir cache when writing steps, so the old size isn't cached. */
FILEMAN->FlushDirCache( Dirname(f.GetRealPath()) );
LOG->Trace( "Steps::WriteDWINotesTag" );
switch( out.m_StepsType )
-9
View File
@@ -228,9 +228,6 @@ static RString GetSMNotesTag( const Song &song, const Steps &in, bool bSavingCac
bool NotesWriterSM::Write( RString sPath, const Song &out, const vector<Steps*>& vpStepsToSave, bool bSavingCache )
{
/* Flush dir cache when writing steps, so the old size isn't cached. */
FILEMAN->FlushDirCache( Dirname(sPath) );
int flags = RageFile::WRITE;
/* If we're not saving cache, we're saving real data, so enable SLOW_FLUSH
@@ -313,9 +310,6 @@ bool NotesWriterSM::WriteEditFileToMachine( const Song *pSong, Steps *pSteps, RS
RString sPath = sDir + GetEditFileName(pSong,pSteps);
/* Flush dir cache when writing steps, so the old size isn't cached. */
FILEMAN->FlushDirCache( Dirname(sPath) );
// Check to make sure that we're not clobering an existing file before opening.
bool bFileNameChanging =
pSteps->GetSavedToDisk() &&
@@ -348,9 +342,6 @@ bool NotesWriterSM::WriteEditFileToMachine( const Song *pSong, Steps *pSteps, RS
FILEMAN->Remove( pSteps->GetFilename() );
pSteps->SetFilename( sPath );
/* Flush dir cache or else the new file won't be seen. */
FILEMAN->FlushDirCache( Dirname(sPath) );
return true;
}
-9
View File
@@ -935,8 +935,6 @@ bool Profile::SaveAllToDir( RString sDir, bool bSignData ) const
FILEMAN->CreateDir( sDir + EDIT_COURSES_SUBDIR );
FILEMAN->CreateDir( sDir + SCREENSHOTS_SUBDIR );
FILEMAN->FlushDirCache( sDir );
return bSaved;
}
@@ -998,17 +996,11 @@ bool Profile::SaveStatsXmlToDir( RString sDir, bool bSignData ) const
}
}
// Update file cache, or else IsAFile in CryptManager won't see this new file.
FILEMAN->FlushDirCache( sDir );
if( bSignData )
{
RString sStatsXmlSigFile = fn+SIGNATURE_APPEND;
CryptManager::SignFileToFile(fn, sStatsXmlSigFile);
// Update file cache, or else IsAFile in CryptManager won't see sStatsXmlSigFile.
FILEMAN->FlushDirCache( sDir );
// Save the "don't share" file
RString sDontShareFile = sDir + DONT_SHARE_SIG;
CryptManager::SignFileToFile(sStatsXmlSigFile, sDontShareFile);
@@ -1815,7 +1807,6 @@ RString Profile::MakeUniqueFileNameNoExtension( RString sDir, RString sFileNameB
//
// Find a file name for the screenshot
//
FILEMAN->FlushDirCache( sDir );
vector<RString> files;
GetDirListing( sDir + sFileNameBeginning+"*", files, false, false );
+6 -1
View File
@@ -78,7 +78,12 @@ bool RageFile::Open( const RString& path, int mode )
void RageFile::Close()
{
SAFE_DELETE( m_File );
if( m_File == NULL )
return;
delete m_File;
if( m_Mode & WRITE )
FILEMAN->CacheFile( m_File, m_Path );
m_File = NULL;
}
#define ASSERT_OPEN ASSERT_M( IsOpen(), ssprintf("\"%s\" is not open.", m_Path.c_str()) );
+1 -1
View File
@@ -64,7 +64,7 @@ int RageFileDriver::GetFileHash( const RString &sPath )
void RageFileDriver::FlushDirCache( const RString &sPath )
{
FDB->FlushDirCache();
FDB->FlushDirCache( sPath );
}
+1
View File
@@ -19,6 +19,7 @@ public:
virtual int GetFileHash( const RString &sPath );
virtual int GetPathValue( const RString &sPath );
virtual void FlushDirCache( const RString &sPath );
virtual void CacheFile( const RString &sPath ) { }
virtual bool Move( const RString &sOldPath, const RString &sNewPath ) { return false; }
virtual bool Remove( const RString &sPath ) { return false; }
+6 -3
View File
@@ -77,7 +77,7 @@ static RString MakeTempFilename( const RString &sPath )
return Dirname(sPath) + "new." + Basename(sPath) + ".new";
}
RageFileObjDirect *MakeFileObjDirect( RString sPath, int iMode, int &iError )
static RageFileObjDirect *MakeFileObjDirect( RString sPath, int iMode, int &iError )
{
int iFD;
if( iMode & RageFile::READ )
@@ -153,13 +153,16 @@ bool RageFileDriverDirect::Move( const RString &sOldPath_, const RString &sNewPa
const RString sDir = Dirname(sNewPath);
CreateDirectories( m_sRoot + sDir );
}
int size = FDB->GetFileSize( sOldPath );
int hash = FDB->GetFileHash( sOldPath );
TRACE( ssprintf("rename \"%s\" -> \"%s\"", (m_sRoot + sOldPath).c_str(), (m_sRoot + sNewPath).c_str()) );
if( DoRename(m_sRoot + sOldPath, m_sRoot + sNewPath) == -1 )
{
WARN( ssprintf("rename(%s,%s) failed: %s", (m_sRoot + sOldPath).c_str(), (m_sRoot + sNewPath).c_str(), strerror(errno)) );
return false;
}
FDB->DelFile( sOldPath );
FDB->AddFile( sNewPath, size, hash, NULL );
return true;
}
@@ -350,7 +353,6 @@ RageFileObjDirect::~RageFileObjDirect()
SetError( strerror(errno) );
break;
}
#endif
if( m_iMode & RageFile::SLOW_FLUSH )
{
@@ -364,6 +366,7 @@ RageFileObjDirect::~RageFileObjDirect()
/* Success. */
return;
#endif
} while(0);
/* The write or the rename failed. Delete the incomplete temporary file. */
+54 -2
View File
@@ -197,6 +197,59 @@ void DirectFilenameDB::SetRoot( RString root_ )
root.erase( root.size()-1, 1 );
}
void DirectFilenameDB::CacheFile( const RString &sPath )
{
CHECKPOINT_M( root+sPath );
RString sDir = Dirname( sPath );
FileSet *pFileSet = GetFileSet( sDir, false );
if( pFileSet == NULL )
{
// This directory isn't cached so do nothing.
m_Mutex.Unlock(); // Locked by GetFileSet()
return;
}
while( !pFileSet->m_bFilled )
m_Mutex.Wait();
#if defined(WIN32)
// There is almost surely a better way to do this
WIN32_FIND_DATA fd;
HANDLE hFind = DoFindFirstFile( root+sPath, &fd );
if( hFind == INVALID_HANDLE_VALUE )
{
m_Mutex.Unlock(); // Locked by GetFileSet()
return;
}
File f( fd.cFileName );
f.dir = !!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY);
f.size = fd.nFileSizeLow;
f.hash = fd.ftLastWriteTime.dwLowDateTime;
pFileSet->files.insert( f );
FindClose( hFind );
#else
File f( Basename(sPath) );
struct stat st;
if( DoStat(root+sPath, &st) == -1 )
{
int iError = errno;
/* If it's a broken symlink, ignore it. Otherwise, warn. */
/* Huh? */
WARN( ssprintf("File '%s' is gone! (%s)",
sPath.c_str(), strerror(iError)) );
}
else
{
f.dir = (st.st_mode & S_IFDIR);
f.size = (int)st.st_size;
f.hash = st.st_mtime;
}
pFileSet->files.insert(f);
#endif
m_Mutex.Unlock(); // Locked by GetFileSet()
}
void DirectFilenameDB::PopulateFileSet( FileSet &fs, const RString &path )
{
@@ -259,8 +312,7 @@ void DirectFilenameDB::PopulateFileSet( FileSet &fs, const RString &path )
if( !strcmp(pEnt->d_name, "..") )
continue;
File f;
f.SetName( pEnt->d_name );
File f( pEnt->d_name );
struct stat st;
if( DoStat(root+sPath + "/" + pEnt->d_name, &st) == -1 )
+1 -1
View File
@@ -40,7 +40,7 @@ class DirectFilenameDB: public FilenameDB
public:
DirectFilenameDB( RString root );
void SetRoot( RString root );
void CacheFile( const RString &sPath );
protected:
virtual void PopulateFileSet( FileSet &fs, const RString &sPath );
RString root;
+16 -12
View File
@@ -42,6 +42,7 @@ struct LoadedDriver
};
static vector<LoadedDriver *> g_pDrivers;
static map<const RageFileBasic *,LoadedDriver *> g_mFileDriverMap;
static void ReferenceAllDrivers( vector<LoadedDriver *> &apDriverList )
{
@@ -439,15 +440,10 @@ bool RageFileManager::Remove( const RString &sPath_ )
void RageFileManager::CreateDir( const RString &sDir )
{
RString sTempFile = sDir + "temp";
RString sTempFile = sDir + "newdir.temp.newdir";
RageFile f;
f.Open( sTempFile, RageFile::WRITE );
f.Close();
// YUCK: The dir cache doesn't have this new file we just created,
// so the delete will fail unless we flush.
FlushDirCache( sDir );
Remove( sTempFile );
}
@@ -772,6 +768,19 @@ RageFileBasic *RageFileManager::Open( const RString &sPath_, int mode, int &err
return OpenForReading( sPath, mode, err );
}
void RageFileManager::CacheFile( const RageFileBasic *fb, const RString &sPath_ )
{
map<const RageFileBasic *,LoadedDriver *>::iterator it = g_mFileDriverMap.find( fb );
ASSERT_M( it != g_mFileDriverMap.end(), ssprintf("No recorded driver for file: %s", sPath_.c_str()) );
RString sPath = sPath_;
NormalizePath( sPath );
sPath = it->second->GetPath( sPath );
it->second->m_pDriver->FDB->CacheFile( sPath );
g_mFileDriverMap.erase( it );
}
RageFileBasic *RageFileManager::OpenForReading( const RString &sPath, int mode, int &err )
{
vector<LoadedDriver *> apDriverList;
@@ -863,6 +872,7 @@ RageFileBasic *RageFileManager::OpenForWriting( const RString &sPath, int mode,
RageFileBasic *pRet = ld.m_pDriver->Open( sDriverPath, mode, iThisError );
if( pRet )
{
g_mFileDriverMap[pRet] = &ld;
UnreferenceAllDrivers( apDriverList );
return pRet;
}
@@ -986,12 +996,6 @@ bool DeleteRecursive( const RString &sDir )
return FILEMAN->Remove( sDir );
}
void FlushDirCache()
{
FILEMAN->FlushDirCache( "" );
}
unsigned int GetHashForFile( const RString &sPath )
{
return FILEMAN->GetFileHash( sPath );
+2 -1
View File
@@ -50,10 +50,11 @@ public:
};
void GetLoadedDrivers( vector<DriverLocation> &asMounts );
void FlushDirCache( const RString &sPath );
void FlushDirCache( const RString &sPath = RString() );
/* Used only by RageFile: */
RageFileBasic *Open( const RString &sPath, int iMode, int &iError );
void CacheFile( const RageFileBasic *fb, const RString &sPath );
/* Retrieve or release a reference to the low-level driver for a mountpoint. */
RageFileDriver *GetFileDriver( RString sMountpoint );
-1
View File
@@ -584,7 +584,6 @@ bool DoesFileExist( const RString &sPath );
bool IsAFile( const RString &sPath );
bool IsADirectory( const RString &sPath );
unsigned GetFileSizeInBytes( const RString &sFilePath );
void FlushDirCache();
// call FixSlashesInPlace on any path that came from the user
void FixSlashesInPlace( RString &sPath );
+41 -26
View File
@@ -412,18 +412,14 @@ void FilenameDB::AddFile( const RString &sPath_, int iSize, int iHash, void *pPr
FileSet *fs = GetFileSet( dir );
ASSERT( m_Mutex.IsLockedByThisThread() );
File f;
f.SetName( fn );
if( fs->files.find( f ) == fs->files.end() )
// const_cast to cast away the constness that is only needed for the name
File &f = const_cast<File&>(*fs->files.insert( fn ).first);
f.dir = IsDir;
if( !IsDir )
{
f.dir = IsDir;
if( !IsDir )
{
f.size = iSize;
f.hash = iHash;
f.priv = pPriv;
}
fs->files.insert( f );
f.size = iSize;
f.hash = iHash;
f.priv = pPriv;
}
m_Mutex.Unlock(); /* locked by GetFileSet */
IsDir = true;
@@ -475,19 +471,35 @@ void FilenameDB::DelFile( const RString &sPath )
SplitPath(sPath, Dir, Name);
FileSet *Parent = GetFileSet( Dir, false );
if( Parent )
{
set<File>::iterator i = Parent->files.find( File(Name) );
if( i != Parent->files.end() )
{
Parent->files.erase( i );
}
}
Parent->files.erase( Name );
m_Mutex.Unlock(); /* locked by GetFileSet */
}
void FilenameDB::FlushDirCache()
void FilenameDB::FlushDirCache( const RString &sDir )
{
FileSet *pFileSet = NULL;
m_Mutex.Lock();
if( !sDir.empty() )
{
RString lower = sDir;
lower.MakeLower();
map<RString, FileSet *>::iterator it = dirs.find( lower );
if( it != dirs.end() )
{
pFileSet = it->second;
dirs.erase( it );
while( !pFileSet->m_bFilled )
m_Mutex.Wait();
delete pFileSet;
}
else
{
LOG->Warn( "Trying to flush an unknown directory %s.", sDir.c_str() );
}
m_Mutex.Unlock();
return;
}
while( true )
{
if( dirs.empty() )
@@ -495,7 +507,7 @@ void FilenameDB::FlushDirCache()
/* Grab the first entry. Take it out of the list while we hold the
* lock, to guarantee that we own it. */
FileSet *pFileSet = dirs.begin()->second;
pFileSet = dirs.begin()->second;
dirs.erase( dirs.begin() );
@@ -522,14 +534,10 @@ const File *FilenameDB::GetFile( const RString &sPath )
if( it == fs->files.end() )
return NULL;
/* Oops. &*it is a const File &, because you can't change the order
* of something once it's in a list. Cast away the const; we won't
* change the filename (used for the ordering), but the rest of the
* values are non-const. */
return const_cast<File *> (&*it);
return &*it;
}
const void *FilenameDB::GetFilePriv( const RString &path )
void *FilenameDB::GetFilePriv( const RString &path )
{
ASSERT( !m_Mutex.IsLockedByThisThread() );
@@ -592,6 +600,13 @@ void FilenameDB::GetFileSetCopy( const RString &sDir, FileSet &out )
m_Mutex.Unlock(); /* locked by GetFileSet */
}
void FilenameDB::CacheFile( const RString &sPath )
{
LOG->Warn( "Slow cache due to: %s", sPath.c_str() );
FlushDirCache( Dirname(sPath) );
}
/*
* Copyright (c) 2003-2004 Glenn Maynard
* All rights reserved.
+8 -8
View File
@@ -39,7 +39,7 @@ struct File
File( const RString &fn )
{
SetName( fn );
dir=false; size=-1; hash=-1; priv=NULL;
dir=false; size=-1; hash=-1; priv=NULL; dirp=NULL;
}
bool operator== (const File &rhs) const { return lname==rhs.lname; }
@@ -88,8 +88,7 @@ public:
void AddFile( const RString &sPath, int iSize, int iHash, void *pPriv=NULL );
void DelFile( const RString &sPath );
const File *GetFile( const RString &sPath );
const void *GetFilePriv( const RString &sPath );
void *GetFilePriv( const RString &sPath );
/* This handles at most two * wildcards. If we need anything more complicated,
* we'll need to use fnmatch or regex. */
@@ -105,13 +104,16 @@ public:
int GetFileHash( const RString &sFilePath );
void GetDirListing( const RString &sPath, vector<RString> &asAddTo, bool bOnlyDirs, bool bReturnPathToo );
void FlushDirCache();
void FlushDirCache( const RString &sDir = RString() );
void GetFileSetCopy( const RString &dir, FileSet &out );
/* Probably slow, so override it. */
virtual void CacheFile( const RString &sPath );
protected:
RageEvent m_Mutex;
const File *GetFile( const RString &sPath );
FileSet *GetFileSet( const RString &sDir, bool create=true );
/* Directories we have cached: */
@@ -134,9 +136,7 @@ class NullFilenameDB: public FilenameDB
{
public:
NullFilenameDB() { ExpireSeconds = -1; }
protected:
void PopulateFileSet( FileSet &fs, const RString &sPath ) { }
void CacheFile( const RString &sPath ) { }
};
#endif
-1
View File
@@ -210,7 +210,6 @@ void ScreenEditMenu::MenuStart( const InputEventPlus &input )
}
file.Close();
FILEMAN->FlushDirCache( sDir );
FILEMAN->Remove( sTempFile );
break;
}
-2
View File
@@ -2801,8 +2801,6 @@ void ScreenGameplay::SaveReplay()
//
// Find a file name for the screenshot
//
FILEMAN->FlushDirCache( "Save/" );
vector<RString> files;
GetDirListing( "Save/replay*", files, false, false );
sort( files.begin(), files.end() );
+2 -1
View File
@@ -127,7 +127,8 @@ void ScreenOptionsManageCourses::BeginScreen()
}
m_vpCourses.clear();
FlushDirCache();
// XXX: Why are we flushing here?
FILEMAN->FlushDirCache();
PROFILEMAN->LoadMachineProfileEdits();
switch( EDIT_MODE.GetValue() )
-1
View File
@@ -639,7 +639,6 @@ void ScreenPackages::HTTPUpdate()
if( m_bIsPackage && m_iResponseCode < 300 )
{
m_fOutputFile.Close();
FlushDirCache();
RefreshPackages();
m_iDownloaded = 0;
}
-7
View File
@@ -203,8 +203,6 @@ static void CopyEdits( const RString &sFromProfileDir, const RString &sToProfile
continue;
}
}
FILEMAN->FlushDirCache( sToDir );
}
// TODO: Seprarate copying stats for steps and courses
@@ -225,10 +223,7 @@ static void CopyEdits( const RString &sFromProfileDir, const RString &sToProfile
else
iNumErrored++;
}
FILEMAN->FlushDirCache( sToDir );
}
}
static LocalizedString EDITS_NOT_COPIED ( "ScreenServiceAction", "Edits not copied - No memory cards ready." );
@@ -298,8 +293,6 @@ static void SyncFiles( const RString &sFromDir, const RString &sToDir, const RSt
else
++iNumFailed;
}
FILEMAN->FlushDirCache( sToDir );
}
static void SyncEdits( const RString &sFromDir, const RString &sToDir, int &iNumAdded, int &iNumDeleted, int &iNumOverwritten, int &iNumFailed )
-1
View File
@@ -283,7 +283,6 @@ void StatsManager::CommitStatsToProfiles( const StageStats *pSS )
if( bSaved )
{
FILEMAN->FlushDirCache( UPLOAD_DIR );
RString sStatsXmlSigFile = fn + SIGNATURE_APPEND;
CryptManager::SignFileToFile(fn, sStatsXmlSigFile);
}
-5
View File
@@ -1224,11 +1224,6 @@ RString StepMania::SaveScreenshot( RString sDir, bool bSaveCompressed, bool bMak
SCREENMAN->PlayScreenshotSound();
// We wrote a new file, and SignFile won't pick it up unless we invalidate
// the Dir cache. There's got to be a better way of doing this than
// thowing out all the cache. -Chris
FILEMAN->FlushDirCache( sDir );
if( PREFSMAN->m_bSignProfileData && bMakeSignature )
CryptManager::SignFileToFile( sPath );
-4
View File
@@ -591,10 +591,6 @@ try_element_again:
if( asElementPaths.size() > 1 )
{
/* Why do we flush here? Aborting ends, retrying calls ReloadMetrics which will flush
* and ignoring doesn't do anything further, nothing that would seem to require
* flushing the cache. -- Steve */
FILEMAN->FlushDirCache( GetThemeDirFromName(sThemeName) );
g_ThemePathCache[category].clear();
RString message = ssprintf(