Allow virtual memory and paging on the Xbox

This commit is contained in:
Ryan Dortmans
2004-11-08 02:35:14 +00:00
parent 6eb08e4140
commit 4fb20257b8
4 changed files with 715 additions and 2 deletions
+35 -2
View File
@@ -60,6 +60,11 @@
#include "CryptManager.h"
#include "NetworkSyncManager.h"
#if defined(XBOX)
#include "Archutils/Xbox/VirtualMemory.h"
#include <new.h>
#endif
#define ZIPS_DIR "Packages/"
#ifdef _WINDOWS
@@ -913,16 +918,30 @@ static void ProcessArgsSecond()
}
#ifdef _XBOX
void XboxMain();
void __cdecl main()
{
_set_new_handler(NoMemory);
_set_new_mode(1);
__try
{
XboxMain();
}
__except(CheckPageFault(GetExceptionInformation())) { }
}
void XboxMain()
#else
int main(int argc, char* argv[])
#endif
{
#ifdef _XBOX
#if defined(XBOX)
int argc = 1;
char *argv[] = {"default.xbe"};
#endif
g_argc = argc;
g_argv = argv;
@@ -932,6 +951,7 @@ int main(int argc, char* argv[])
CString g_sErrorString = "";
#if !defined(DEBUG)
/* Always catch RageExceptions; they should always have distinct strings. */
try { /* RageException */
/* Tricky: for other exceptions, we want a backtrace. To do this in Windows,
@@ -962,6 +982,18 @@ int main(int argc, char* argv[])
ApplyLogPreferences();
#if defined(XBOX)
if(PREFSMAN->m_bEnableVirtualMemory)
{
if(!vmem_Manager.Init(1024 * 1024 * PREFSMAN->m_iPageFileSize, 1024 * PREFSMAN->m_iPageSize, 1024 * PREFSMAN->m_iPageThreshold))
return;
}
/* Logging the virtual memory manager seems to crash on exit, so it should be enabled only
* for debugging. */
vmem_Manager.SetLogging(PREFSMAN->m_bLogVirtualMemory);
#endif
WriteLogHeader();
/* Set up alternative filesystem trees. */
@@ -1179,6 +1211,7 @@ int main(int argc, char* argv[])
#ifndef _XBOX
return 0;
#endif
}
+12
View File
@@ -395,6 +395,12 @@ cl /Zl /nologo /c verstub.cpp /Fo$(IntDir)\
<File
RelativePath="ScreenEnding.h">
</File>
<File
RelativePath="ScreenEndlessBreak.cpp">
</File>
<File
RelativePath="ScreenEndlessBreak.h">
</File>
<File
RelativePath="ScreenEvaluation.cpp">
</File>
@@ -1321,6 +1327,12 @@ cl /Zl /nologo /c verstub.cpp /Fo$(IntDir)\
<File
RelativePath=".\archutils\Xbox\VirtualKeyboard.h">
</File>
<File
RelativePath=".\archutils\Xbox\VirtualMemory.cpp">
</File>
<File
RelativePath=".\archutils\Xbox\VirtualMemory.h">
</File>
</Filter>
</Filter>
<Filter
@@ -0,0 +1,578 @@
#include "global.h"
#include "VirtualMemory.h"
#include "RageLog.h"
#include <new>
VirtualMemoryManager vmem_Manager;
VirtualMemoryManager::VirtualMemoryManager()
{
pages = 0;
pageLRU = -1;
inited = false;
}
VirtualMemoryManager::~VirtualMemoryManager()
{
Destroy();
}
bool VirtualMemoryManager::Init(unsigned long totalPageSize, unsigned long sizePerPage, unsigned long thold)
{
threshold = thold;
totalPages = totalPageSize / sizePerPage;
pageSize = sizePerPage;
if(totalPageSize % sizePerPage != 0)
totalPageSize = sizePerPage * totalPages;
// initialise the pages array
// bypass the overridden new by using HeapAlloc
pages = (vm_page*)HeapAlloc(GetProcessHeap(), 0, totalPages * sizeof(vm_page));
if(pages == NULL)
return false;
// create the page file on Z drive
vmemFile = CreateFile( "Z:\\xxpagefile.sys", GENERIC_READ|GENERIC_WRITE, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL ) ;
if(vmemFile == INVALID_HANDLE_VALUE)
return false;
// set the file size
// find the current file size
unsigned long fileSize = SetFilePointer(vmemFile, 0, 0, FILE_END);
if(fileSize == INVALID_SET_FILE_POINTER)
return false;
if(fileSize < totalPageSize)
{
fileSize = SetFilePointer(vmemFile, totalPageSize, 0, FILE_BEGIN);
if(fileSize == INVALID_SET_FILE_POINTER)
return false;
}
// Reserve the virtual memory and get the base address
baseAddress = (DWORD)VirtualAlloc(NULL, totalPageSize, MEM_RESERVE, PAGE_NOACCESS);
if(baseAddress == NULL)
return false;
// initialise the page array
for(unsigned long i = 0; i < totalPages; i++)
{
pages[i].startAddress = baseAddress + (i * pageSize);
pages[i].headPage = -1;
pages[i].committed = false;
pages[i].sizeInPages = 0;
pages[i].sizeInBytes = 0;
pages[i].pageFaults = 0;
pages[i].locked = false;
}
inited = true;
return true;
}
void VirtualMemoryManager::Destroy()
{
if(pages != 0)
{
VirtualFree((LPVOID)baseAddress, 0, MEM_RELEASE);
HeapFree(GetProcessHeap(), 0, pages);
CloseHandle(vmemFile);
}
}
void* VirtualMemoryManager::Allocate(size_t size)
{
if(!inited)
return NULL;
unsigned long startPage = -1;
unsigned long freeSegments = 0;
unsigned long sizeInPages = (size / pageSize) + 1;
if(size % pageSize == 0)
sizeInPages--;
// find a contiguous group of pages that will fit the data
for(unsigned long i = 0; i < totalPages; i++)
{
if(pages[i].sizeInPages != 0)
{
startPage = -1;
freeSegments = 0;
i += pages[i].sizeInPages - 1; // go to next page segment
}
else
{
if(startPage == -1)
{
startPage = i;
freeSegments = 1;
}
else
freeSegments++;
if(sizeInPages == freeSegments)
{
if(LOG && logging)
LOG->Trace("Allocating pages %u to %u", startPage, startPage + freeSegments - 1);
// commit this to memory
DWORD ret = (DWORD)VirtualAlloc((LPVOID)pages[startPage].startAddress, size, MEM_COMMIT, PAGE_READWRITE);
while(ret == NULL)
{
bool swappedOut = DecommitLRU();
if(!swappedOut)
{
if(LOG)
LOG->Trace("VMem error: out of memory with no pages to swap out left");
return NULL;
}
ret = (DWORD)VirtualAlloc((LPVOID)pages[startPage].startAddress, size, MEM_COMMIT, PAGE_READWRITE);
}
pageLRU = (startPage + sizeInPages) % totalPages;
for(unsigned long j = startPage; j < startPage + freeSegments; j++)
{
pages[j].headPage = startPage;
pages[j].pageFaults = 0;
pages[j].sizeInPages = freeSegments;
pages[j].sizeInBytes = size;
pages[j].committed = true;
pages[i].locked = false;
}
return (void*) ret;
}
}
}
if(LOG)
LOG->Trace("VMem error: Couldn't find contiguous group of pages to allocate");
return NULL;
}
bool VirtualMemoryManager::Free(void *ptr)
{
if(!inited)
return false;
// check that the address is within the virtual address bounds
if((DWORD)ptr < baseAddress || (DWORD)ptr >= baseAddress + (totalPages * pageSize))
{
return false;
}
// find the page(s) to free
DWORD offset = (DWORD)ptr - baseAddress;
unsigned long pageIndex = offset / pageSize;
if(pages[pageIndex].headPage == -1)
{
return false;
}
pageIndex = pages[pageIndex].headPage;
ptr = (void *)pages[pageIndex].startAddress;
unsigned long endPage = pageIndex + pages[pageIndex].sizeInPages;
unsigned long size = pages[pageIndex].sizeInBytes;
if(size == 0)
{
return false; // trying to free unallocated memory
}
if(LOG && logging)
LOG->Trace("Freeing pages %u to %u", pageIndex, endPage - 1);
if(pages[pageIndex].committed)
VirtualFree(ptr, size, MEM_DECOMMIT);
for(unsigned long i = pageIndex; i < endPage; i++)
{
pages[i].headPage = -1;
pages[i].committed = false;
pages[i].pageFaults = 0;
pages[i].sizeInBytes = 0;
pages[i].sizeInPages = 0;
pages[i].locked = false;
}
return true;
}
bool VirtualMemoryManager::PageFault(void *ptr)
{
if(!inited)
return false;
// check that the address is within the virtual address bounds
if((DWORD)ptr < baseAddress || (DWORD)ptr >= baseAddress + (totalPages * pageSize))
{
if(LOG)
{
LOG->Trace("Vmem error: Page fault outside virtual memory bounds");
LOG->Trace("Address: %u, bounds: %u to %u", (DWORD)ptr, baseAddress, baseAddress + (totalPages * pageSize));
}
return false;
}
// find the page segment that the fault occurred
unsigned long offset = (DWORD)ptr - baseAddress;
unsigned long pageIndex = offset / pageSize;
unsigned long startPage = pages[pageIndex].headPage;
if(startPage == -1)
{
if(LOG)
LOG->Trace("VMem error: Trying to access memory that wasn't allocated");
// trying to access memory that wasn't allocated
return false;
}
pageLRU = (startPage + pages[startPage].sizeInPages) % totalPages;
if(LOG && logging)
LOG->Trace("Reallocating pages %u to %u", startPage, startPage + pages[startPage].sizeInPages - 1);
DWORD ret = (DWORD)VirtualAlloc((LPVOID)pages[startPage].startAddress, pages[startPage].sizeInBytes, MEM_COMMIT, PAGE_READWRITE);
while(ret == NULL)
{
bool swappedOut = DecommitLRU();
if(!swappedOut)
{
if(LOG)
LOG->Trace("VMem error: no pages left to swap out while reallocating");
return false;
}
ret = (DWORD)VirtualAlloc((LPVOID)pages[startPage].startAddress, pages[startPage].sizeInBytes, MEM_COMMIT, PAGE_READWRITE);
}
for(unsigned long i = startPage; i < startPage + pages[startPage].sizeInPages; i++)
{
pages[i].committed = true;
pages[i].pageFaults++;
}
DWORD numRead;
SetFilePointer(vmemFile, pages[startPage].startAddress - baseAddress, 0, FILE_BEGIN);
ReadFile(vmemFile, (void *)pages[startPage].startAddress, pages[startPage].sizeInBytes, &numRead, NULL);
return true;
}
bool VirtualMemoryManager::DecommitLRU()
{
if(!inited)
return false;
// choose random LRU
pageLRU = rand() % totalPages;
for(unsigned long i = 0; i < totalPages; i++)
{
unsigned long index = (pageLRU + i) % totalPages;
if(index == 0)
index++;
if(pages[index].headPage == index && pages[index].committed && !pages[index].locked) // this is a head page
{
DWORD addr = pages[index].startAddress;
unsigned long size = pages[index].sizeInPages;
// decommit this page
// write to the page file
if(SetFilePointer(vmemFile, addr - baseAddress, 0, FILE_BEGIN) == INVALID_SET_FILE_POINTER)
return false;
DWORD written;
WriteFile(vmemFile, (LPCVOID)addr, pages[index].sizeInBytes, &written, NULL);
// reset the page data
if(LOG && logging)
LOG->Trace("Swapping out pages %i to %i", index, index + size - 1);
for(unsigned long j = index; j < index + size; j++)
{
pages[j].committed = false;
}
if(VirtualFree((LPVOID)addr, pages[index].sizeInBytes, MEM_DECOMMIT) == 0)
{
return false;
}
pageLRU = (pageLRU + pages[index].sizeInPages) % totalPages;
return true;
}
}
return false;
}
bool VirtualMemoryManager::EnsureFreeMemory(size_t size)
{
if(!inited)
return false;
MEMORYSTATUS ms;
GlobalMemoryStatus(&ms);
while(ms.dwAvailPhys < size)
{
if(LOG && logging)
LOG->Trace("Freeing memory: need %i, have %i", size, ms.dwAvailPhys);
if(!DecommitLRU())
{
if(LOG)
LOG->Trace("VMem error: No pages left to free while reserving memory");
return false;
}
}
return true;
}
void VirtualMemoryManager::Lock(void *ptr)
{
if(!inited)
return;
// check that the address is within the virtual address bounds
if((DWORD)ptr < baseAddress || (DWORD)ptr >= baseAddress + (totalPages * pageSize))
{
return;
}
// find the page(s) to free
DWORD offset = (DWORD)ptr - baseAddress;
unsigned long pageIndex = offset / pageSize;
if(pages[pageIndex].headPage == -1)
{
return;
}
pageIndex = pages[pageIndex].headPage;
unsigned long endPage = pageIndex + pages[pageIndex].sizeInPages;
if(!pages[pageIndex].committed)
{
if(!PageFault(ptr))
return;
}
for(unsigned long i = pageIndex; i < endPage; i++)
{
pages[i].locked = true;
}
}
void VirtualMemoryManager::Unlock(void *ptr)
{
if(!inited)
return;
// check that the address is within the virtual address bounds
if((DWORD)ptr < baseAddress || (DWORD)ptr >= baseAddress + (totalPages * pageSize))
{
return;
}
// find the page(s) to free
DWORD offset = (DWORD)ptr - baseAddress;
unsigned long pageIndex = offset / pageSize;
if(pages[pageIndex].headPage == -1)
{
return;
}
pageIndex = pages[pageIndex].headPage;
unsigned long endPage = pageIndex + pages[pageIndex].sizeInPages;
for(unsigned long i = pageIndex; i < endPage; i++)
{
pages[i].locked = false;
}
}
void* operator new (size_t size)
{
if(!vmem_Manager.IsValid())
return HeapAlloc(GetProcessHeap(), 0, size);
if(size > vmem_Manager.GetThreshold())
return vmem_Manager.Allocate(size);
else
{
void *ret = HeapAlloc(GetProcessHeap(), 0, size);
while(ret == NULL)
{
if(!vmem_Manager.DecommitLRU())
return NULL;
ret = HeapAlloc(GetProcessHeap(), 0, size);
}
return ret;
}
}
void* operator new[] (size_t size)
{
if(!vmem_Manager.IsValid())
return HeapAlloc(GetProcessHeap(), 0, size);
if(size > vmem_Manager.GetThreshold())
return vmem_Manager.Allocate(size);
else
{
void *ret = HeapAlloc(GetProcessHeap(), 0, size);
while(ret == NULL)
{
if(!vmem_Manager.DecommitLRU())
return NULL;
ret = HeapAlloc(GetProcessHeap(), 0, size);
}
return ret;
}
}
void operator delete (void *p)
{
if(vmem_Manager.IsValid())
{
if(vmem_Manager.Free(p))
return;
}
HeapFree(GetProcessHeap(), 0, p);
}
void operator delete[] (void *p)
{
if(vmem_Manager.IsValid())
{
if(vmem_Manager.Free(p))
return;
}
HeapFree(GetProcessHeap(), 0, p);
}
void *valloc(size_t size)
{
if(!vmem_Manager.IsValid())
return HeapAlloc(GetProcessHeap(), 0, size);
if(size > vmem_Manager.GetThreshold())
return vmem_Manager.Allocate(size);
else
{
void *ret = HeapAlloc(GetProcessHeap(), 0, size);
while(ret == NULL)
{
if(!vmem_Manager.DecommitLRU())
return NULL;
ret = HeapAlloc(GetProcessHeap(), 0, size);
}
return ret;
}
}
void vfree(void *ptr)
{
if(vmem_Manager.IsValid())
{
if(vmem_Manager.Free(ptr))
return;
}
HeapFree(GetProcessHeap(), 0, ptr);
}
int CheckPageFault(LPEXCEPTION_POINTERS e)
{
if(LOG && e->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION)
LOG->Trace("Exception: %u", e->ExceptionRecord->ExceptionCode);
if (e->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION)
return EXCEPTION_CONTINUE_SEARCH;
DWORD addr = (DWORD)e->ExceptionRecord->ExceptionInformation[1];
if(vmem_Manager.IsValid())
{
if(LOG && vmem_Manager.IsLogging())
LOG->Trace("Page fault");
if(vmem_Manager.PageFault((void *)addr))
return EXCEPTION_CONTINUE_EXECUTION;
}
return EXCEPTION_CONTINUE_SEARCH;
}
int NoMemory(size_t size)
{
if(LOG && vmem_Manager.IsLogging())
LOG->Trace("Out of memory, freeing up some...");
if(vmem_Manager.DecommitLRU())
{
if(LOG && vmem_Manager.IsLogging())
LOG->Trace("Freed some memory, trying again");
return 1;
}
else
{
if(LOG)
LOG->Trace("No memory left to free. Failed");
return 0;
}
}
/*
* (c) 2004 Ryan Dortmans
* All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, and/or sell copies of the Software, and to permit persons to
* whom the Software is furnished to do so, provided that the above
* copyright notice(s) and this permission notice appear in all copies of
* the Software and that both the above copyright notice(s) and this
* permission notice appear in supporting documentation.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
* THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
* INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT
* OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
* OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
* PERFORMANCE OF THIS SOFTWARE.
*/
@@ -0,0 +1,90 @@
#include <xtl.h>
#if !defined(VIRTUALMEMORY_H)
#define VIRTUALMEMORY_H
#pragma once
struct vm_page
{
DWORD startAddress; // start address for this page
unsigned long headPage; // 0 if not allocated. Otherwise, the index of the first page
// of this segment.
bool committed; // true if this page is committed to RAM (is otherwise in the page file)
bool locked; // true if this page should not be decommitted
int pageFaults; // number of times this page has been accessed when it wasn't committed
unsigned long sizeInPages; // size of the data segment in pages.
size_t sizeInBytes; // size of the data segment in bytes.
};
class VirtualMemoryManager
{
public:
VirtualMemoryManager();
~VirtualMemoryManager();
bool Init(unsigned long totalPageSize, unsigned long sizePerPage, unsigned long thold);
void Destroy();
void* Allocate(size_t size);
bool Free(void *ptr);
bool PageFault(void *ptr);
bool DecommitLRU();
bool EnsureFreeMemory(size_t size);
void Lock(void *ptr);
void Unlock(void *ptr);
void SetLogging(bool l) { logging = l; }
bool IsValid() { return inited; }
bool IsLogging() { return logging; }
unsigned long GetThreshold() { return threshold; }
protected:
vm_page *pages;
HANDLE vmemFile;
DWORD baseAddress;
unsigned long totalPages;
unsigned long pageSize;
unsigned long threshold;
unsigned long pageLRU;
bool inited;
bool logging;
};
void *valloc(size_t size);
void vfree(void *ptr);
//#define malloc(size) valloc(size)
//#define free(memblock) vfree(memblock)
extern VirtualMemoryManager vmem_Manager;
int CheckPageFault(LPEXCEPTION_POINTERS e);
int NoMemory(size_t size);
#endif
/*
* (c) 2004 Ryan Dortmans
* All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, and/or sell copies of the Software, and to permit persons to
* whom the Software is furnished to do so, provided that the above
* copyright notice(s) and this permission notice appear in all copies of
* the Software and that both the above copyright notice(s) and this
* permission notice appear in supporting documentation.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
* THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
* INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT
* OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
* OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
* PERFORMANCE OF THIS SOFTWARE.
*/