diff --git a/Docs/Luadoc/LuaDocumentation.xml b/Docs/Luadoc/LuaDocumentation.xml
index 61391fdbfd..b07c150616 100644
--- a/Docs/Luadoc/LuaDocumentation.xml
+++ b/Docs/Luadoc/LuaDocumentation.xml
@@ -3562,16 +3562,20 @@ end
NETWORK:HttpRequest{
url="https://api.example.com",
- method="GET", -- default: "GET"
- body="", -- default: ""
- multipartBoundary="", -- default: ""
- headers={ -- default: {}
+ method="GET", -- default: "GET"
+ body="", -- default: ""
+ multipartBoundary="", -- default: ""
+ headers={ -- default: {}
["Accept-Language"]="en-US",
["Cookie"]="sessionId=42",
},
- connectTimeout=3, -- default: 60
- transferTimeout=10, -- default: 1800
- onResponse=function(response) -- default: no callback
+ connectTimeout=3, -- default: 60
+ transferTimeout=10, -- default: 1800
+ downloadFile="", -- default: no download file
+ onProgress=function(currentBytes, totalBytes) -- default: no callback
+ ...
+ end,
+ onResponse=function(response) -- default: no callback
...
end,
}
@@ -3598,6 +3602,9 @@ NETWORK:HttpRequest{
errorMessage="access to https://api.example.com is not allowed",
}
+ If the downloadFile parameter is set, the response body is not accumulated in memory, but written to /Downloads/${downloadFile}.
+ The file is available in the onResponse callback where it can be unzipped/copied to another location using FILEMAN:Unzip()/FILEMAN:Copy() respectively.
+ The file is deleted once the callback returns.
Returns the URL encoded representation of value.
diff --git a/extern/CMakeProject-ixwebsocket.cmake b/extern/CMakeProject-ixwebsocket.cmake
index 0d9a995873..da56c6f874 100644
--- a/extern/CMakeProject-ixwebsocket.cmake
+++ b/extern/CMakeProject-ixwebsocket.cmake
@@ -149,5 +149,4 @@ set(IXWEBSOCKET_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/IXWebSocket-11.3.2")
target_include_directories(ixwebsocket PUBLIC
"zlib"
$
- $
)
diff --git a/src/NetworkManager.cpp b/src/NetworkManager.cpp
index 390241ecae..b5ceedc174 100644
--- a/src/NetworkManager.cpp
+++ b/src/NetworkManager.cpp
@@ -1,16 +1,22 @@
#include "global.h"
#include "NetworkManager.h"
#include "LuaManager.h"
+#include "ProductInfo.h"
+#include "RageFile.h"
+#include "RageFileManager.h"
#include "RageLog.h"
#include "RageUtil.h"
#include "StdString.h"
+#include "ver.h"
#include
#include
#include
#include
+#include
#include
+#include
#include
#include
@@ -23,6 +29,7 @@ Preference NetworkManager::httpAllowHosts("HttpAllowHosts", "", nullptr
static const char *HttpErrorCodeNames[] = {
"Blocked",
"UnknownError",
+ "FileError",
"CannotConnect",
"Timeout",
"Gzip",
@@ -43,7 +50,7 @@ XToString(HttpErrorCode);
StringToX(HttpErrorCode);
LuaXType(HttpErrorCode);
-NetworkManager::NetworkManager() : httpClient(true)
+NetworkManager::NetworkManager() : httpClient(true), downloadClient(true)
{
ix::initNetSystem();
@@ -54,6 +61,8 @@ NetworkManager::NetworkManager() : httpClient(true)
lua_setglobal(L, "NETWORK");
LUA->Release(L);
}
+
+ this->ClearDownloads();
}
NetworkManager::~NetworkManager()
@@ -97,24 +106,75 @@ bool NetworkManager::IsUrlAllowed(const std::string& url)
HttpRequestFuturePtr NetworkManager::HttpRequest(const HttpRequestArgs& args)
{
- ix::HttpRequestArgsPtr req = this->httpClient.createRequest(args.url, args.method);
+ auto &client = args.downloadFile.empty() ? this->httpClient : this->downloadClient;
+ auto downloadFile = std::make_shared();
+ std::string downloadFilename;
+
+ ix::HttpRequestArgsPtr req = client.createRequest(args.url, args.method);
req->body = args.body;
req->multipartBoundary = args.multipartBoundary;
+ req->extraHeaders["User-Agent"] = this->GetUserAgent();
for (const auto& entry : args.headers)
{
req->extraHeaders[entry.first] = entry.second;
}
- if (args.connectTimeout > 0) {
+ if (!args.downloadFile.empty())
+ {
+ downloadFilename = std::string("/Downloads/") + args.downloadFile;
+ if (!downloadFile->Open(downloadFilename, RageFile::WRITE | RageFile::STREAMED))
+ {
+ req->cancel = true;
+ }
+
+ req->onChunkCallback = [args, downloadFile, req](const std::string& data) {
+ if (downloadFile->Write(data.c_str(), data.size()) < 0)
+ {
+ req->cancel = true;
+ }
+ };
+
+ // Don't timeout downloads by default. This value might be
+ // overwritten below if a value is specified in the request.
+ req->transferTimeout = INT_MAX;
+
+ }
+
+ if (args.connectTimeout > 0)
req->connectTimeout = args.connectTimeout;
- }
- if (args.transferTimeout > 0) {
+ if (args.transferTimeout > 0)
req->transferTimeout = args.transferTimeout;
- }
- this->httpClient.performRequest(req, args.onResponse);
+ if (args.onProgress)
+ req->onProgressCallback = args.onProgress;
+
+ client.performRequest(req, [args, downloadFile, downloadFilename](const ix::HttpResponsePtr& response) {
+ if (downloadFile->IsOpen())
+ {
+ RString error = downloadFile->GetError();
+ downloadFile->Close();
+
+ if (!error.empty())
+ {
+ std::string errorMessage = "could not write to " + downloadFile->GetPath() + ": " + error;
+ if (args.onFileError)
+ args.onFileError(errorMessage);
+
+ FILEMAN->Remove(downloadFilename);
+ return;
+ }
+ }
+
+ if (args.onResponse)
+ args.onResponse(response);
+
+ if (!args.downloadFile.empty())
+ {
+ FILEMAN->Remove(downloadFilename);
+ }
+ });
return std::make_shared(req);
}
@@ -129,7 +189,28 @@ std::string NetworkManager::EncodeQueryParameters(const std::unordered_maphttpClient.serializeHttpParameters(query);
}
-int HttpRequestFuture::collect(lua_State *L)
+std::string NetworkManager::GetUserAgent()
+{
+ std::stringstream ss;
+ ss << PRODUCT_FAMILY << "/" << product_version;
+ return ss.str();
+}
+
+void NetworkManager::ClearDownloads()
+{
+ vector files;
+ FILEMAN->GetDirListing("/Downloads/*", files, false, true);
+
+ for (const auto& file : files)
+ {
+ if (FILEMAN->IsADirectory(file))
+ FILEMAN->DeleteRecursive(file + "/");
+ else
+ FILEMAN->Remove(file);
+ }
+}
+
+int HttpRequestFuture::Collect(lua_State *L)
{
void *udata = luaL_checkudata(L, 1, "HttpRequestFuture");
auto futptr = static_cast(udata);
@@ -152,7 +233,7 @@ int HttpRequestFuture::Cancel(lua_State *L)
static void registerHttpRequestMetatable(lua_State *L)
{
const luaL_Reg HttpRequest_meta[] = {
- {"__gc", HttpRequestFuture::collect},
+ {"__gc", HttpRequestFuture::Collect},
{"Cancel", HttpRequestFuture::Cancel},
{NULL, NULL},
};
@@ -183,6 +264,8 @@ public:
luaL_checktype(L, 1, LUA_TTABLE);
HttpRequestArgs args;
+ int onProgressRef = LUA_NOREF;
+ int onResponseRef = LUA_NOREF;
lua_getfield(L, 1, "url");
if (lua_isnil(L, -1))
@@ -311,9 +394,44 @@ public:
}
lua_pop(L, 1);
- int onResponseRef = LUA_NOREF;
+ lua_getfield(L, 1, "downloadFile");
+ if (!lua_isnil(L, -1))
+ {
+ if (lua_isstring(L, -1))
+ {
+ args.downloadFile = lua_tostring(L, -1);
+ }
+ else
+ {
+ luaL_error(L, "downloadFile must be a string");
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_getfield(L, 1, "onProgress");
+ if (!lua_isnil(L, -1))
+ {
+ if (lua_isfunction(L, -1))
+ {
+ lua_pushvalue(L, -1);
+ onProgressRef = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ args.onProgress = [onProgressRef](int current, int total)
+ {
+ Lua *L = LUA->Get();
+ handleProgress(L, current, total, onProgressRef);
+ LUA->Release(L);
+
+ return true;
+ };
+ }
+ else
+ {
+ luaL_error(L, "onProgress must be a function");
+ }
+ }
+ lua_pop(L, 1);
- // do this as the last check, so we don't leak lua references on error
lua_getfield(L, 1, "onResponse");
if (!lua_isnil(L, -1))
{
@@ -321,21 +439,39 @@ public:
{
lua_pushvalue(L, -1);
onResponseRef = luaL_ref(L, LUA_REGISTRYINDEX);
-
- args.onResponse = [onResponseRef](const ix::HttpResponsePtr& response)
- {
- Lua *L = LUA->Get();
- handleHttpResponse(L, response, onResponseRef);
- LUA->Release(L);
- };
}
else
{
+ luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
luaL_error(L, "onResponse must be a function");
}
}
lua_pop(L, 1);
+ args.onFileError = [onProgressRef, onResponseRef](const std::string& errorMessage)
+ {
+ Lua *L = LUA->Get();
+
+ luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
+
+ if (onResponseRef != LUA_NOREF)
+ handleFileError(L, errorMessage, onResponseRef);
+
+ LUA->Release(L);
+ };
+
+ args.onResponse = [onProgressRef, onResponseRef](const ix::HttpResponsePtr& response)
+ {
+ Lua *L = LUA->Get();
+
+ luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
+
+ if (onResponseRef != LUA_NOREF)
+ handleHttpResponse(L, response, onResponseRef);
+
+ LUA->Release(L);
+ };
+
if (p->IsUrlAllowed(args.url))
{
auto fut = p->HttpRequest(args);
@@ -349,6 +485,7 @@ public:
else
{
LOG->Warn("blocked access to %s", args.url.c_str());
+ luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
if (onResponseRef != LUA_NOREF)
{
handleUrlForbidden(L, args.url, onResponseRef);
@@ -431,6 +568,23 @@ private:
LuaHelpers::RunScriptOnStack(L, error, 1, 0, true);
}
+ static void handleFileError(Lua *L, const std::string& errorMessage, int onResponseRef)
+ {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, onResponseRef);
+ luaL_unref(L, LUA_REGISTRYINDEX, onResponseRef);
+
+ lua_newtable(L);
+
+ LuaHelpers::Push(L, HttpErrorCode_FileError);
+ lua_setfield(L, -2, "error");
+
+ lua_pushstring(L, errorMessage.c_str());
+ lua_setfield(L, -2, "errorMessage");
+
+ RString error = "Lua error in HTTP response handler: ";
+ LuaHelpers::RunScriptOnStack(L, error, 1, 0, true);
+ }
+
static void handleHttpResponse(Lua *L, const ix::HttpResponsePtr& response, int onResponseRef)
{
lua_rawgeti(L, LUA_REGISTRYINDEX, onResponseRef);
@@ -493,6 +647,16 @@ private:
RString error = "Lua error in HTTP response handler: ";
LuaHelpers::RunScriptOnStack(L, error, 1, 0, true);
}
+
+ static void handleProgress(Lua *L, int current, int total, int onProgressRef)
+ {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, onProgressRef);
+ lua_pushinteger(L, current);
+ lua_pushinteger(L, total);
+
+ RString error = "Lua error in HTTP progress handler: ";
+ LuaHelpers::RunScriptOnStack(L, error, 2, 0, true);
+ }
};
LUA_REGISTER_CLASS(NetworkManager)
diff --git a/src/NetworkManager.h b/src/NetworkManager.h
index 2bf2e57887..c5b1f0b842 100644
--- a/src/NetworkManager.h
+++ b/src/NetworkManager.h
@@ -4,8 +4,10 @@
#include "Preference.h"
#include "StdString.h"
+#include
#include
#include
+#include
#include
#include
@@ -21,6 +23,7 @@ enum HttpErrorCode
{
HttpErrorCode_Blocked,
HttpErrorCode_UnknownError,
+ HttpErrorCode_FileError,
// from IXWebSocket
HttpErrorCode_CannotConnect,
@@ -44,7 +47,7 @@ enum HttpErrorCode
};
const RString& HttpErrorCodeToString(HttpErrorCode dc);
HttpErrorCode StringToHttpErrorCode(const RString& sDC);
-LuaDeclareType( HttpErrorCode );
+LuaDeclareType(HttpErrorCode);
struct HttpRequestArgs
{
@@ -55,7 +58,10 @@ struct HttpRequestArgs
std::unordered_map headers;
int connectTimeout = -1;
int transferTimeout = -1;
- std::function onResponse = [](const ix::HttpResponsePtr& response) {};
+ std::string downloadFile;
+ std::function onProgress;
+ std::function onResponse;
+ std::function onFileError;
};
class HttpRequestFuture
@@ -63,7 +69,7 @@ class HttpRequestFuture
public:
HttpRequestFuture(ix::HttpRequestArgsPtr& args) : args(args) {};
- static int collect(lua_State *L);
+ static int Collect(lua_State *L);
static int Cancel(lua_State *L);
private:
@@ -87,7 +93,11 @@ public:
void PushSelf(lua_State *L);
private:
+ std::string GetUserAgent();
+ void ClearDownloads();
+
ix::HttpClient httpClient;
+ ix::HttpClient downloadClient;
static Preference httpEnabled;
static Preference httpAllowHosts;
diff --git a/src/arch/ArchHooks/ArchHooks_MacOSX.mm b/src/arch/ArchHooks/ArchHooks_MacOSX.mm
index e748eee1ed..060ff5b5b2 100644
--- a/src/arch/ArchHooks/ArchHooks_MacOSX.mm
+++ b/src/arch/ArchHooks/ArchHooks_MacOSX.mm
@@ -354,6 +354,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable )
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/CDTitles", dir), "/CDTitles" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Characters", dir), "/Characters" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Courses", dir), "/Courses" );
+ FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Downloads", dir), "/Downloads" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/NoteSkins", dir), "/NoteSkins" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Packages", dir), "/Packages" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Songs", dir), "/Songs" );
diff --git a/src/arch/ArchHooks/ArchHooks_Unix.cpp b/src/arch/ArchHooks/ArchHooks_Unix.cpp
index c48bac13fc..24dbaf455a 100644
--- a/src/arch/ArchHooks/ArchHooks_Unix.cpp
+++ b/src/arch/ArchHooks/ArchHooks_Unix.cpp
@@ -403,6 +403,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable )
FILEMAN->Mount( "dir", sUserDataPath + "/CDTitles", "/CDTitles" );
FILEMAN->Mount( "dir", sUserDataPath + "/Characters", "/Characters" );
FILEMAN->Mount( "dir", sUserDataPath + "/Courses", "/Courses" );
+ FILEMAN->Mount( "dir", sUserDataPath + "/Downloads", "/Downloads" );
FILEMAN->Mount( "dir", sUserDataPath + "/Logs", "/Logs" );
FILEMAN->Mount( "dir", sUserDataPath + "/NoteSkins", "/NoteSkins" );
FILEMAN->Mount( "dir", sUserDataPath + "/Packages", "/Packages" );
diff --git a/src/arch/ArchHooks/ArchHooks_Win32Static.cpp b/src/arch/ArchHooks/ArchHooks_Win32Static.cpp
index 8676702add..c07ac0085f 100644
--- a/src/arch/ArchHooks/ArchHooks_Win32Static.cpp
+++ b/src/arch/ArchHooks/ArchHooks_Win32Static.cpp
@@ -82,6 +82,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable )
FILEMAN->Mount( "dir", sAppDataDir + "/CDTitles", "/CDTitles" );
FILEMAN->Mount( "dir", sAppDataDir + "/Characters", "/Characters" );
FILEMAN->Mount( "dir", sAppDataDir + "/Courses", "/Courses" );
+ FILEMAN->Mount( "dir", sAppDataDir + "/Downloads", "/Downloads" );
FILEMAN->Mount( "dir", sAppDataDir + "/Logs", "/Logs" );
FILEMAN->Mount( "dir", sAppDataDir + "/NoteSkins", "/NoteSkins" );
FILEMAN->Mount( "dir", sAppDataDir + "/Packages", "/Packages" );