Add support for onProgress and downloadFile to NETWORK:HttpRequest()

Docs/Luadoc/Lua.xml

@@ -2486,21 +2486,22 @@
 		<Enum name='HttpErrorCode'>
 			<EnumValue name='&apos;HttpErrorCode_Blocked&apos;' value='0'/>
 			<EnumValue name='&apos;HttpErrorCode_UnknownError&apos;' value='1'/>
-			<EnumValue name='&apos;HttpErrorCode_CannotConnect&apos;' value='2'/>
+			<EnumValue name='&apos;HttpErrorCode_FileError&apos;' value='2'/>
-			<EnumValue name='&apos;HttpErrorCode_Timeout&apos;' value='3'/>
+			<EnumValue name='&apos;HttpErrorCode_CannotConnect&apos;' value='3'/>
-			<EnumValue name='&apos;HttpErrorCode_Gzip&apos;' value='4'/>
+			<EnumValue name='&apos;HttpErrorCode_Timeout&apos;' value='4'/>
-			<EnumValue name='&apos;HttpErrorCode_UrlMalformed&apos;' value='5'/>
+			<EnumValue name='&apos;HttpErrorCode_Gzip&apos;' value='5'/>
-			<EnumValue name='&apos;HttpErrorCode_CannotCreateSocket&apos;' value='6'/>
+			<EnumValue name='&apos;HttpErrorCode_UrlMalformed&apos;' value='6'/>
-			<EnumValue name='&apos;HttpErrorCode_SendError&apos;' value='7'/>
+			<EnumValue name='&apos;HttpErrorCode_CannotCreateSocket&apos;' value='7'/>
-			<EnumValue name='&apos;HttpErrorCode_ReadError&apos;' value='8'/>
+			<EnumValue name='&apos;HttpErrorCode_SendError&apos;' value='8'/>
-			<EnumValue name='&apos;HttpErrorCode_CannotReadStatusLine&apos;' value='9'/>
+			<EnumValue name='&apos;HttpErrorCode_ReadError&apos;' value='9'/>
-			<EnumValue name='&apos;HttpErrorCode_MissingStatus&apos;' value='10'/>
+			<EnumValue name='&apos;HttpErrorCode_CannotReadStatusLine&apos;' value='10'/>
-			<EnumValue name='&apos;HttpErrorCode_HeaderParsingError&apos;' value='11'/>
+			<EnumValue name='&apos;HttpErrorCode_MissingStatus&apos;' value='11'/>
-			<EnumValue name='&apos;HttpErrorCode_MissingLocation&apos;' value='12'/>
+			<EnumValue name='&apos;HttpErrorCode_HeaderParsingError&apos;' value='12'/>
-			<EnumValue name='&apos;HttpErrorCode_TooManyRedirects&apos;' value='13'/>
+			<EnumValue name='&apos;HttpErrorCode_MissingLocation&apos;' value='13'/>
-			<EnumValue name='&apos;HttpErrorCode_ChunkReadError&apos;' value='14'/>
+			<EnumValue name='&apos;HttpErrorCode_TooManyRedirects&apos;' value='14'/>
-			<EnumValue name='&apos;HttpErrorCode_CannotReadBody&apos;' value='15'/>
+			<EnumValue name='&apos;HttpErrorCode_ChunkReadError&apos;' value='15'/>
-			<EnumValue name='&apos;HttpErrorCode_Cancelled&apos;' value='16'/>
+			<EnumValue name='&apos;HttpErrorCode_CannotReadBody&apos;' value='16'/>
+			<EnumValue name='&apos;HttpErrorCode_Cancelled&apos;' value='17'/>
 		</Enum>
 		<Enum name='ImageCacheMode'>
 			<EnumValue name='&apos;ImageCacheMode_Off&apos;' value='0'/>

Docs/Luadoc/LuaDocumentation.xml

@@ -3562,16 +3562,20 @@ end
 <pre><code>
 NETWORK:HttpRequest{
 	url="https://api.example.com",
-	method="GET",                           -- default: "GET"
+	method="GET",                           	-- default: "GET"
-	body="",                                -- default: ""
+	body="",                                	-- default: ""
-	multipartBoundary="",                   -- default: ""
+	multipartBoundary="",                   	-- default: ""
-	headers={                               -- default: {}
+	headers={                               	-- default: {}
 		["Accept-Language"]="en-US",
 		["Cookie"]="sessionId=42",
 	},
-	connectTimeout=3,                       -- default: 60
+	connectTimeout=3,                       	-- default: 60
-	transferTimeout=10,                     -- default: 1800
+	transferTimeout=10,                     	-- default: 1800
-	onResponse=function(response)           -- default: no callback
+	downloadFile="",				-- default: no download file
+	onProgress=function(currentBytes, totalBytes)	-- default: no callback
+		...
+	end,
+	onResponse=function(response)           	-- default: no callback
 		...
 	end,
 }
@@ -3598,6 +3602,9 @@ NETWORK:HttpRequest{
 	errorMessage="access to https://api.example.com is not allowed",
 }
 </code></pre>
+	If the <code>downloadFile</code> parameter is set, the response body is not accumulated in memory, but written to <code>/Downloads/${downloadFile}</code>.
+	The file is available in the <code>onResponse</code> callback where it can be unzipped/copied to another location using <code>FILEMAN:Unzip()</code>/<code>FILEMAN:Copy()</code> respectively.
+	The file is deleted once the callback returns.
 	</Function>
 	<Function name='UrlEncode' return='string' arguments='string value'>
 		Returns the URL encoded representation of <code>value</code>.

extern/CMakeProject-ixwebsocket.cmake

@@ -149,5 +149,4 @@ set(IXWEBSOCKET_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/IXWebSocket-11.3.2")
 target_include_directories(ixwebsocket PUBLIC
   "zlib"
   $<BUILD_INTERFACE:${IXWEBSOCKET_INCLUDE_DIRS}/>
-  $<INSTALL_INTERFACE:include/ixwebsocket>
 )

src/NetworkManager.cpp

@@ -1,16 +1,22 @@
 #include "global.h"
 #include "NetworkManager.h"
 #include "LuaManager.h"
+#include "ProductInfo.h"
+#include "RageFile.h"
+#include "RageFileManager.h"
 #include "RageLog.h"
 #include "RageUtil.h"
 #include "StdString.h"
+#include "ver.h"
 
 #include <ixwebsocket/IXHttpClient.h>
 #include <ixwebsocket/IXNetSystem.h>
 #include <ixwebsocket/IXUrlParser.h>
 
 #include <algorithm>
+#include <climits>
 #include <memory>
+#include <sstream>
 #include <unordered_map>
 #include <vector>
 
@@ -23,6 +29,7 @@ Preference<RString> NetworkManager::httpAllowHosts("HttpAllowHosts", "", nullptr
 static const char *HttpErrorCodeNames[] = {
 	"Blocked",
 	"UnknownError",
+	"FileError",
 	"CannotConnect",
 	"Timeout",
 	"Gzip",
@@ -43,7 +50,7 @@ XToString(HttpErrorCode);
 StringToX(HttpErrorCode);
 LuaXType(HttpErrorCode);
 
-NetworkManager::NetworkManager() : httpClient(true)
+NetworkManager::NetworkManager() : httpClient(true), downloadClient(true)
 {
 	ix::initNetSystem();
 
@@ -54,6 +61,8 @@ NetworkManager::NetworkManager() : httpClient(true)
 		lua_setglobal(L, "NETWORK");
 		LUA->Release(L);
 	}
+
+	this->ClearDownloads();
 }
 
 NetworkManager::~NetworkManager()
@@ -97,24 +106,75 @@ bool NetworkManager::IsUrlAllowed(const std::string& url)
 
 HttpRequestFuturePtr NetworkManager::HttpRequest(const HttpRequestArgs& args)
 {
-	ix::HttpRequestArgsPtr req = this->httpClient.createRequest(args.url, args.method);
+	auto &client = args.downloadFile.empty() ? this->httpClient : this->downloadClient;
+	auto downloadFile = std::make_shared<RageFile>();
+	std::string downloadFilename;
+
+	ix::HttpRequestArgsPtr req = client.createRequest(args.url, args.method);
 	req->body = args.body;
 	req->multipartBoundary = args.multipartBoundary;
 
+	req->extraHeaders["User-Agent"] = this->GetUserAgent();
 	for (const auto& entry : args.headers)
 	{
 		req->extraHeaders[entry.first] = entry.second;
 	}
 
-	if (args.connectTimeout > 0) {
+	if (!args.downloadFile.empty())
+	{
+		downloadFilename = std::string("/Downloads/") + args.downloadFile;
+		if (!downloadFile->Open(downloadFilename, RageFile::WRITE | RageFile::STREAMED))
+		{
+			req->cancel = true;
+		}
+
+		req->onChunkCallback = [args, downloadFile, req](const std::string& data) {
+			if (downloadFile->Write(data.c_str(), data.size()) < 0)
+			{
+				req->cancel = true;
+			}
+		};
+
+		// Don't timeout downloads by default. This value might be
+		// overwritten below if a value is specified in the request.
+		req->transferTimeout = INT_MAX;
+
+	}
+
+	if (args.connectTimeout > 0)
 		req->connectTimeout = args.connectTimeout;
-	}
 
-	if (args.transferTimeout > 0) {
+	if (args.transferTimeout > 0)
 		req->transferTimeout = args.transferTimeout;
-	}
 
-	this->httpClient.performRequest(req, args.onResponse);
+	if (args.onProgress)
+		req->onProgressCallback = args.onProgress;
+
+	client.performRequest(req, [args, downloadFile, downloadFilename](const ix::HttpResponsePtr& response) {
+		if (downloadFile->IsOpen())
+		{
+			RString error = downloadFile->GetError();
+			downloadFile->Close();
+
+			if (!error.empty())
+			{
+				std::string errorMessage = "could not write to " + downloadFile->GetPath() + ": " + error;
+				if (args.onFileError)
+					args.onFileError(errorMessage);
+
+				FILEMAN->Remove(downloadFilename);
+				return;
+			}
+		}
+
+		if (args.onResponse)
+			args.onResponse(response);
+
+		if (!args.downloadFile.empty())
+		{
+			FILEMAN->Remove(downloadFilename);
+		}
+	});
 
 	return std::make_shared<HttpRequestFuture>(req);
 }
@@ -129,7 +189,28 @@ std::string NetworkManager::EncodeQueryParameters(const std::unordered_map<std::
 	return this->httpClient.serializeHttpParameters(query);
 }
 
-int HttpRequestFuture::collect(lua_State *L)
+std::string NetworkManager::GetUserAgent()
+{
+	std::stringstream ss;
+	ss << PRODUCT_FAMILY << "/" << product_version;
+	return ss.str();
+}
+
+void NetworkManager::ClearDownloads()
+{
+	vector<RString> files;
+	FILEMAN->GetDirListing("/Downloads/*", files, false, true);
+
+	for (const auto& file : files)
+	{
+		if (FILEMAN->IsADirectory(file))
+			FILEMAN->DeleteRecursive(file + "/");
+		else
+			FILEMAN->Remove(file);
+	}
+}
+
+int HttpRequestFuture::Collect(lua_State *L)
 {
 	void *udata = luaL_checkudata(L, 1, "HttpRequestFuture");
 	auto futptr = static_cast<HttpRequestFuturePtr*>(udata);
@@ -152,7 +233,7 @@ int HttpRequestFuture::Cancel(lua_State *L)
 static void registerHttpRequestMetatable(lua_State *L)
 {
 	const luaL_Reg HttpRequest_meta[] = {
-		{"__gc", HttpRequestFuture::collect},
+		{"__gc", HttpRequestFuture::Collect},
 		{"Cancel", HttpRequestFuture::Cancel},
 		{NULL, NULL},
 	};
@@ -183,6 +264,8 @@ public:
 		luaL_checktype(L, 1, LUA_TTABLE);
 
 		HttpRequestArgs args;
+		int onProgressRef = LUA_NOREF;
+		int onResponseRef = LUA_NOREF;
 
 		lua_getfield(L, 1, "url");
 		if (lua_isnil(L, -1))
@@ -311,9 +394,44 @@ public:
 		}
 		lua_pop(L, 1);
 
-		int onResponseRef = LUA_NOREF;
+		lua_getfield(L, 1, "downloadFile");
+		if (!lua_isnil(L, -1))
+		{
+			if (lua_isstring(L, -1))
+			{
+				args.downloadFile = lua_tostring(L, -1);
+			}
+			else
+			{
+				luaL_error(L, "downloadFile must be a string");
+			}
+		}
+		lua_pop(L, 1);
+
+		lua_getfield(L, 1, "onProgress");
+		if (!lua_isnil(L, -1))
+		{
+			if (lua_isfunction(L, -1))
+			{
+				lua_pushvalue(L, -1);
+				onProgressRef = luaL_ref(L, LUA_REGISTRYINDEX);
+
+				args.onProgress = [onProgressRef](int current, int total)
+				{
+					Lua *L = LUA->Get();
+					handleProgress(L, current, total, onProgressRef);
+					LUA->Release(L);
+
+					return true;
+				};
+			}
+			else
+			{
+				luaL_error(L, "onProgress must be a function");
+			}
+		}
+		lua_pop(L, 1);
 
-		// do this as the last check, so we don't leak lua references on error
 		lua_getfield(L, 1, "onResponse");
 		if (!lua_isnil(L, -1))
 		{
@@ -321,21 +439,39 @@ public:
 			{
 				lua_pushvalue(L, -1);
 				onResponseRef = luaL_ref(L, LUA_REGISTRYINDEX);
-
-				args.onResponse = [onResponseRef](const ix::HttpResponsePtr& response)
-				{
-					Lua *L = LUA->Get();
-					handleHttpResponse(L, response, onResponseRef);
-					LUA->Release(L);
-				};
 			}
 			else
 			{
+				luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
 				luaL_error(L, "onResponse must be a function");
 			}
 		}
 		lua_pop(L, 1);
 
+		args.onFileError = [onProgressRef, onResponseRef](const std::string& errorMessage)
+		{
+			Lua *L = LUA->Get();
+
+			luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
+
+			if (onResponseRef != LUA_NOREF)
+				handleFileError(L, errorMessage, onResponseRef);
+
+			LUA->Release(L);
+		};
+
+		args.onResponse = [onProgressRef, onResponseRef](const ix::HttpResponsePtr& response)
+		{
+			Lua *L = LUA->Get();
+
+			luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
+
+			if (onResponseRef != LUA_NOREF)
+				handleHttpResponse(L, response, onResponseRef);
+
+			LUA->Release(L);
+		};
+
 		if (p->IsUrlAllowed(args.url))
 		{
 			auto fut = p->HttpRequest(args);
@@ -349,6 +485,7 @@ public:
 		else
 		{
 			LOG->Warn("blocked access to %s", args.url.c_str());
+			luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
 			if (onResponseRef != LUA_NOREF)
 			{
 				handleUrlForbidden(L, args.url, onResponseRef);
@@ -431,6 +568,23 @@ private:
 		LuaHelpers::RunScriptOnStack(L, error, 1, 0, true);
 	}
 
+	static void handleFileError(Lua *L, const std::string& errorMessage, int onResponseRef)
+	{
+		lua_rawgeti(L, LUA_REGISTRYINDEX, onResponseRef);
+		luaL_unref(L, LUA_REGISTRYINDEX, onResponseRef);
+
+		lua_newtable(L);
+
+		LuaHelpers::Push(L, HttpErrorCode_FileError);
+		lua_setfield(L, -2, "error");
+
+		lua_pushstring(L, errorMessage.c_str());
+		lua_setfield(L, -2, "errorMessage");
+
+		RString error = "Lua error in HTTP response handler: ";
+		LuaHelpers::RunScriptOnStack(L, error, 1, 0, true);
+	}
+
 	static void handleHttpResponse(Lua *L, const ix::HttpResponsePtr& response, int onResponseRef)
 	{
 		lua_rawgeti(L, LUA_REGISTRYINDEX, onResponseRef);
@@ -493,6 +647,16 @@ private:
 		RString error = "Lua error in HTTP response handler: ";
 		LuaHelpers::RunScriptOnStack(L, error, 1, 0, true);
 	}
+
+	static void handleProgress(Lua *L, int current, int total, int onProgressRef)
+	{
+		lua_rawgeti(L, LUA_REGISTRYINDEX, onProgressRef);
+		lua_pushinteger(L, current);
+		lua_pushinteger(L, total);
+
+		RString error = "Lua error in HTTP progress handler: ";
+		LuaHelpers::RunScriptOnStack(L, error, 2, 0, true);
+	}
 };
 
 LUA_REGISTER_CLASS(NetworkManager)

src/NetworkManager.h

@@ -4,8 +4,10 @@
 #include "Preference.h"
 #include "StdString.h"
 
+#include <atomic>
 #include <functional>
 #include <memory>
+#include <mutex>
 #include <string>
 #include <unordered_map>
 
@@ -21,6 +23,7 @@ enum HttpErrorCode
 {
 	HttpErrorCode_Blocked,
 	HttpErrorCode_UnknownError,
+	HttpErrorCode_FileError,
 
 	// from IXWebSocket
 	HttpErrorCode_CannotConnect,
@@ -44,7 +47,7 @@ enum HttpErrorCode
 };
 const RString& HttpErrorCodeToString(HttpErrorCode dc);
 HttpErrorCode StringToHttpErrorCode(const RString& sDC);
-LuaDeclareType( HttpErrorCode );
+LuaDeclareType(HttpErrorCode);
 
 struct HttpRequestArgs
 {
@@ -55,7 +58,10 @@ struct HttpRequestArgs
 	std::unordered_map<std::string, std::string> headers;
 	int connectTimeout = -1;
 	int transferTimeout = -1;
-	std::function<void(const ix::HttpResponsePtr& response)> onResponse = [](const ix::HttpResponsePtr& response) {};
+	std::string downloadFile;
+	std::function<bool(int current, int total)> onProgress;
+	std::function<void(const ix::HttpResponsePtr& response)> onResponse;
+	std::function<void(const std::string& errorMessage)> onFileError;
 };
 
 class HttpRequestFuture
@@ -63,7 +69,7 @@ class HttpRequestFuture
 public:
 	HttpRequestFuture(ix::HttpRequestArgsPtr& args) : args(args) {};
 
-	static int collect(lua_State *L);
+	static int Collect(lua_State *L);
 	static int Cancel(lua_State *L);
 
 private:
@@ -87,7 +93,11 @@ public:
 	void PushSelf(lua_State *L);
 
 private:
+	std::string GetUserAgent();
+	void ClearDownloads();
+
 	ix::HttpClient httpClient;
+	ix::HttpClient downloadClient;
 
 	static Preference<bool> httpEnabled;
 	static Preference<RString> httpAllowHosts;

src/arch/ArchHooks/ArchHooks_MacOSX.mm

@@ -354,6 +354,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable )
 	FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/CDTitles", dir), "/CDTitles" );
 	FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Characters", dir), "/Characters" );
 	FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Courses", dir), "/Courses" );
+	FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Downloads", dir), "/Downloads" );
 	FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/NoteSkins", dir), "/NoteSkins" );
 	FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Packages", dir), "/Packages" );
 	FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Songs", dir), "/Songs" );

src/arch/ArchHooks/ArchHooks_Unix.cpp

@@ -403,6 +403,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable )
 	FILEMAN->Mount( "dir", sUserDataPath + "/CDTitles", "/CDTitles" );
 	FILEMAN->Mount( "dir", sUserDataPath + "/Characters", "/Characters" );
 	FILEMAN->Mount( "dir", sUserDataPath + "/Courses", "/Courses" );
+	FILEMAN->Mount( "dir", sUserDataPath + "/Downloads", "/Downloads" );
 	FILEMAN->Mount( "dir", sUserDataPath + "/Logs", "/Logs" );
 	FILEMAN->Mount( "dir", sUserDataPath + "/NoteSkins", "/NoteSkins" );
 	FILEMAN->Mount( "dir", sUserDataPath + "/Packages", "/Packages" );

src/arch/ArchHooks/ArchHooks_Win32Static.cpp

@@ -82,6 +82,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable )
 	FILEMAN->Mount( "dir", sAppDataDir + "/CDTitles", "/CDTitles" );
 	FILEMAN->Mount( "dir", sAppDataDir + "/Characters", "/Characters" );
 	FILEMAN->Mount( "dir", sAppDataDir + "/Courses", "/Courses" );
+	FILEMAN->Mount( "dir", sAppDataDir + "/Downloads", "/Downloads" );
 	FILEMAN->Mount( "dir", sAppDataDir + "/Logs", "/Logs" );
 	FILEMAN->Mount( "dir", sAppDataDir + "/NoteSkins", "/NoteSkins" );
 	FILEMAN->Mount( "dir", sAppDataDir + "/Packages", "/Packages" );