From ca86f80743298f33223ea95cb610df1eafa733a8 Mon Sep 17 00:00:00 2001 From: Martin Natano Date: Sun, 23 Jan 2022 18:21:50 +0100 Subject: [PATCH] Add support for onProgress and downloadFile to NETWORK:HttpRequest() --- Docs/Luadoc/Lua.xml | 31 +-- Docs/Luadoc/LuaDocumentation.xml | 21 +- extern/CMakeProject-ixwebsocket.cmake | 1 - src/NetworkManager.cpp | 200 +++++++++++++++++-- src/NetworkManager.h | 16 +- src/arch/ArchHooks/ArchHooks_MacOSX.mm | 1 + src/arch/ArchHooks/ArchHooks_Unix.cpp | 1 + src/arch/ArchHooks/ArchHooks_Win32Static.cpp | 1 + 8 files changed, 228 insertions(+), 44 deletions(-) diff --git a/Docs/Luadoc/Lua.xml b/Docs/Luadoc/Lua.xml index 80bb6b880a..c75ce374d7 100644 --- a/Docs/Luadoc/Lua.xml +++ b/Docs/Luadoc/Lua.xml @@ -2486,21 +2486,22 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/Docs/Luadoc/LuaDocumentation.xml b/Docs/Luadoc/LuaDocumentation.xml index 61391fdbfd..b07c150616 100644 --- a/Docs/Luadoc/LuaDocumentation.xml +++ b/Docs/Luadoc/LuaDocumentation.xml @@ -3562,16 +3562,20 @@ end 

 NETWORK:HttpRequest{
 	url="https://api.example.com",
-	method="GET",                           -- default: "GET"
-	body="",                                -- default: ""
-	multipartBoundary="",                   -- default: ""
-	headers={                               -- default: {}
+	method="GET",                           	-- default: "GET"
+	body="",                                	-- default: ""
+	multipartBoundary="",                   	-- default: ""
+	headers={                               	-- default: {}
 		["Accept-Language"]="en-US",
 		["Cookie"]="sessionId=42",
 	},
-	connectTimeout=3,                       -- default: 60
-	transferTimeout=10,                     -- default: 1800
-	onResponse=function(response)           -- default: no callback
+	connectTimeout=3,                       	-- default: 60
+	transferTimeout=10,                     	-- default: 1800
+	downloadFile="",				-- default: no download file
+	onProgress=function(currentBytes, totalBytes)	-- default: no callback
+		...
+	end,
+	onResponse=function(response)           	-- default: no callback
 		...
 	end,
 }
@@ -3598,6 +3602,9 @@ NETWORK:HttpRequest{
 	errorMessage="access to https://api.example.com is not allowed",
 }
+ If the downloadFile parameter is set, the response body is not accumulated in memory, but written to /Downloads/${downloadFile}. + The file is available in the onResponse callback where it can be unzipped/copied to another location using FILEMAN:Unzip()/FILEMAN:Copy() respectively. + The file is deleted once the callback returns. Returns the URL encoded representation of value. diff --git a/extern/CMakeProject-ixwebsocket.cmake b/extern/CMakeProject-ixwebsocket.cmake index 0d9a995873..da56c6f874 100644 --- a/extern/CMakeProject-ixwebsocket.cmake +++ b/extern/CMakeProject-ixwebsocket.cmake @@ -149,5 +149,4 @@ set(IXWEBSOCKET_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/IXWebSocket-11.3.2") target_include_directories(ixwebsocket PUBLIC "zlib" $ - $ ) diff --git a/src/NetworkManager.cpp b/src/NetworkManager.cpp index 390241ecae..b5ceedc174 100644 --- a/src/NetworkManager.cpp +++ b/src/NetworkManager.cpp @@ -1,16 +1,22 @@ #include "global.h" #include "NetworkManager.h" #include "LuaManager.h" +#include "ProductInfo.h" +#include "RageFile.h" +#include "RageFileManager.h" #include "RageLog.h" #include "RageUtil.h" #include "StdString.h" +#include "ver.h" #include #include #include #include +#include #include +#include #include #include @@ -23,6 +29,7 @@ Preference NetworkManager::httpAllowHosts("HttpAllowHosts", "", nullptr static const char *HttpErrorCodeNames[] = { "Blocked", "UnknownError", + "FileError", "CannotConnect", "Timeout", "Gzip", @@ -43,7 +50,7 @@ XToString(HttpErrorCode); StringToX(HttpErrorCode); LuaXType(HttpErrorCode); -NetworkManager::NetworkManager() : httpClient(true) +NetworkManager::NetworkManager() : httpClient(true), downloadClient(true) { ix::initNetSystem(); @@ -54,6 +61,8 @@ NetworkManager::NetworkManager() : httpClient(true) lua_setglobal(L, "NETWORK"); LUA->Release(L); } + + this->ClearDownloads(); } NetworkManager::~NetworkManager() @@ -97,24 +106,75 @@ bool NetworkManager::IsUrlAllowed(const std::string& url) HttpRequestFuturePtr NetworkManager::HttpRequest(const HttpRequestArgs& args) { - ix::HttpRequestArgsPtr req = this->httpClient.createRequest(args.url, args.method); + auto &client = args.downloadFile.empty() ? this->httpClient : this->downloadClient; + auto downloadFile = std::make_shared(); + std::string downloadFilename; + + ix::HttpRequestArgsPtr req = client.createRequest(args.url, args.method); req->body = args.body; req->multipartBoundary = args.multipartBoundary; + req->extraHeaders["User-Agent"] = this->GetUserAgent(); for (const auto& entry : args.headers) { req->extraHeaders[entry.first] = entry.second; } - if (args.connectTimeout > 0) { + if (!args.downloadFile.empty()) + { + downloadFilename = std::string("/Downloads/") + args.downloadFile; + if (!downloadFile->Open(downloadFilename, RageFile::WRITE | RageFile::STREAMED)) + { + req->cancel = true; + } + + req->onChunkCallback = [args, downloadFile, req](const std::string& data) { + if (downloadFile->Write(data.c_str(), data.size()) < 0) + { + req->cancel = true; + } + }; + + // Don't timeout downloads by default. This value might be + // overwritten below if a value is specified in the request. + req->transferTimeout = INT_MAX; + + } + + if (args.connectTimeout > 0) req->connectTimeout = args.connectTimeout; - } - if (args.transferTimeout > 0) { + if (args.transferTimeout > 0) req->transferTimeout = args.transferTimeout; - } - this->httpClient.performRequest(req, args.onResponse); + if (args.onProgress) + req->onProgressCallback = args.onProgress; + + client.performRequest(req, [args, downloadFile, downloadFilename](const ix::HttpResponsePtr& response) { + if (downloadFile->IsOpen()) + { + RString error = downloadFile->GetError(); + downloadFile->Close(); + + if (!error.empty()) + { + std::string errorMessage = "could not write to " + downloadFile->GetPath() + ": " + error; + if (args.onFileError) + args.onFileError(errorMessage); + + FILEMAN->Remove(downloadFilename); + return; + } + } + + if (args.onResponse) + args.onResponse(response); + + if (!args.downloadFile.empty()) + { + FILEMAN->Remove(downloadFilename); + } + }); return std::make_shared(req); } @@ -129,7 +189,28 @@ std::string NetworkManager::EncodeQueryParameters(const std::unordered_maphttpClient.serializeHttpParameters(query); } -int HttpRequestFuture::collect(lua_State *L) +std::string NetworkManager::GetUserAgent() +{ + std::stringstream ss; + ss << PRODUCT_FAMILY << "/" << product_version; + return ss.str(); +} + +void NetworkManager::ClearDownloads() +{ + vector files; + FILEMAN->GetDirListing("/Downloads/*", files, false, true); + + for (const auto& file : files) + { + if (FILEMAN->IsADirectory(file)) + FILEMAN->DeleteRecursive(file + "/"); + else + FILEMAN->Remove(file); + } +} + +int HttpRequestFuture::Collect(lua_State *L) { void *udata = luaL_checkudata(L, 1, "HttpRequestFuture"); auto futptr = static_cast(udata); @@ -152,7 +233,7 @@ int HttpRequestFuture::Cancel(lua_State *L) static void registerHttpRequestMetatable(lua_State *L) { const luaL_Reg HttpRequest_meta[] = { - {"__gc", HttpRequestFuture::collect}, + {"__gc", HttpRequestFuture::Collect}, {"Cancel", HttpRequestFuture::Cancel}, {NULL, NULL}, }; @@ -183,6 +264,8 @@ public: luaL_checktype(L, 1, LUA_TTABLE); HttpRequestArgs args; + int onProgressRef = LUA_NOREF; + int onResponseRef = LUA_NOREF; lua_getfield(L, 1, "url"); if (lua_isnil(L, -1)) @@ -311,9 +394,44 @@ public: } lua_pop(L, 1); - int onResponseRef = LUA_NOREF; + lua_getfield(L, 1, "downloadFile"); + if (!lua_isnil(L, -1)) + { + if (lua_isstring(L, -1)) + { + args.downloadFile = lua_tostring(L, -1); + } + else + { + luaL_error(L, "downloadFile must be a string"); + } + } + lua_pop(L, 1); + + lua_getfield(L, 1, "onProgress"); + if (!lua_isnil(L, -1)) + { + if (lua_isfunction(L, -1)) + { + lua_pushvalue(L, -1); + onProgressRef = luaL_ref(L, LUA_REGISTRYINDEX); + + args.onProgress = [onProgressRef](int current, int total) + { + Lua *L = LUA->Get(); + handleProgress(L, current, total, onProgressRef); + LUA->Release(L); + + return true; + }; + } + else + { + luaL_error(L, "onProgress must be a function"); + } + } + lua_pop(L, 1); - // do this as the last check, so we don't leak lua references on error lua_getfield(L, 1, "onResponse"); if (!lua_isnil(L, -1)) { @@ -321,21 +439,39 @@ public: { lua_pushvalue(L, -1); onResponseRef = luaL_ref(L, LUA_REGISTRYINDEX); - - args.onResponse = [onResponseRef](const ix::HttpResponsePtr& response) - { - Lua *L = LUA->Get(); - handleHttpResponse(L, response, onResponseRef); - LUA->Release(L); - }; } else { + luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef); luaL_error(L, "onResponse must be a function"); } } lua_pop(L, 1); + args.onFileError = [onProgressRef, onResponseRef](const std::string& errorMessage) + { + Lua *L = LUA->Get(); + + luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef); + + if (onResponseRef != LUA_NOREF) + handleFileError(L, errorMessage, onResponseRef); + + LUA->Release(L); + }; + + args.onResponse = [onProgressRef, onResponseRef](const ix::HttpResponsePtr& response) + { + Lua *L = LUA->Get(); + + luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef); + + if (onResponseRef != LUA_NOREF) + handleHttpResponse(L, response, onResponseRef); + + LUA->Release(L); + }; + if (p->IsUrlAllowed(args.url)) { auto fut = p->HttpRequest(args); @@ -349,6 +485,7 @@ public: else { LOG->Warn("blocked access to %s", args.url.c_str()); + luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef); if (onResponseRef != LUA_NOREF) { handleUrlForbidden(L, args.url, onResponseRef); @@ -431,6 +568,23 @@ private: LuaHelpers::RunScriptOnStack(L, error, 1, 0, true); } + static void handleFileError(Lua *L, const std::string& errorMessage, int onResponseRef) + { + lua_rawgeti(L, LUA_REGISTRYINDEX, onResponseRef); + luaL_unref(L, LUA_REGISTRYINDEX, onResponseRef); + + lua_newtable(L); + + LuaHelpers::Push(L, HttpErrorCode_FileError); + lua_setfield(L, -2, "error"); + + lua_pushstring(L, errorMessage.c_str()); + lua_setfield(L, -2, "errorMessage"); + + RString error = "Lua error in HTTP response handler: "; + LuaHelpers::RunScriptOnStack(L, error, 1, 0, true); + } + static void handleHttpResponse(Lua *L, const ix::HttpResponsePtr& response, int onResponseRef) { lua_rawgeti(L, LUA_REGISTRYINDEX, onResponseRef); @@ -493,6 +647,16 @@ private: RString error = "Lua error in HTTP response handler: "; LuaHelpers::RunScriptOnStack(L, error, 1, 0, true); } + + static void handleProgress(Lua *L, int current, int total, int onProgressRef) + { + lua_rawgeti(L, LUA_REGISTRYINDEX, onProgressRef); + lua_pushinteger(L, current); + lua_pushinteger(L, total); + + RString error = "Lua error in HTTP progress handler: "; + LuaHelpers::RunScriptOnStack(L, error, 2, 0, true); + } }; LUA_REGISTER_CLASS(NetworkManager) diff --git a/src/NetworkManager.h b/src/NetworkManager.h index 2bf2e57887..c5b1f0b842 100644 --- a/src/NetworkManager.h +++ b/src/NetworkManager.h @@ -4,8 +4,10 @@ #include "Preference.h" #include "StdString.h" +#include #include #include +#include #include #include @@ -21,6 +23,7 @@ enum HttpErrorCode { HttpErrorCode_Blocked, HttpErrorCode_UnknownError, + HttpErrorCode_FileError, // from IXWebSocket HttpErrorCode_CannotConnect, @@ -44,7 +47,7 @@ enum HttpErrorCode }; const RString& HttpErrorCodeToString(HttpErrorCode dc); HttpErrorCode StringToHttpErrorCode(const RString& sDC); -LuaDeclareType( HttpErrorCode ); +LuaDeclareType(HttpErrorCode); struct HttpRequestArgs { @@ -55,7 +58,10 @@ struct HttpRequestArgs std::unordered_map headers; int connectTimeout = -1; int transferTimeout = -1; - std::function onResponse = [](const ix::HttpResponsePtr& response) {}; + std::string downloadFile; + std::function onProgress; + std::function onResponse; + std::function onFileError; }; class HttpRequestFuture @@ -63,7 +69,7 @@ class HttpRequestFuture public: HttpRequestFuture(ix::HttpRequestArgsPtr& args) : args(args) {}; - static int collect(lua_State *L); + static int Collect(lua_State *L); static int Cancel(lua_State *L); private: @@ -87,7 +93,11 @@ public: void PushSelf(lua_State *L); private: + std::string GetUserAgent(); + void ClearDownloads(); + ix::HttpClient httpClient; + ix::HttpClient downloadClient; static Preference httpEnabled; static Preference httpAllowHosts; diff --git a/src/arch/ArchHooks/ArchHooks_MacOSX.mm b/src/arch/ArchHooks/ArchHooks_MacOSX.mm index e748eee1ed..060ff5b5b2 100644 --- a/src/arch/ArchHooks/ArchHooks_MacOSX.mm +++ b/src/arch/ArchHooks/ArchHooks_MacOSX.mm @@ -354,6 +354,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable ) FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/CDTitles", dir), "/CDTitles" ); FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Characters", dir), "/Characters" ); FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Courses", dir), "/Courses" ); + FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Downloads", dir), "/Downloads" ); FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/NoteSkins", dir), "/NoteSkins" ); FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Packages", dir), "/Packages" ); FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Songs", dir), "/Songs" ); diff --git a/src/arch/ArchHooks/ArchHooks_Unix.cpp b/src/arch/ArchHooks/ArchHooks_Unix.cpp index c48bac13fc..24dbaf455a 100644 --- a/src/arch/ArchHooks/ArchHooks_Unix.cpp +++ b/src/arch/ArchHooks/ArchHooks_Unix.cpp @@ -403,6 +403,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable ) FILEMAN->Mount( "dir", sUserDataPath + "/CDTitles", "/CDTitles" ); FILEMAN->Mount( "dir", sUserDataPath + "/Characters", "/Characters" ); FILEMAN->Mount( "dir", sUserDataPath + "/Courses", "/Courses" ); + FILEMAN->Mount( "dir", sUserDataPath + "/Downloads", "/Downloads" ); FILEMAN->Mount( "dir", sUserDataPath + "/Logs", "/Logs" ); FILEMAN->Mount( "dir", sUserDataPath + "/NoteSkins", "/NoteSkins" ); FILEMAN->Mount( "dir", sUserDataPath + "/Packages", "/Packages" ); diff --git a/src/arch/ArchHooks/ArchHooks_Win32Static.cpp b/src/arch/ArchHooks/ArchHooks_Win32Static.cpp index 8676702add..c07ac0085f 100644 --- a/src/arch/ArchHooks/ArchHooks_Win32Static.cpp +++ b/src/arch/ArchHooks/ArchHooks_Win32Static.cpp @@ -82,6 +82,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable ) FILEMAN->Mount( "dir", sAppDataDir + "/CDTitles", "/CDTitles" ); FILEMAN->Mount( "dir", sAppDataDir + "/Characters", "/Characters" ); FILEMAN->Mount( "dir", sAppDataDir + "/Courses", "/Courses" ); + FILEMAN->Mount( "dir", sAppDataDir + "/Downloads", "/Downloads" ); FILEMAN->Mount( "dir", sAppDataDir + "/Logs", "/Logs" ); FILEMAN->Mount( "dir", sAppDataDir + "/NoteSkins", "/NoteSkins" ); FILEMAN->Mount( "dir", sAppDataDir + "/Packages", "/Packages" );