actions/itgmania212121
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for onProgress and downloadFile to NETWORK:HttpRequest()

Browse Source
This commit is contained in:
Martin Natano
2022-03-13 11:31:49 -07:00
committed by teejusb
parent 394bc93915
commit ca86f80743
8 changed files with 228 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Docs/Luadoc/Lua.xml
+16 -15
View File
@@ -2486,21 +2486,22 @@
<Enum name='HttpErrorCode'>
<EnumValue name='&apos;HttpErrorCode_Blocked&apos;' value='0'/>
<EnumValue name='&apos;HttpErrorCode_UnknownError&apos;' value='1'/>
<EnumValue name='&apos;HttpErrorCode_CannotConnect&apos;' value='2'/>
<EnumValue name='&apos;HttpErrorCode_Timeout&apos;' value='3'/>
<EnumValue name='&apos;HttpErrorCode_Gzip&apos;' value='4'/>
<EnumValue name='&apos;HttpErrorCode_UrlMalformed&apos;' value='5'/>
<EnumValue name='&apos;HttpErrorCode_CannotCreateSocket&apos;' value='6'/>
<EnumValue name='&apos;HttpErrorCode_SendError&apos;' value='7'/>
<EnumValue name='&apos;HttpErrorCode_ReadError&apos;' value='8'/>
<EnumValue name='&apos;HttpErrorCode_CannotReadStatusLine&apos;' value='9'/>
<EnumValue name='&apos;HttpErrorCode_MissingStatus&apos;' value='10'/>
<EnumValue name='&apos;HttpErrorCode_HeaderParsingError&apos;' value='11'/>
<EnumValue name='&apos;HttpErrorCode_MissingLocation&apos;' value='12'/>
<EnumValue name='&apos;HttpErrorCode_TooManyRedirects&apos;' value='13'/>
<EnumValue name='&apos;HttpErrorCode_ChunkReadError&apos;' value='14'/>
<EnumValue name='&apos;HttpErrorCode_CannotReadBody&apos;' value='15'/>
<EnumValue name='&apos;HttpErrorCode_Cancelled&apos;' value='16'/>
<EnumValue name='&apos;HttpErrorCode_FileError&apos;' value='2'/>
<EnumValue name='&apos;HttpErrorCode_CannotConnect&apos;' value='3'/>
<EnumValue name='&apos;HttpErrorCode_Timeout&apos;' value='4'/>
<EnumValue name='&apos;HttpErrorCode_Gzip&apos;' value='5'/>
<EnumValue name='&apos;HttpErrorCode_UrlMalformed&apos;' value='6'/>
<EnumValue name='&apos;HttpErrorCode_CannotCreateSocket&apos;' value='7'/>
<EnumValue name='&apos;HttpErrorCode_SendError&apos;' value='8'/>
<EnumValue name='&apos;HttpErrorCode_ReadError&apos;' value='9'/>
<EnumValue name='&apos;HttpErrorCode_CannotReadStatusLine&apos;' value='10'/>
<EnumValue name='&apos;HttpErrorCode_MissingStatus&apos;' value='11'/>
<EnumValue name='&apos;HttpErrorCode_HeaderParsingError&apos;' value='12'/>
<EnumValue name='&apos;HttpErrorCode_MissingLocation&apos;' value='13'/>
<EnumValue name='&apos;HttpErrorCode_TooManyRedirects&apos;' value='14'/>
<EnumValue name='&apos;HttpErrorCode_ChunkReadError&apos;' value='15'/>
<EnumValue name='&apos;HttpErrorCode_CannotReadBody&apos;' value='16'/>
<EnumValue name='&apos;HttpErrorCode_Cancelled&apos;' value='17'/>
</Enum>
<Enum name='ImageCacheMode'>
<EnumValue name='&apos;ImageCacheMode_Off&apos;' value='0'/>
Docs/Luadoc/LuaDocumentation.xml
+14 -7
View File
@@ -3562,16 +3562,20 @@ end
<pre><code>
NETWORK:HttpRequest{
url="https://api.example.com",
method="GET", -- default: "GET"
body="", -- default: ""
multipartBoundary="", -- default: ""
headers={ -- default: {}
method="GET", -- default: "GET"
body="", -- default: ""
multipartBoundary="", -- default: ""
headers={ -- default: {}
["Accept-Language"]="en-US",
["Cookie"]="sessionId=42",
},
connectTimeout=3, -- default: 60
transferTimeout=10, -- default: 1800
onResponse=function(response) -- default: no callback
connectTimeout=3, -- default: 60
transferTimeout=10, -- default: 1800
downloadFile="", -- default: no download file
onProgress=function(currentBytes, totalBytes) -- default: no callback
...
end,
onResponse=function(response) -- default: no callback
...
end,
}
@@ -3598,6 +3602,9 @@ NETWORK:HttpRequest{
errorMessage="access to https://api.example.com is not allowed",
}
</code></pre>
If the <code>downloadFile</code> parameter is set, the response body is not accumulated in memory, but written to <code>/Downloads/${downloadFile}</code>.
The file is available in the <code>onResponse</code> callback where it can be unzipped/copied to another location using <code>FILEMAN:Unzip()</code>/<code>FILEMAN:Copy()</code> respectively.
The file is deleted once the callback returns.
</Function>
<Function name='UrlEncode' return='string' arguments='string value'>
Returns the URL encoded representation of <code>value</code>.
extern/CMakeProject-ixwebsocket.cmake
Vendored
-1
View File
@@ -149,5 +149,4 @@ set(IXWEBSOCKET_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/IXWebSocket-11.3.2")
target_include_directories(ixwebsocket PUBLIC
"zlib"
$<BUILD_INTERFACE:${IXWEBSOCKET_INCLUDE_DIRS}/>
$<INSTALL_INTERFACE:include/ixwebsocket>
)
src/NetworkManager.cpp
+182 -18
View File
@@ -1,16 +1,22 @@
#include "global.h"
#include "NetworkManager.h"
#include "LuaManager.h"
#include "ProductInfo.h"
#include "RageFile.h"
#include "RageFileManager.h"
#include "RageLog.h"
#include "RageUtil.h"
#include "StdString.h"
#include "ver.h"
#include <ixwebsocket/IXHttpClient.h>
#include <ixwebsocket/IXNetSystem.h>
#include <ixwebsocket/IXUrlParser.h>
#include <algorithm>
#include <climits>
#include <memory>
#include <sstream>
#include <unordered_map>
#include <vector>
@@ -23,6 +29,7 @@ Preference<RString> NetworkManager::httpAllowHosts("HttpAllowHosts", "", nullptr
static const char *HttpErrorCodeNames[] = {
"Blocked",
"UnknownError",
"FileError",
"CannotConnect",
"Timeout",
"Gzip",
@@ -43,7 +50,7 @@ XToString(HttpErrorCode);
StringToX(HttpErrorCode);
LuaXType(HttpErrorCode);
NetworkManager::NetworkManager() : httpClient(true)
NetworkManager::NetworkManager() : httpClient(true), downloadClient(true)
{
ix::initNetSystem();
@@ -54,6 +61,8 @@ NetworkManager::NetworkManager() : httpClient(true)
lua_setglobal(L, "NETWORK");
LUA->Release(L);
}
this->ClearDownloads();
}
NetworkManager::~NetworkManager()
@@ -97,24 +106,75 @@ bool NetworkManager::IsUrlAllowed(const std::string& url)
HttpRequestFuturePtr NetworkManager::HttpRequest(const HttpRequestArgs& args)
{
ix::HttpRequestArgsPtr req = this->httpClient.createRequest(args.url, args.method);
auto &client = args.downloadFile.empty() ? this->httpClient : this->downloadClient;
auto downloadFile = std::make_shared<RageFile>();
std::string downloadFilename;
ix::HttpRequestArgsPtr req = client.createRequest(args.url, args.method);
req->body = args.body;
req->multipartBoundary = args.multipartBoundary;
req->extraHeaders["User-Agent"] = this->GetUserAgent();
for (const auto& entry : args.headers)
{
req->extraHeaders[entry.first] = entry.second;
}
if (args.connectTimeout > 0) {
if (!args.downloadFile.empty())
{
downloadFilename = std::string("/Downloads/") + args.downloadFile;
if (!downloadFile->Open(downloadFilename, RageFile::WRITE | RageFile::STREAMED))
{
req->cancel = true;
}
req->onChunkCallback = [args, downloadFile, req](const std::string& data) {
if (downloadFile->Write(data.c_str(), data.size()) < 0)
{
req->cancel = true;
}
};
// Don't timeout downloads by default. This value might be
// overwritten below if a value is specified in the request.
req->transferTimeout = INT_MAX;
}
if (args.connectTimeout > 0)
req->connectTimeout = args.connectTimeout;
}
if (args.transferTimeout > 0) {
if (args.transferTimeout > 0)
req->transferTimeout = args.transferTimeout;
}
this->httpClient.performRequest(req, args.onResponse);
if (args.onProgress)
req->onProgressCallback = args.onProgress;
client.performRequest(req, [args, downloadFile, downloadFilename](const ix::HttpResponsePtr& response) {
if (downloadFile->IsOpen())
{
RString error = downloadFile->GetError();
downloadFile->Close();
if (!error.empty())
{
std::string errorMessage = "could not write to " + downloadFile->GetPath() + ": " + error;
if (args.onFileError)
args.onFileError(errorMessage);
FILEMAN->Remove(downloadFilename);
return;
}
}
if (args.onResponse)
args.onResponse(response);
if (!args.downloadFile.empty())
{
FILEMAN->Remove(downloadFilename);
}
});
return std::make_shared<HttpRequestFuture>(req);
}
@@ -129,7 +189,28 @@ std::string NetworkManager::EncodeQueryParameters(const std::unordered_map<std::
return this->httpClient.serializeHttpParameters(query);
}
int HttpRequestFuture::collect(lua_State *L)
std::string NetworkManager::GetUserAgent()
{
std::stringstream ss;
ss << PRODUCT_FAMILY << "/" << product_version;
return ss.str();
}
void NetworkManager::ClearDownloads()
{
vector<RString> files;
FILEMAN->GetDirListing("/Downloads/*", files, false, true);
for (const auto& file : files)
{
if (FILEMAN->IsADirectory(file))
FILEMAN->DeleteRecursive(file + "/");
else
FILEMAN->Remove(file);
}
}
int HttpRequestFuture::Collect(lua_State *L)
{
void *udata = luaL_checkudata(L, 1, "HttpRequestFuture");
auto futptr = static_cast<HttpRequestFuturePtr*>(udata);
@@ -152,7 +233,7 @@ int HttpRequestFuture::Cancel(lua_State *L)
static void registerHttpRequestMetatable(lua_State *L)
{
const luaL_Reg HttpRequest_meta[] = {
{"__gc", HttpRequestFuture::collect},
{"__gc", HttpRequestFuture::Collect},
{"Cancel", HttpRequestFuture::Cancel},
{NULL, NULL},
};
@@ -183,6 +264,8 @@ public:
luaL_checktype(L, 1, LUA_TTABLE);
HttpRequestArgs args;
int onProgressRef = LUA_NOREF;
int onResponseRef = LUA_NOREF;
lua_getfield(L, 1, "url");
if (lua_isnil(L, -1))
@@ -311,9 +394,44 @@ public:
}
lua_pop(L, 1);
int onResponseRef = LUA_NOREF;
lua_getfield(L, 1, "downloadFile");
if (!lua_isnil(L, -1))
{
if (lua_isstring(L, -1))
{
args.downloadFile = lua_tostring(L, -1);
}
else
{
luaL_error(L, "downloadFile must be a string");
}
}
lua_pop(L, 1);
lua_getfield(L, 1, "onProgress");
if (!lua_isnil(L, -1))
{
if (lua_isfunction(L, -1))
{
lua_pushvalue(L, -1);
onProgressRef = luaL_ref(L, LUA_REGISTRYINDEX);
args.onProgress = [onProgressRef](int current, int total)
{
Lua *L = LUA->Get();
handleProgress(L, current, total, onProgressRef);
LUA->Release(L);
return true;
};
}
else
{
luaL_error(L, "onProgress must be a function");
}
}
lua_pop(L, 1);
// do this as the last check, so we don't leak lua references on error
lua_getfield(L, 1, "onResponse");
if (!lua_isnil(L, -1))
{
@@ -321,21 +439,39 @@ public:
{
lua_pushvalue(L, -1);
onResponseRef = luaL_ref(L, LUA_REGISTRYINDEX);
args.onResponse = [onResponseRef](const ix::HttpResponsePtr& response)
{
Lua *L = LUA->Get();
handleHttpResponse(L, response, onResponseRef);
LUA->Release(L);
};
}
else
{
luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
luaL_error(L, "onResponse must be a function");
}
}
lua_pop(L, 1);
args.onFileError = [onProgressRef, onResponseRef](const std::string& errorMessage)
{
Lua *L = LUA->Get();
luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
if (onResponseRef != LUA_NOREF)
handleFileError(L, errorMessage, onResponseRef);
LUA->Release(L);
};
args.onResponse = [onProgressRef, onResponseRef](const ix::HttpResponsePtr& response)
{
Lua *L = LUA->Get();
luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
if (onResponseRef != LUA_NOREF)
handleHttpResponse(L, response, onResponseRef);
LUA->Release(L);
};
if (p->IsUrlAllowed(args.url))
{
auto fut = p->HttpRequest(args);
@@ -349,6 +485,7 @@ public:
else
{
LOG->Warn("blocked access to %s", args.url.c_str());
luaL_unref(L, LUA_REGISTRYINDEX, onProgressRef);
if (onResponseRef != LUA_NOREF)
{
handleUrlForbidden(L, args.url, onResponseRef);
@@ -431,6 +568,23 @@ private:
LuaHelpers::RunScriptOnStack(L, error, 1, 0, true);
}
static void handleFileError(Lua *L, const std::string& errorMessage, int onResponseRef)
{
lua_rawgeti(L, LUA_REGISTRYINDEX, onResponseRef);
luaL_unref(L, LUA_REGISTRYINDEX, onResponseRef);
lua_newtable(L);
LuaHelpers::Push(L, HttpErrorCode_FileError);
lua_setfield(L, -2, "error");
lua_pushstring(L, errorMessage.c_str());
lua_setfield(L, -2, "errorMessage");
RString error = "Lua error in HTTP response handler: ";
LuaHelpers::RunScriptOnStack(L, error, 1, 0, true);
}
static void handleHttpResponse(Lua *L, const ix::HttpResponsePtr& response, int onResponseRef)
{
lua_rawgeti(L, LUA_REGISTRYINDEX, onResponseRef);
@@ -493,6 +647,16 @@ private:
RString error = "Lua error in HTTP response handler: ";
LuaHelpers::RunScriptOnStack(L, error, 1, 0, true);
}
static void handleProgress(Lua *L, int current, int total, int onProgressRef)
{
lua_rawgeti(L, LUA_REGISTRYINDEX, onProgressRef);
lua_pushinteger(L, current);
lua_pushinteger(L, total);
RString error = "Lua error in HTTP progress handler: ";
LuaHelpers::RunScriptOnStack(L, error, 2, 0, true);
}
};
LUA_REGISTER_CLASS(NetworkManager)
src/NetworkManager.h
+13 -3
View File
@@ -4,8 +4,10 @@
#include "Preference.h"
#include "StdString.h"
#include <atomic>
#include <functional>
#include <memory>
#include <mutex>
#include <string>
#include <unordered_map>
@@ -21,6 +23,7 @@ enum HttpErrorCode
{
HttpErrorCode_Blocked,
HttpErrorCode_UnknownError,
HttpErrorCode_FileError,
// from IXWebSocket
HttpErrorCode_CannotConnect,
@@ -44,7 +47,7 @@ enum HttpErrorCode
};
const RString& HttpErrorCodeToString(HttpErrorCode dc);
HttpErrorCode StringToHttpErrorCode(const RString& sDC);
LuaDeclareType( HttpErrorCode );
LuaDeclareType(HttpErrorCode);
struct HttpRequestArgs
{
@@ -55,7 +58,10 @@ struct HttpRequestArgs
std::unordered_map<std::string, std::string> headers;
int connectTimeout = -1;
int transferTimeout = -1;
std::function<void(const ix::HttpResponsePtr& response)> onResponse = [](const ix::HttpResponsePtr& response) {};
std::string downloadFile;
std::function<bool(int current, int total)> onProgress;
std::function<void(const ix::HttpResponsePtr& response)> onResponse;
std::function<void(const std::string& errorMessage)> onFileError;
};
class HttpRequestFuture
@@ -63,7 +69,7 @@ class HttpRequestFuture
public:
HttpRequestFuture(ix::HttpRequestArgsPtr& args) : args(args) {};
static int collect(lua_State *L);
static int Collect(lua_State *L);
static int Cancel(lua_State *L);
private:
@@ -87,7 +93,11 @@ public:
void PushSelf(lua_State *L);
private:
std::string GetUserAgent();
void ClearDownloads();
ix::HttpClient httpClient;
ix::HttpClient downloadClient;
static Preference<bool> httpEnabled;
static Preference<RString> httpAllowHosts;
src/arch/ArchHooks/ArchHooks_MacOSX.mm
+1
View File
@@ -354,6 +354,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable )
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/CDTitles", dir), "/CDTitles" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Characters", dir), "/Characters" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Courses", dir), "/Courses" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Downloads", dir), "/Downloads" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/NoteSkins", dir), "/NoteSkins" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Packages", dir), "/Packages" );
FILEMAN->Mount( "dir", ssprintf("%s/" PRODUCT_ID "/Songs", dir), "/Songs" );
src/arch/ArchHooks/ArchHooks_Unix.cpp
+1
View File
@@ -403,6 +403,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable )
FILEMAN->Mount( "dir", sUserDataPath + "/CDTitles", "/CDTitles" );
FILEMAN->Mount( "dir", sUserDataPath + "/Characters", "/Characters" );
FILEMAN->Mount( "dir", sUserDataPath + "/Courses", "/Courses" );
FILEMAN->Mount( "dir", sUserDataPath + "/Downloads", "/Downloads" );
FILEMAN->Mount( "dir", sUserDataPath + "/Logs", "/Logs" );
FILEMAN->Mount( "dir", sUserDataPath + "/NoteSkins", "/NoteSkins" );
FILEMAN->Mount( "dir", sUserDataPath + "/Packages", "/Packages" );
src/arch/ArchHooks/ArchHooks_Win32Static.cpp
+1
View File
@@ -82,6 +82,7 @@ void ArchHooks::MountUserFilesystems( const RString &sDirOfExecutable )
FILEMAN->Mount( "dir", sAppDataDir + "/CDTitles", "/CDTitles" );
FILEMAN->Mount( "dir", sAppDataDir + "/Characters", "/Characters" );
FILEMAN->Mount( "dir", sAppDataDir + "/Courses", "/Courses" );
FILEMAN->Mount( "dir", sAppDataDir + "/Downloads", "/Downloads" );
FILEMAN->Mount( "dir", sAppDataDir + "/Logs", "/Logs" );
FILEMAN->Mount( "dir", sAppDataDir + "/NoteSkins", "/NoteSkins" );
FILEMAN->Mount( "dir", sAppDataDir + "/Packages", "/Packages" );