HttpServerConnection: Log noticable CPU semaphore wait time

Author: yhabteab

lib/remote/httpserverconnection.cpp

@@ -435,6 +435,7 @@ bool ProcessRequest(
 	boost::beast::http::request<boost::beast::http::string_body>& request,
 	ApiUser::Ptr& authenticatedUser,
 	boost::beast::http::response<boost::beast::http::string_body>& response,
+	std::chrono::steady_clock::duration& cpuBoundWorkTime,
 	boost::asio::yield_context& yc
 )
 {
@@ -443,7 +444,10 @@ bool ProcessRequest(
 	auto stream (server.GetStream());
 
 	try {
+		// Cache the elapsed time to acquire a CPU semaphore used to detect extremely heavy workloads.
+		auto start (std::chrono::steady_clock::now());
 		CpuBoundWork handlingRequest (yc);
+		cpuBoundWorkTime = std::chrono::steady_clock::now() - start;
 
 		HttpHandler::ProcessRequest(server, authenticatedUser, request, response, yc);
 	} catch (const std::exception& ex) {
@@ -534,9 +538,15 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc)
 				<< ", user: " << (authenticatedUser ? authenticatedUser->GetName() : "<unauthenticated>")
 				<< ", agent: " << request[http::field::user_agent]; //operator[] - Returns the value for a field, or "" if it does not exist.
 
-			Defer addRespCode ([&response, start, &logMsg]() {
-				logMsg << ", status: " << response.result() << ") took "
-					<< ch::duration_cast<ch::milliseconds>(ch::steady_clock::now() - start).count() << "ms.";
+			ch::steady_clock::duration cpuBoundWorkTime(0);
+			Defer addRespCode ([&response, start, &logMsg, &cpuBoundWorkTime]() {
+				logMsg << ", status: " << response.result() << ")";
+
+				if (ch::duration_cast<ch::seconds>(cpuBoundWorkTime).count()) {
+					logMsg << " waited '" << ch::duration_cast<ch::milliseconds>(cpuBoundWorkTime).count() << "ms' on semaphore and";
+				}
+
+				logMsg << " took total " << ch::duration_cast<ch::milliseconds>(ch::steady_clock::now() - start).count() << "ms.";
 			});
 
 			if (!HandleAccessControl(*m_Stream, request, response, yc)) {
@@ -557,7 +567,7 @@ void HttpServerConnection::ProcessMessages(boost::asio::yield_context yc)
 
 			m_Seen = std::numeric_limits<decltype(m_Seen)>::max();
 
-			if (!ProcessRequest(*this, request, authenticatedUser, response, yc)) {
+			if (!ProcessRequest(*this, request, authenticatedUser, response, cpuBoundWorkTime, yc)) {
 				break;
 			}