src/HTTPHandler.h
author František Kučera <franta-hg@frantovo.cz>
Tue, 05 Apr 2022 00:06:12 +0200
branchv_0
changeset 25 dbeae485a3fd
parent 23 33f8162a4971
child 27 b679797949e9
permissions -rw-r--r--
request body support: unfinished version with read callback
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     1
/**
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     2
 * Relational pipes
5
165f6162524d introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
     3
 * Copyright © 2022 František Kučera (Frantovo.cz, GlobalCode.info)
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     4
 *
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     5
 * This program is free software: you can redistribute it and/or modify
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     6
 * it under the terms of the GNU General Public License as published by
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     7
 * the Free Software Foundation, version 3 of the License.
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     8
 *
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     9
 * This program is distributed in the hope that it will be useful,
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    10
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    11
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    12
 * GNU General Public License for more details.
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    13
 *
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    14
 * You should have received a copy of the GNU General Public License
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    15
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    16
 */
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    17
#pragma once
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    18
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    19
#include <memory>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    20
#include <string>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    21
#include <vector>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    22
#include <codecvt>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    23
#include <regex>
18
d8efcefdf906 support common-header filtering by URL regex patterns
František Kučera <franta-hg@frantovo.cz>
parents: 17
diff changeset
    24
#include <locale>
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    25
#include <stdexcept>
22
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
    26
#include <iomanip>
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    27
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    28
#include <curl/curl.h>
21
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
    29
#include <uuid/uuid.h>
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    30
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    31
#include <relpipe/common/type/typedefs.h>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    32
#include <relpipe/reader/TypeId.h>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    33
#include <relpipe/reader/handlers/RelationalReaderStringHandler.h>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    34
#include <relpipe/reader/handlers/AttributeMetadata.h>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    35
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    36
#include <relpipe/writer/Factory.h>
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    37
#include <relpipe/writer/TypeId.h>
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    38
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    39
#include <relpipe/cli/RelpipeCLIException.h>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    40
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    41
#include "Configuration.h"
5
165f6162524d introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
    42
#include "HTTPClient.h"
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    43
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    44
namespace relpipe {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    45
namespace tr {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    46
namespace http {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    47
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    48
class HTTPHandler : public relpipe::reader::handlers::RelationalReaderStringHandler {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    49
private:
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    50
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    51
	class HeaderDefinition {
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    52
	public:
20
cad9f6d421ee support header filtering also by request ID pattern, not only URL pattern
František Kučera <franta-hg@frantovo.cz>
parents: 19
diff changeset
    53
		std::wregex request = std::wregex(L".*");
18
d8efcefdf906 support common-header filtering by URL regex patterns
František Kučera <franta-hg@frantovo.cz>
parents: 17
diff changeset
    54
		std::wregex url = std::wregex(L".*");
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    55
		relpipe::common::type::StringX name;
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    56
		relpipe::common::type::StringX value;
18
d8efcefdf906 support common-header filtering by URL regex patterns
František Kučera <franta-hg@frantovo.cz>
parents: 17
diff changeset
    57
20
cad9f6d421ee support header filtering also by request ID pattern, not only URL pattern
František Kučera <franta-hg@frantovo.cz>
parents: 19
diff changeset
    58
		bool matches(const relpipe::common::type::StringX& request, const relpipe::common::type::StringX& url) const {
cad9f6d421ee support header filtering also by request ID pattern, not only URL pattern
František Kučera <franta-hg@frantovo.cz>
parents: 19
diff changeset
    59
			bool result = false;
cad9f6d421ee support header filtering also by request ID pattern, not only URL pattern
František Kučera <franta-hg@frantovo.cz>
parents: 19
diff changeset
    60
			result |= std::regex_match(request, this->request);
cad9f6d421ee support header filtering also by request ID pattern, not only URL pattern
František Kučera <franta-hg@frantovo.cz>
parents: 19
diff changeset
    61
			result |= std::regex_match(url, this->url);
cad9f6d421ee support header filtering also by request ID pattern, not only URL pattern
František Kučera <franta-hg@frantovo.cz>
parents: 19
diff changeset
    62
			return result;
18
d8efcefdf906 support common-header filtering by URL regex patterns
František Kučera <franta-hg@frantovo.cz>
parents: 17
diff changeset
    63
		}
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    64
	};
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    65
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    66
	std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings.
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    67
	shared_ptr<relpipe::writer::RelationalWriter> relationalWriter;
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    68
	Configuration configuration;
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    69
	relpipe::common::type::StringX currentRelationName;
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    70
	std::vector<relpipe::reader::handlers::AttributeMetadata> currentReaderMetadata;
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    71
	std::vector<relpipe::writer::AttributeMetadata> currentWriterMetadata;
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    72
	HeaderDefinition requestHeader;
19
0fc76872a921 support request ID (for JOINing requests + responses + response headers)
František Kučera <franta-hg@frantovo.cz>
parents: 18
diff changeset
    73
	relpipe::common::type::StringX requestId;
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
    74
	HTTPClient::Request request;
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    75
	std::vector<HeaderDefinition> requestHeaders;
6
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    76
	std::vector<relpipe::common::type::StringX> responseHeaders;
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    77
	size_t currentAttributeIndex = 0;
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    78
	size_t currentRecordNumber = 1;
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    79
6
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    80
	void writeHeaders() {
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    81
		if (responseHeaders.size()) {
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
    82
			relationalWriter->startRelation(L"header",{
19
0fc76872a921 support request ID (for JOINing requests + responses + response headers)
František Kučera <franta-hg@frantovo.cz>
parents: 18
diff changeset
    83
				{L"request", relpipe::writer::TypeId::STRING},
6
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    84
				{L"url", relpipe::writer::TypeId::STRING},
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    85
				{L"name", relpipe::writer::TypeId::STRING},
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    86
				{L"value", relpipe::writer::TypeId::STRING},
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    87
			}, true);
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    88
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    89
			for (auto s : responseHeaders) relationalWriter->writeAttribute(s);
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    90
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    91
			responseHeaders.clear();
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    92
		}
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    93
	}
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
    94
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
    95
	HTTPClient::Method parseMethod(const relpipe::common::type::StringX& value) {
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
    96
		if (value.size() == 0) return HTTPClient::Method::GET;
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
    97
		else if (value == L"GET") return HTTPClient::Method::GET;
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
    98
		else if (value == L"HEAD") return HTTPClient::Method::HEAD;
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
    99
		else if (value == L"POST") return HTTPClient::Method::POST;
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   100
		else if (value == L"PUT") return HTTPClient::Method::PUT;
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   101
		else if (value == L"DELETE") return HTTPClient::Method::DELETE;
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   102
		else if (value == L"PATCH") return HTTPClient::Method::PATCH;
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   103
		else throw std::invalid_argument("Unsupported HTTP method: " + convertor.to_bytes(value));
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   104
	}
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   105
17
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   106
	relpipe::common::type::StringX getHeaderAttributePrefix() {
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   107
		// might be configurable - parametrized
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   108
		return L"header.";
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   109
	}
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   110
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   111
	bool isHeaderAttribute(const relpipe::common::type::StringX& attributeName) {
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   112
		return attributeName.rfind(getHeaderAttributePrefix(), 0) == 0;
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   113
	}
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   114
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   115
	relpipe::common::type::StringX fetchHeaderName(const relpipe::common::type::StringX& attributeName) {
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   116
		return attributeName.substr(getHeaderAttributePrefix().size());
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   117
	}
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   118
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   119
	void appendRequestHeader(const relpipe::common::type::StringX& name, const relpipe::common::type::StringX& value) {
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   120
		request.headers.push_back(convertor.to_bytes(name));
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   121
		request.headers.push_back(convertor.to_bytes(value));
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   122
	}
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   123
21
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   124
	relpipe::common::type::StringX generateRequestId() {
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   125
		char buffer[37];
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   126
		uuid_t uuid;
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   127
		uuid_generate_random(uuid);
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   128
		// uuid_generate_time(uuid);
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   129
		uuid_unparse_lower(uuid, buffer);
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   130
		return convertor.from_bytes(buffer);
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   131
	}
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   132
22
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   133
	relpipe::common::type::StringX bodyToText(const std::string& body, bool* validEncoding = nullptr) {
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   134
		try {
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   135
			if (validEncoding) *validEncoding = true;
23
33f8162a4971 return at least ASCII text when unable to decode
František Kučera <franta-hg@frantovo.cz>
parents: 22
diff changeset
   136
			// TODO: use encoding from the HTTP response headers instead of the constant one?
22
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   137
			return convertor.from_bytes(body);
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   138
		} catch (...) {
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   139
			if (validEncoding) *validEncoding = false;
23
33f8162a4971 return at least ASCII text when unable to decode
František Kučera <franta-hg@frantovo.cz>
parents: 22
diff changeset
   140
			std::stringstream filtered;
33f8162a4971 return at least ASCII text when unable to decode
František Kučera <franta-hg@frantovo.cz>
parents: 22
diff changeset
   141
			for (char ch : body) filtered << (ch >= ' ' && ch < 127 ? ch : '.');
33f8162a4971 return at least ASCII text when unable to decode
František Kučera <franta-hg@frantovo.cz>
parents: 22
diff changeset
   142
			return convertor.from_bytes(filtered.str());
22
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   143
		}
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   144
	}
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   145
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   146
	relpipe::common::type::StringX bodyToHex(const std::string& body) {
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   147
		std::stringstream hex;
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   148
		hex << std::hex << std::setfill('0') << std::hex;
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   149
		for (size_t i = 0, size = body.size(); i < size; i++) hex << std::setw(2) << (0xff & body[i]);
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   150
		return convertor.from_bytes(hex.str());
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   151
	}
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   152
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   153
public:
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   154
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   155
	HTTPHandler(shared_ptr<relpipe::writer::RelationalWriter> relationalWriter, Configuration configuration) : relationalWriter(relationalWriter), configuration(configuration) {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   156
	}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   157
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   158
	virtual ~HTTPHandler() {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   159
	}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   160
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   161
	void startRelation(relpipe::common::type::StringX name, std::vector<relpipe::reader::handlers::AttributeMetadata> attributes) override {
6
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
   162
		writeHeaders(); // from previous relation
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   163
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   164
		currentRelationName = name;
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   165
		currentReaderMetadata = attributes;
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   166
		currentAttributeIndex = 0;
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   167
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   168
		if (currentRelationName == L"header") {
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   169
			// TODO: analyze header attributes
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   170
		} else if (currentRelationName == L"request") {
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   171
			relationalWriter->startRelation(L"response",{
19
0fc76872a921 support request ID (for JOINing requests + responses + response headers)
František Kučera <franta-hg@frantovo.cz>
parents: 18
diff changeset
   172
				{L"request", relpipe::writer::TypeId::STRING},
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   173
				{L"url", relpipe::writer::TypeId::STRING},
22
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   174
				{L"text", relpipe::writer::TypeId::STRING},
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   175
				{L"data", relpipe::writer::TypeId::STRING},
23
33f8162a4971 return at least ASCII text when unable to decode
František Kučera <franta-hg@frantovo.cz>
parents: 22
diff changeset
   176
				{L"text_valid", relpipe::writer::TypeId::BOOLEAN},
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   177
				{L"code", relpipe::writer::TypeId::INTEGER},
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   178
			}, true);
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   179
		}
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   180
	}
3
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   181
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   182
	void attribute(const relpipe::common::type::StringX& value) override {
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   183
		if (currentRelationName == L"header") headerAttribute(value);
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   184
		else if (currentRelationName == L"request") requestAttribute(value);
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   185
		else throw std::invalid_argument("Unsupported relation: " + convertor.to_bytes(currentRelationName));
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   186
	}
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   187
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   188
private:
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   189
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   190
	void headerAttribute(const relpipe::common::type::StringX& value) {
17
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   191
		auto attributeName = currentReaderMetadata[currentAttributeIndex].getAttributeName();
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   192
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   193
		if (attributeName == L"name") requestHeader.name = value;
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   194
		else if (attributeName == L"value") requestHeader.value = value;
20
cad9f6d421ee support header filtering also by request ID pattern, not only URL pattern
František Kučera <franta-hg@frantovo.cz>
parents: 19
diff changeset
   195
		else if (attributeName == L"request") requestHeader.request = std::wregex(value.size() ? value : L".*"); // TODO: null instead of empty value (when supported)
18
d8efcefdf906 support common-header filtering by URL regex patterns
František Kučera <franta-hg@frantovo.cz>
parents: 17
diff changeset
   196
		else if (attributeName == L"url") requestHeader.url = std::wregex(value.size() ? value : L".*"); // TODO: null instead of empty value (when supported)
17
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   197
		else throw std::invalid_argument("Unsupported attribute in the header relation: " + convertor.to_bytes(attributeName + L" = " + value));
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   198
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   199
		currentAttributeIndex++;
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   200
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   201
		if (currentAttributeIndex % currentReaderMetadata.size() == 0) {
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   202
			currentAttributeIndex = 0;
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   203
			requestHeaders.push_back(requestHeader);
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   204
			requestHeader = HeaderDefinition();
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   205
		}
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   206
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   207
	}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   208
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   209
	void requestAttribute(const relpipe::common::type::StringX& value) {
17
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   210
		auto attributeName = currentReaderMetadata[currentAttributeIndex].getAttributeName();
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   211
19
0fc76872a921 support request ID (for JOINing requests + responses + response headers)
František Kučera <franta-hg@frantovo.cz>
parents: 18
diff changeset
   212
		if (attributeName == L"id") requestId = value;
0fc76872a921 support request ID (for JOINing requests + responses + response headers)
František Kučera <franta-hg@frantovo.cz>
parents: 18
diff changeset
   213
		else if (attributeName == L"url") request.url = convertor.to_bytes(value);
17
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   214
		else if (attributeName == L"method") request.method = parseMethod(value);
25
dbeae485a3fd request body support: unfinished version with read callback
František Kučera <franta-hg@frantovo.cz>
parents: 23
diff changeset
   215
		else if (attributeName == L"text") request.body = convertor.to_bytes(value);
dbeae485a3fd request body support: unfinished version with read callback
František Kučera <franta-hg@frantovo.cz>
parents: 23
diff changeset
   216
		else if (attributeName == L"data") request.body = "TODO: read binary data: " + convertor.to_bytes(value); // TODO: read hex/binary request body
17
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   217
		else if (isHeaderAttribute(attributeName)) appendRequestHeader(fetchHeaderName(attributeName), value);
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   218
		else throw std::invalid_argument("Unsupported attribute in the header relation: " + convertor.to_bytes(attributeName + L" = " + value));
2
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   219
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   220
		currentAttributeIndex++;
4
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
   221
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   222
		if (currentAttributeIndex % currentReaderMetadata.size() == 0) {
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   223
			currentAttributeIndex = 0;
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   224
			std::shared_ptr<HTTPClient> http(HTTPClient::open());
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   225
21
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   226
			if (requestId.empty()) requestId = generateRequestId();
1a4174f4693a generate random request ID (UUID) if missing
František Kučera <franta-hg@frantovo.cz>
parents: 20
diff changeset
   227
20
cad9f6d421ee support header filtering also by request ID pattern, not only URL pattern
František Kučera <franta-hg@frantovo.cz>
parents: 19
diff changeset
   228
			for (const HeaderDefinition& h : requestHeaders) if (h.matches(requestId, convertor.from_bytes(request.url))) appendRequestHeader(h.name, h.value);
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   229
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   230
			std::string body;
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   231
			relpipe::common::type::Integer responseCode = -1;
7
0b0374746e48 request headers support in HTTPClient
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   232
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   233
			try {
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   234
				HTTPClient::Response response = http->exchange(request);
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   235
				responseCode = response.responseCode;
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   236
				body = response.body;
9
9fdbfbe24161 throw and report exception on error (e.g. 'Couldn't connect to server')
František Kučera <franta-hg@frantovo.cz>
parents: 8
diff changeset
   237
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   238
				for (size_t i = 0; i < response.headers.size(); i += 2) {
19
0fc76872a921 support request ID (for JOINing requests + responses + response headers)
František Kučera <franta-hg@frantovo.cz>
parents: 18
diff changeset
   239
					responseHeaders.push_back(requestId);
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   240
					responseHeaders.push_back(convertor.from_bytes(request.url));
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   241
					responseHeaders.push_back(convertor.from_bytes(response.headers[i]));
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   242
					responseHeaders.push_back(convertor.from_bytes(response.headers[i + 1]));
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   243
				}
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   244
			} catch (const HTTPClient::Exception& e) {
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   245
				body = e.getFullMessage();
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   246
				// TODO: move error message into separate attribute?
9
9fdbfbe24161 throw and report exception on error (e.g. 'Couldn't connect to server')
František Kučera <franta-hg@frantovo.cz>
parents: 8
diff changeset
   247
			}
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   248
23
33f8162a4971 return at least ASCII text when unable to decode
František Kučera <franta-hg@frantovo.cz>
parents: 22
diff changeset
   249
			bool validText = false;
19
0fc76872a921 support request ID (for JOINing requests + responses + response headers)
František Kučera <franta-hg@frantovo.cz>
parents: 18
diff changeset
   250
			relationalWriter->writeAttribute(requestId);
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   251
			relationalWriter->writeAttribute(convertor.from_bytes(request.url));
23
33f8162a4971 return at least ASCII text when unable to decode
František Kučera <franta-hg@frantovo.cz>
parents: 22
diff changeset
   252
			relationalWriter->writeAttribute(bodyToText(body, &validText));
22
1bbfcb95db82 format response body as unicode text and hexadecimal
František Kučera <franta-hg@frantovo.cz>
parents: 21
diff changeset
   253
			relationalWriter->writeAttribute(bodyToHex(body)); // TODO: return as an octet-string (when supported) instead of hexadecimal
23
33f8162a4971 return at least ASCII text when unable to decode
František Kučera <franta-hg@frantovo.cz>
parents: 22
diff changeset
   254
			relationalWriter->writeAttribute(&validText, typeid (validText));
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   255
			relationalWriter->writeAttribute(&responseCode, typeid (responseCode));
17
aa43616375c6 support also per-request headers
František Kučera <franta-hg@frantovo.cz>
parents: 15
diff changeset
   256
15
25be376736cc request methods support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   257
			request = HTTPClient::Request();
19
0fc76872a921 support request ID (for JOINing requests + responses + response headers)
František Kučera <franta-hg@frantovo.cz>
parents: 18
diff changeset
   258
			requestId.clear();
9
9fdbfbe24161 throw and report exception on error (e.g. 'Couldn't connect to server')
František Kučera <franta-hg@frantovo.cz>
parents: 8
diff changeset
   259
		}
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   260
	}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   261
8
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   262
public:
3f4e60990393 request headers support in HTTPHandler
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   263
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   264
	void endOfPipe() {
6
59c9ca066322 write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
   265
		writeHeaders(); // from last relation
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   266
	}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   267
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   268
};
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   269
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   270
}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   271
}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   272
}