author | František Kučera <franta-hg@frantovo.cz> |
Sun, 13 Mar 2022 01:17:59 +0100 | |
branch | v_0 |
changeset 6 | 59c9ca066322 |
parent 5 | 165f6162524d |
child 7 | 0b0374746e48 |
permissions | -rw-r--r-- |
0
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
1 |
/** |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
2 |
* Relational pipes |
5
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
3 |
* Copyright © 2022 František Kučera (Frantovo.cz, GlobalCode.info) |
0
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
4 |
* |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
5 |
* This program is free software: you can redistribute it and/or modify |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
6 |
* it under the terms of the GNU General Public License as published by |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
7 |
* the Free Software Foundation, version 3 of the License. |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
8 |
* |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
9 |
* This program is distributed in the hope that it will be useful, |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
10 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
11 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
12 |
* GNU General Public License for more details. |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
13 |
* |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
14 |
* You should have received a copy of the GNU General Public License |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
15 |
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
16 |
*/ |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
17 |
#pragma once |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
18 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
19 |
#include <memory> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
20 |
#include <string> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
21 |
#include <vector> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
22 |
#include <codecvt> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
23 |
#include <regex> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
24 |
#include <stdexcept> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
25 |
|
1
ddc12e789d00
link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents:
0
diff
changeset
|
26 |
#include <curl/curl.h> |
ddc12e789d00
link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents:
0
diff
changeset
|
27 |
|
0
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
28 |
#include <relpipe/common/type/typedefs.h> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
29 |
#include <relpipe/reader/TypeId.h> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
30 |
#include <relpipe/reader/handlers/RelationalReaderStringHandler.h> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
31 |
#include <relpipe/reader/handlers/AttributeMetadata.h> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
32 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
33 |
#include <relpipe/writer/Factory.h> |
1
ddc12e789d00
link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents:
0
diff
changeset
|
34 |
#include <relpipe/writer/TypeId.h> |
0
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
35 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
36 |
#include <relpipe/cli/RelpipeCLIException.h> |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
37 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
38 |
#include "Configuration.h" |
5
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
39 |
#include "HTTPClient.h" |
0
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
40 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
41 |
namespace relpipe { |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
42 |
namespace tr { |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
43 |
namespace http { |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
44 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
45 |
class HTTPHandler : public relpipe::reader::handlers::RelationalReaderStringHandler { |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
46 |
private: |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
47 |
std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings. |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
48 |
shared_ptr<relpipe::writer::RelationalWriter> relationalWriter; |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
49 |
Configuration configuration; |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
50 |
std::vector<relpipe::reader::handlers::AttributeMetadata> currentReaderMetadata; |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
51 |
std::vector<relpipe::writer::AttributeMetadata> currentWriterMetadata; |
6
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
52 |
std::vector<relpipe::common::type::StringX> responseHeaders; |
0
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
53 |
size_t currentAttributeIndex = 0; |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
54 |
size_t currentRecordNumber = 1; |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
55 |
|
6
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
56 |
void writeHeaders() { |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
57 |
if (responseHeaders.size()) { |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
58 |
relationalWriter->startRelation(L"response_header",{ |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
59 |
// TODO: request ID instead of URL (or both) |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
60 |
{L"url", relpipe::writer::TypeId::STRING}, |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
61 |
{L"name", relpipe::writer::TypeId::STRING}, |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
62 |
{L"value", relpipe::writer::TypeId::STRING}, |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
63 |
}, true); |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
64 |
|
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
65 |
for (auto s : responseHeaders) relationalWriter->writeAttribute(s); |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
66 |
|
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
67 |
responseHeaders.clear(); |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
68 |
} |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
69 |
} |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
70 |
|
0
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
71 |
public: |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
72 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
73 |
HTTPHandler(shared_ptr<relpipe::writer::RelationalWriter> relationalWriter, Configuration configuration) : relationalWriter(relationalWriter), configuration(configuration) { |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
74 |
} |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
75 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
76 |
virtual ~HTTPHandler() { |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
77 |
} |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
78 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
79 |
void startRelation(relpipe::common::type::StringX name, std::vector<relpipe::reader::handlers::AttributeMetadata> attributes) override { |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
80 |
|
6
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
81 |
writeHeaders(); // from previous relation |
1
ddc12e789d00
link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents:
0
diff
changeset
|
82 |
|
3
9c710397ced6
return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
83 |
relationalWriter->startRelation(name + L"_curl_info",{ |
9c710397ced6
return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
84 |
{L"name", relpipe::writer::TypeId::STRING}, |
2
dccedac46e7e
do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents:
1
diff
changeset
|
85 |
{L"value", relpipe::writer::TypeId::STRING} |
1
ddc12e789d00
link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents:
0
diff
changeset
|
86 |
}, true); |
ddc12e789d00
link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents:
0
diff
changeset
|
87 |
|
3
9c710397ced6
return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
88 |
relationalWriter->writeAttribute(L"curl version"); |
9c710397ced6
return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
89 |
relationalWriter->writeAttribute(convertor.from_bytes(curl_version())); |
9c710397ced6
return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
90 |
|
9c710397ced6
return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
91 |
relationalWriter->startRelation(name + L"_curl_response",{ |
6
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
92 |
// TODO: request ID |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
93 |
// TODO: body in hexadecimal/binary format |
3
9c710397ced6
return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
94 |
{L"url", relpipe::writer::TypeId::STRING}, |
9c710397ced6
return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
95 |
{L"body", relpipe::writer::TypeId::STRING}, |
5
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
96 |
{L"response_code", relpipe::writer::TypeId::INTEGER}, |
3
9c710397ced6
return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
97 |
}, true); |
1
ddc12e789d00
link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents:
0
diff
changeset
|
98 |
|
0
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
99 |
} |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
100 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
101 |
void attribute(const relpipe::common::type::StringX& value) override { |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
102 |
|
5
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
103 |
std::shared_ptr<HTTPClient> http(HTTPClient::open()); |
2
dccedac46e7e
do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents:
1
diff
changeset
|
104 |
|
5
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
105 |
HTTPClient::Request request; |
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
106 |
request.method = HTTPClient::Method::GET; |
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
107 |
request.url = convertor.to_bytes(value); |
4
602462d04c57
print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents:
3
diff
changeset
|
108 |
|
5
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
109 |
HTTPClient::Response response = http->exchange(request); |
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
110 |
relpipe::common::type::Integer responseCode = response.responseCode; |
4
602462d04c57
print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents:
3
diff
changeset
|
111 |
|
5
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
112 |
relationalWriter->writeAttribute(value); |
165f6162524d
introduce HTTPClient wrapper around CURL
František Kučera <franta-hg@frantovo.cz>
parents:
4
diff
changeset
|
113 |
relationalWriter->writeAttribute(convertor.from_bytes(response.body)); |
3
9c710397ced6
return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
114 |
relationalWriter->writeAttribute(&responseCode, typeid (responseCode)); |
6
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
115 |
|
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
116 |
for (size_t i = 0; i < response.headers.size(); i += 2) { |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
117 |
responseHeaders.push_back(convertor.from_bytes(request.url)); |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
118 |
responseHeaders.push_back(convertor.from_bytes(response.headers[i])); |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
119 |
responseHeaders.push_back(convertor.from_bytes(response.headers[i + 1])); |
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
120 |
} |
0
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
121 |
} |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
122 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
123 |
void endOfPipe() { |
6
59c9ca066322
write response headers as a relation
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
124 |
writeHeaders(); // from last relation |
0
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
125 |
} |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
126 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
127 |
}; |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
128 |
|
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
129 |
} |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
130 |
} |
ce520a238309
new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff
changeset
|
131 |
} |