src/HTTPHandler.h
author František Kučera <franta-hg@frantovo.cz>
Sat, 12 Mar 2022 02:03:44 +0100
branchv_0
changeset 4 602462d04c57
parent 3 9c710397ced6
child 5 165f6162524d
permissions -rw-r--r--
print somehow HTTP headers on STDERR
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     1
/**
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     2
 * Relational pipes
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     3
 * Copyright © 2020 František Kučera (Frantovo.cz, GlobalCode.info)
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     4
 *
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     5
 * This program is free software: you can redistribute it and/or modify
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     6
 * it under the terms of the GNU General Public License as published by
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     7
 * the Free Software Foundation, version 3 of the License.
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     8
 *
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     9
 * This program is distributed in the hope that it will be useful,
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    10
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    11
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    12
 * GNU General Public License for more details.
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    13
 *
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    14
 * You should have received a copy of the GNU General Public License
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    15
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    16
 */
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    17
#pragma once
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    18
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    19
#include <memory>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    20
#include <string>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    21
#include <vector>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    22
#include <codecvt>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    23
#include <regex>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    24
#include <stdexcept>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    25
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    26
#include <curl/curl.h>
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    27
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    28
#include <relpipe/common/type/typedefs.h>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    29
#include <relpipe/reader/TypeId.h>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    30
#include <relpipe/reader/handlers/RelationalReaderStringHandler.h>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    31
#include <relpipe/reader/handlers/AttributeMetadata.h>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    32
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    33
#include <relpipe/writer/Factory.h>
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    34
#include <relpipe/writer/TypeId.h>
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    35
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    36
#include <relpipe/cli/RelpipeCLIException.h>
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    37
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    38
#include "Configuration.h"
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    39
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    40
namespace relpipe {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    41
namespace tr {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    42
namespace http {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    43
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    44
class HTTPHandler : public relpipe::reader::handlers::RelationalReaderStringHandler {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    45
private:
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    46
	std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings.
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    47
	shared_ptr<relpipe::writer::RelationalWriter> relationalWriter;
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    48
	Configuration configuration;
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    49
	std::vector<relpipe::reader::handlers::AttributeMetadata> currentReaderMetadata;
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    50
	std::vector<relpipe::writer::AttributeMetadata> currentWriterMetadata;
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    51
	size_t currentAttributeIndex = 0;
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    52
	size_t currentRecordNumber = 1;
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    53
2
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    54
	void writeCallback(std::string value) {
3
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    55
		// TODO: write this attribute even if this method was not called
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    56
		// TODO: support also binary data or other encodings, not only UTF-8 text
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    57
		relationalWriter->writeAttribute(convertor.from_bytes(value));
2
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    58
	}
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    59
3
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    60
	static uint writeCurlCallback(char* buffer, size_t size, size_t nmemb, HTTPHandler* instance) {
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    61
		size_t r = size * nmemb;
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    62
		instance->writeCallback(std::string(buffer, r));
2
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    63
		return r;
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    64
	}
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    65
4
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    66
	void headersCallback(std::string value) {
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    67
		// TODO: parse name=value and store for later use
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    68
		std::cerr << std::endl << "HTTP response headers:" << std::endl << ">>>" << value << "<<<" << std::endl << "--------" << std::endl;
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    69
	}
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    70
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    71
	static uint headersCurlCallback(char* buffer, size_t size, size_t nmemb, HTTPHandler* instance) {
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    72
		size_t r = size * nmemb;
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    73
		instance->headersCallback(std::string(buffer, r));
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    74
		return r;
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    75
	}
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    76
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    77
public:
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    78
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    79
	HTTPHandler(shared_ptr<relpipe::writer::RelationalWriter> relationalWriter, Configuration configuration) : relationalWriter(relationalWriter), configuration(configuration) {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    80
	}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    81
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    82
	virtual ~HTTPHandler() {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    83
	}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    84
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    85
	void startRelation(relpipe::common::type::StringX name, std::vector<relpipe::reader::handlers::AttributeMetadata> attributes) override {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    86
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    87
3
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    88
		relationalWriter->startRelation(name + L"_curl_info",{
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    89
			{L"name", relpipe::writer::TypeId::STRING},
2
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    90
			{L"value", relpipe::writer::TypeId::STRING}
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    91
		}, true);
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    92
3
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    93
		relationalWriter->writeAttribute(L"curl version");
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    94
		relationalWriter->writeAttribute(convertor.from_bytes(curl_version()));
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    95
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    96
		relationalWriter->startRelation(name + L"_curl_response",{
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    97
			{L"url", relpipe::writer::TypeId::STRING},
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    98
			{L"body", relpipe::writer::TypeId::STRING},
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
    99
			{L"response_code", relpipe::writer::TypeId::INTEGER}
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   100
		}, true);
1
ddc12e789d00 link to the CURL library
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   101
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   102
	}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   103
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   104
	void attribute(const relpipe::common::type::StringX& value) override {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   105
3
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   106
		relationalWriter->writeAttribute(value);
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   107
2
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   108
		std::string url = convertor.to_bytes(value);
3
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   109
		relpipe::common::type::Integer responseCode = 0;
2
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   110
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   111
		CURL* curl = curl_easy_init();
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   112
		curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
4
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
   113
2
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   114
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   115
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCurlCallback);
4
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
   116
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
   117
		curl_easy_setopt(curl, CURLOPT_HEADERDATA, this);
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
   118
		curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headersCurlCallback);
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
   119
2
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   120
		curl_easy_perform(curl);
4
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
   121
3
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   122
		curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode);
4
602462d04c57 print somehow HTTP headers on STDERR
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
   123
2
dccedac46e7e do HTTP request for each attribute value
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   124
		curl_easy_cleanup(curl);
3
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   125
9c710397ced6 return also the HTTP response code and URL
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   126
		relationalWriter->writeAttribute(&responseCode, typeid (responseCode));
0
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   127
	}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   128
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   129
	void endOfPipe() {
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   130
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   131
	}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   132
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   133
};
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   134
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   135
}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   136
}
ce520a238309 new project relpipe-tr-http
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   137
}