src/KafkaCommand.cpp
author František Kučera <franta-hg@frantovo.cz>
Wed, 27 Apr 2022 01:47:35 +0200
branchv_0
changeset 1 6a2ae23c53c4
parent 0 5499cbd842ab
permissions -rw-r--r--
first version of Kafka consumer
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     1
/**
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     2
 * Relational pipes
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     3
 * Copyright © 2022 František Kučera (Frantovo.cz, GlobalCode.info)
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     4
 *
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     5
 * This program is free software: you can redistribute it and/or modify
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     6
 * it under the terms of the GNU General Public License as published by
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     7
 * the Free Software Foundation, version 3 of the License.
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     8
 *
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     9
 * This program is distributed in the hope that it will be useful,
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    10
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    11
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    12
 * GNU General Public License for more details.
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    13
 *
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    14
 * You should have received a copy of the GNU General Public License
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    15
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    16
 */
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    17
#include <vector>
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    18
#include <memory>
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    19
#include <locale>
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    20
#include <algorithm>
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    21
1
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    22
#include <librdkafka/rdkafkacpp.h>
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    23
0
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    24
#include <relpipe/writer/RelationalWriter.h>
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    25
#include <relpipe/writer/RelpipeWriterException.h>
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    26
#include <relpipe/writer/AttributeMetadata.h>
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    27
#include <relpipe/writer/Factory.h>
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    28
#include <relpipe/writer/TypeId.h>
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    29
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    30
#include <relpipe/cli/CLI.h>
1
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    31
#include <condition_variable>
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    32
#include <unistd.h>
0
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    33
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    34
#include "KafkaCommand.h"
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    35
#include "Hex.h"
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    36
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    37
using namespace std;
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    38
using namespace relpipe::cli;
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    39
using namespace relpipe::writer;
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    40
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    41
namespace relpipe {
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    42
namespace in {
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    43
namespace kafka {
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    44
1
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    45
static void check(RdKafka::Conf::ConfResult result, const std::string& errString) {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    46
	if (result != RdKafka::Conf::CONF_OK) {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    47
		throw std::logic_error("Unable to configure Kafka: " + errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    48
	}
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    49
}
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    50
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    51
static void check(RdKafka::ErrorCode result, const std::string& errString) {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    52
	if (result != RdKafka::ERR_NO_ERROR) {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    53
		throw std::logic_error("Kafka error: " + errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    54
	}
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    55
}
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    56
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    57
static void check(std::shared_ptr<RdKafka::KafkaConsumer> kafkaConsumer, const std::string& errString) {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    58
	if (kafkaConsumer.get() == nullptr) {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    59
		throw std::logic_error("Unable to create Kafka consumer: " + errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    60
	}
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    61
}
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    62
0
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    63
void KafkaCommand::process(std::shared_ptr<writer::RelationalWriter> writer, Configuration& configuration) {
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    64
	vector<AttributeMetadata> metadata;
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    65
1
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    66
	std::string errString;
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    67
	std::shared_ptr<RdKafka::Conf> consumerConf(RdKafka::Conf::create(RdKafka::Conf::ConfType::CONF_GLOBAL));
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    68
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    69
	// TODO: configurable groupId, clientId and other parameters
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    70
	std::string groupId = "relpipe-in-kafka-group-" + std::to_string(getpid());
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    71
	std::string clientId = "relpipe-in-kafka-client-" + std::to_string(getpid());
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    72
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    73
	check(consumerConf->set("client.id", clientId, errString), errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    74
	check(consumerConf->set("group.id", groupId, errString), errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    75
	check(consumerConf->set("bootstrap.servers", "plaintext://127.0.0.1:9092", errString), errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    76
	//check(consumerConf->set("auto.offset.reset", "earliest", errString), errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    77
	//check(consumerConf->set("debug", "all", errString), errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    78
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    79
	std::shared_ptr<RdKafka::KafkaConsumer> kafkaConsumer(RdKafka::KafkaConsumer::create(consumerConf.get(), errString));
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    80
	check(kafkaConsumer, errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    81
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    82
	check(kafkaConsumer->subscribe({"relpipe"}), errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    83
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    84
0
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    85
	writer->startRelation(configuration.relation,{
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    86
		{L"queue", TypeId::STRING},
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    87
		{L"text", TypeId::STRING},
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    88
		{L"data", TypeId::STRING}
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    89
	}, true);
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    90
1
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    91
	for (int i = configuration.messageCount; continueProcessing && i > 0;) {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    92
		shared_ptr<RdKafka::Message> message(kafkaConsumer->consume(100));
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    93
		if (message.get() && message->err() == 0) {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    94
			std::string payload = message->payload() ? std::string((const char*) message->payload(), message->len()) : std::string("");
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    95
			writer->writeAttribute(convertor.from_bytes(message->topic_name()));
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    96
			writer->writeAttribute(Hex::toTxt(payload));
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    97
			writer->writeAttribute(Hex::toHex(payload));
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    98
			i--;
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    99
		} else if (message->err() == RdKafka::ErrorCode::ERR__TIMED_OUT) {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   100
			// timeout → try again
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   101
		} else if (message->err() == RdKafka::ErrorCode::ERR__PARTITION_EOF) {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   102
			// reached the end of the topic/partition → try again
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   103
		} else {
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   104
			std::string m = "error while reading message: " + (message.get() ? message->errstr() : "message is missing");
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   105
			writer->writeAttribute(configuration.queue);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   106
			writer->writeAttribute(Hex::toTxt(m));
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   107
			writer->writeAttribute(Hex::toHex(m));
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   108
			break;
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   109
		}
0
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   110
	}
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   111
1
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   112
	// TODO: wrap and close even on exception
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   113
	check(kafkaConsumer->close(), errString);
6a2ae23c53c4 first version of Kafka consumer
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   114
0
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   115
}
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   116
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   117
KafkaCommand::~KafkaCommand() {
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   118
}
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   119
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   120
}
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   121
}
5499cbd842ab establish project
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   122
}