--- a/src/KafkaCommand.cpp Sun Apr 24 22:21:02 2022 +0200
+++ b/src/KafkaCommand.cpp Wed Apr 27 01:47:35 2022 +0200
@@ -19,6 +19,8 @@
#include <locale>
#include <algorithm>
+#include <librdkafka/rdkafkacpp.h>
+
#include <relpipe/writer/RelationalWriter.h>
#include <relpipe/writer/RelpipeWriterException.h>
#include <relpipe/writer/AttributeMetadata.h>
@@ -26,6 +28,8 @@
#include <relpipe/writer/TypeId.h>
#include <relpipe/cli/CLI.h>
+#include <condition_variable>
+#include <unistd.h>
#include "KafkaCommand.h"
#include "Hex.h"
@@ -38,22 +42,76 @@
namespace in {
namespace kafka {
+static void check(RdKafka::Conf::ConfResult result, const std::string& errString) {
+ if (result != RdKafka::Conf::CONF_OK) {
+ throw std::logic_error("Unable to configure Kafka: " + errString);
+ }
+}
+
+static void check(RdKafka::ErrorCode result, const std::string& errString) {
+ if (result != RdKafka::ERR_NO_ERROR) {
+ throw std::logic_error("Kafka error: " + errString);
+ }
+}
+
+static void check(std::shared_ptr<RdKafka::KafkaConsumer> kafkaConsumer, const std::string& errString) {
+ if (kafkaConsumer.get() == nullptr) {
+ throw std::logic_error("Unable to create Kafka consumer: " + errString);
+ }
+}
+
void KafkaCommand::process(std::shared_ptr<writer::RelationalWriter> writer, Configuration& configuration) {
vector<AttributeMetadata> metadata;
+ std::string errString;
+ std::shared_ptr<RdKafka::Conf> consumerConf(RdKafka::Conf::create(RdKafka::Conf::ConfType::CONF_GLOBAL));
+
+ // TODO: configurable groupId, clientId and other parameters
+ std::string groupId = "relpipe-in-kafka-group-" + std::to_string(getpid());
+ std::string clientId = "relpipe-in-kafka-client-" + std::to_string(getpid());
+
+ check(consumerConf->set("client.id", clientId, errString), errString);
+ check(consumerConf->set("group.id", groupId, errString), errString);
+ check(consumerConf->set("bootstrap.servers", "plaintext://127.0.0.1:9092", errString), errString);
+ //check(consumerConf->set("auto.offset.reset", "earliest", errString), errString);
+ //check(consumerConf->set("debug", "all", errString), errString);
+
+ std::shared_ptr<RdKafka::KafkaConsumer> kafkaConsumer(RdKafka::KafkaConsumer::create(consumerConf.get(), errString));
+ check(kafkaConsumer, errString);
+
+ check(kafkaConsumer->subscribe({"relpipe"}), errString);
+
+
writer->startRelation(configuration.relation,{
{L"queue", TypeId::STRING},
{L"text", TypeId::STRING},
{L"data", TypeId::STRING}
}, true);
- for (int i = configuration.messageCount; i > 0; i--) {
- std::string message = "TODO: read message from Kafka";
- writer->writeAttribute(configuration.queue);
- writer->writeAttribute(Hex::toTxt(message));
- writer->writeAttribute(Hex::toHex(message));
+ for (int i = configuration.messageCount; continueProcessing && i > 0;) {
+ shared_ptr<RdKafka::Message> message(kafkaConsumer->consume(100));
+ if (message.get() && message->err() == 0) {
+ std::string payload = message->payload() ? std::string((const char*) message->payload(), message->len()) : std::string("");
+ writer->writeAttribute(convertor.from_bytes(message->topic_name()));
+ writer->writeAttribute(Hex::toTxt(payload));
+ writer->writeAttribute(Hex::toHex(payload));
+ i--;
+ } else if (message->err() == RdKafka::ErrorCode::ERR__TIMED_OUT) {
+ // timeout → try again
+ } else if (message->err() == RdKafka::ErrorCode::ERR__PARTITION_EOF) {
+ // reached the end of the topic/partition → try again
+ } else {
+ std::string m = "error while reading message: " + (message.get() ? message->errstr() : "message is missing");
+ writer->writeAttribute(configuration.queue);
+ writer->writeAttribute(Hex::toTxt(m));
+ writer->writeAttribute(Hex::toHex(m));
+ break;
+ }
}
+ // TODO: wrap and close even on exception
+ check(kafkaConsumer->close(), errString);
+
}
KafkaCommand::~KafkaCommand() {