/**
* Relational pipes
* Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include<functional>
#include <memory>
#include <string>
#include <vector>
#include <iostream>
#include <sstream>
#include <locale>
#include <codecvt>
#include <regex>
#include <unistd.h>
#include <wait.h>
#include <ext/stdio_filebuf.h>
#include <relpipe/reader/typedefs.h>
#include <relpipe/reader/TypeId.h>
#include <relpipe/reader/handlers/RelationalReaderStringHandler.h>
#include <relpipe/reader/handlers/AttributeMetadata.h>
#include <relpipe/writer/Factory.h>
#include <relpipe/cli/RelpipeCLIException.h>
#include "Configuration.h"
namespace relpipe {
namespace tr {
namespace awk {
using namespace std;
using namespace relpipe;
using namespace relpipe::reader;
using namespace relpipe::reader::handlers;
/**
* This transformation consists of three processes connected together using pipes.
*
* Processes:
* - Parent: process RelationalReaderStringHandler events (read relational data, usually from STDIN) and pass them in the special text format to the pipe1
* - AWK: external program (/usr/bin/awk), read from the pipe1, writes to the pipe2
* - Writer: reads from the pipe2 and writes to relationalWriter (relational data, usually to STDOUT)
*/
class AwkHandler : public RelationalReaderStringHandler {
private:
Configuration configuration;
writer::RelationalWriter* relationalWriter;
std::function<void() > relationalWriterFlush;
std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings
int awkInputWriterFD = -1;
RelationConfiguration* currentRelationConfiguration = nullptr;
std::vector<AttributeMetadata> currentReaderMetadata;
std::vector<writer::AttributeMetadata> currentWriterMetadata;
std::map<string_t, string_t> currenVariablesMapping;
integer_t currentAttributeIndex = 0;
void createPipe(int& readerFD, int& writerFD) {
int fds[2];
int result = pipe(fds);
readerFD = fds[0];
writerFD = fds[1];
if (result < 0) throw cli::RelpipeCLIException(L"Unable to create a pipe.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
}
void redirectFD(int oldfd, int newfd) {
int result = dup2(oldfd, newfd);
if (result < 0) throw cli::RelpipeCLIException(L"Unable redirect FD.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
}
void closeOrThrow(int fd) {
int error = close(fd);
if (error) throw cli::RelpipeCLIException(L"Unable to close FD: " + to_wstring(fd) + L" from PID: " + to_wstring(getpid()), cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
}
void execp(const std::vector<std::string>& args) {
const char** a = new const char*[args.size() + 1];
for (size_t i = 0; i < args.size(); i++) a[i] = args[i].c_str();
a[args.size()] = nullptr;
execvp(a[0], (char*const*) a);
delete[] a;
throw cli::RelpipeCLIException(L"Unable to do execvp().", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
}
/* TODO: move to lib-cli when stable and used in other modules */
void setEnv(const char * name, const string_t& value) {
setenv(name, convertor.to_bytes(value).c_str(), true);
}
/* TODO: move to lib-cli when stable and used in other modules */
void setEnv(const char * name, std::string& value) {
setenv(name, value.c_str(), true);
}
/* TODO: move to lib-cli when stable and used in other modules */
void setEnv(const char * name, const integer_t& value) {
setenv(name, to_string(value).c_str(), true);
}
void addDefinition(std::vector<std::string>& awkCliArgs, DefinitionRecipe& d) {
awkCliArgs.push_back("-v");
awkCliArgs.push_back(convertor.to_bytes(a2v(d.name) + L"=" + d.value));
}
void add(vector<AttributeMetadata>& readerAttributes, vector<writer::AttributeMetadata>& writerAttributes) {
for (AttributeMetadata readerAttributes : readerAttributes)
writerAttributes.push_back({
readerAttributes.getAttributeName(),
relationalWriter->toTypeId(readerAttributes.getTypeName())
});
}
void cleanUp() {
if (awkInputWriterFD >= 0) {
closeOrThrow(awkInputWriterFD);
// FIXME: check exit codes
__pid_t waitResult1 = wait(NULL);
__pid_t waitResult2 = wait(NULL);
awkInputWriterFD = -1;
}
currentAttributeIndex = 0;
currentReaderMetadata.clear();
currentWriterMetadata.clear();
currenVariablesMapping.clear();
currentRelationConfiguration = nullptr;
}
void generateVariableMappings() {
for (AttributeMetadata m : currentReaderMetadata) currenVariablesMapping[m.getAttributeName()] = L"";
for (writer::AttributeMetadata m : currentWriterMetadata) currenVariablesMapping[m.attributeName] = L"";
for (DefinitionRecipe d : configuration.definitions) currenVariablesMapping[d.name] = L"";
for (DefinitionRecipe d : currentRelationConfiguration->definitions) currenVariablesMapping[d.name] = L"";
for (std::pair<string_t, string_t> m : currenVariablesMapping) {
currenVariablesMapping[m.first] = escapeAwkVariableName(m.first);
}
}
string_t a2v(const string_t& attributeName) {
if (currenVariablesMapping.find(attributeName) != currenVariablesMapping.end()) return currenVariablesMapping[attributeName];
else throw cli::RelpipeCLIException(L"Unable to find value in currenVariablesMapping", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
}
template <typename K, typename V> bool containsValue(std::map<K, V> map, V value) {
for (std::pair<K, V> p : map) if (p.second == value) return true;
return false;
}
string_t escapeAwkVariableName(const string_t& attributeName) {
// cat awkgram.y | awk -v FS='\\{"|",' -v ORS='|' '/static const struct token tokentab/, /\};/ { if (/^\{/) { print $2} }'
// cat AwkHandler.h | awk -v FS=' |\\(' -v ORS='|' '/awkScript.*"function/ { print $4; }'
std::wregex awkReservedNames(L"BEGIN|BEGINFILE|END|ENDFILE|adump|and|asort|asorti|atan2|bindtextdomain|break|case|close|compl|continue|cos|dcgettext|dcngettext|default|delete|do|else|eval|exit|exp|fflush|for|func|function|gensub|getline|gsub|if|in|include|index|int|intdiv0|isarray|length|load|log|lshift|match|mktime|namespace|next|nextfile|or|patsplit|print|printf|rand|return|rshift|sin|split|sprintf|sqrt|srand|stopme|strftime|strtonum|sub|substr|switch|system|systime|tolower|toupper|typeof|while|xor");
std::wregex trReservedNames(L"_escape|_unescape|_readVariables|_writeVariables|record");
std::wregex badCharacters(L"[^a-zA-Z0-9_]|^([0-9])");
const string_t& name = std::regex_replace(attributeName, badCharacters, L"_$1");
bool badName = false;
badName |= regex_match(name, awkReservedNames);
badName |= regex_match(name, trReservedNames);
badName |= containsValue(currenVariablesMapping, name);
if (badName) return escapeAwkVariableName(L"_" + name);
else return name;
}
string_t escapeAwkValue(const string_t& value) {
std::wstringstream escaped;
for (wchar_t ch : value) {
if (ch == '\t') escaped << L"\\t";
else if (ch == '\n') escaped << L"\\n";
else if (ch == '\\') escaped << L"\\\\";
else escaped << ch;
}
return escaped.str();
}
void processAwkOutput(int awkOutputReaderFD) {
locale::global(locale("")); // needed for processing unicode texts, otherwise getline() stopped working on first line with non-ascii characters; TODO: move somewhere else?
__gnu_cxx::stdio_filebuf<wchar_t> awkOutputReaderBuffer(awkOutputReaderFD, std::ios::in);
std::wistream awkOutputReader(&awkOutputReaderBuffer);
if (currentRelationConfiguration->drop) {
for (wchar_t ch; awkOutputReader.read(&ch, 1).good();); // just eat the lines from the AWK
} else {
std::wstringstream currentValue;
for (wchar_t ch; awkOutputReader.read(&ch, 1).good();) {
if (ch == '\t' || ch == '\n') {
relationalWriter->writeAttribute(currentValue.str());
currentValue.str(L"");
currentValue.clear();
} else if (ch == '\\') {
ch = awkOutputReader.get();
if (ch == 't') currentValue << L'\t';
else if (ch == 'n') currentValue << L'\n';
else if (ch == '\\') currentValue << L'\\';
else throw cli::RelpipeCLIException(L"Unknown escape sequence. Only \\t, \\n and \\\\ are supported.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
} else {
currentValue << ch;
}
}
}
closeOrThrow(awkOutputReaderFD);
}
public:
/**
* @param relationalWriter
* @param relationalWriterFlush the writer must be flushed before fork() in order to
* avoid duplicate output (otherwise single relation might be written from two processes);
* This is a little hack – if it stops working, we should reconnect the pipes
* and use the writer only from a single process and avoid its effective duplication,
* or use different writers for each relation (or process).
* @param configuration
*/
AwkHandler(writer::RelationalWriter* relationalWriter, std::function<void() > relationalWriterFlush, Configuration& configuration) : relationalWriter(relationalWriter), relationalWriterFlush(relationalWriterFlush), configuration(configuration) {
}
void startRelation(string_t name, vector<AttributeMetadata> attributes) override {
cleanUp();
for (int i = 0; i < configuration.relationConfigurations.size(); i++) {
if (regex_match(name, wregex(configuration.relationConfigurations[i].relation))) {
currentRelationConfiguration = &configuration.relationConfigurations[i];
break; // it there are multiple matches, only the first configuration is used
}
}
currentReaderMetadata = attributes;
// TODO: move to a reusable method (or use same metadata on both reader and writer side?)
if (currentRelationConfiguration && currentRelationConfiguration->writerMetadata.size()) {
if (currentRelationConfiguration->inputAttributesPrepend) add(currentReaderMetadata, currentWriterMetadata);
currentWriterMetadata.insert(currentWriterMetadata.end(), currentRelationConfiguration->writerMetadata.begin(), currentRelationConfiguration->writerMetadata.end());
if (currentRelationConfiguration->inputAttributesAppend) add(currentReaderMetadata, currentWriterMetadata);
} else {
add(currentReaderMetadata, currentWriterMetadata);
}
if (currentRelationConfiguration) {
generateVariableMappings();
int awkInputReaderFD;
int awkOutputReaderFD;
int awkOutputWriterFD;
createPipe(awkInputReaderFD, awkInputWriterFD);
createPipe(awkOutputReaderFD, awkOutputWriterFD);
relationalWriterFlush();
__pid_t awkPid = fork();
if (awkPid < 0) {
throw cli::RelpipeCLIException(L"Unable to fork AWK process.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
} else if (awkPid == 0) {
// AWK child process
closeOrThrow(awkInputWriterFD);
closeOrThrow(awkOutputReaderFD);
redirectFD(awkInputReaderFD, STDIN_FILENO);
redirectFD(awkOutputWriterFD, STDOUT_FILENO);
// Environment variables:
setEnv("relationName", name);
setEnv("inputAttributeCount", currentReaderMetadata.size());
setEnv("outputAttributeCount", currentWriterMetadata.size());
// TODO: better names? ENV vs. AWK variables?
for (int i = 0; i < currentReaderMetadata.size(); i++) {
setEnv((std::string("inputAttributeName") + std::to_string(i)).c_str(), currentReaderMetadata[i].getAttributeName());
setEnv("inputAttributeType" + i, currentReaderMetadata[i].getTypeName());
}
for (int i = 0; i < currentWriterMetadata.size(); i++) {
setEnv("outputAttributeName" + i, currentWriterMetadata[i].attributeName);
// setEnv("outputAttributeType" + i, currentWriterMetadata[i].typeId); // TODO: type?
}
// AWK script:
std::wstringstream awkScript;
awkScript << L"BEGIN {" << std::endl;
awkScript << L"FS=\"\\t\";" << std::endl;
awkScript << L"OFS=\"\\t\";" << std::endl;
awkScript << currentRelationConfiguration->awkBeforeRecords << std::endl;
awkScript << L"};" << std::endl;
awkScript << std::endl;
awkScript << L"END {" << std::endl;
awkScript << currentRelationConfiguration->awkAfterRecords << std::endl;
awkScript << L"};" << std::endl;
awkScript << std::endl;
awkScript << L"function _escape(value) {" << std::endl;
// FIXME: escape function
awkScript << L"return value;" << std::endl;
awkScript << L"};" << std::endl;
awkScript << std::endl;
awkScript << L"function _unescape(value) {" << std::endl;
// FIXME: unescape function
awkScript << L"return value;" << std::endl;
awkScript << L"};" << std::endl;
awkScript << std::endl;
awkScript << L"function _readVariables() {" << std::endl;
for (int i = 0; i < currentReaderMetadata.size(); i++) awkScript << a2v(currentReaderMetadata[i].getAttributeName()) << L"=_unescape($" << (i + 1) << L");" << std::endl;
awkScript << L"};" << std::endl;
awkScript << std::endl;
awkScript << L"function _writeVariables() {" << std::endl;
awkScript << L"NF=" << currentWriterMetadata.size() << ";" << std::endl;
for (int i = 0; i < currentWriterMetadata.size(); i++) awkScript << L"$" << (i + 1) << L"=_escape(" << a2v(currentWriterMetadata[i].attributeName) << L");" << std::endl;
awkScript << L"};" << std::endl;
awkScript << std::endl;
awkScript << L"function record() {" << std::endl;
awkScript << L"_writeVariables();" << std::endl;
awkScript << L"print;" << std::endl;
awkScript << L"};" << std::endl;
awkScript << std::endl;
awkScript << L"{ _readVariables(); }" << std::endl; // read line (input attributes) into AWK variables
awkScript << L"{ _writeVariables(); }" << std::endl; // write AWK variables to the line (so it matches the output attributes and can be implicitly printed without explicit record() call)
awkScript << std::endl;
awkScript << currentRelationConfiguration->awkForEach << std::endl; // user's code – can modify variables, filter results or explicitly call record() (can generate additional records or duplicate them)
// CLI arguments:
std::vector<std::string> args;
args.push_back("awk");
for (auto d : configuration.definitions) addDefinition(args, d);
for (auto d : currentRelationConfiguration->definitions) addDefinition(args, d);
args.push_back(convertor.to_bytes(awkScript.str()));
// Runs AWK program found on $PATH → user can plug-in a custom implementation or a wrapper, but this can be also bit dangerous (however AWK itself is dangerous).
execp(args);
} else {
// Parent process
closeOrThrow(awkInputReaderFD);
closeOrThrow(awkOutputWriterFD);
__pid_t writerPid = fork();
if (writerPid < 0) {
throw cli::RelpipeCLIException(L"Unable to fork Writer process.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
} else if (writerPid == 0) {
// Writer child process
closeOrThrow(awkInputWriterFD);
if (currentRelationConfiguration->drop) {
// TODO: omit whole this process and pipe AWK output to /dev/null?
} else {
relationalWriter->startRelation(name, currentWriterMetadata, true);
}
processAwkOutput(awkOutputReaderFD);
exit(0);
} else {
// Parent process
closeOrThrow(awkOutputReaderFD);
}
}
} else {
relationalWriter->startRelation(name, currentWriterMetadata, true);
}
}
void attribute(const string_t& value) override {
if (currentRelationConfiguration) {
currentAttributeIndex++;
currentAttributeIndex = currentAttributeIndex % currentReaderMetadata.size();
std::string awkValue = convertor.to_bytes(escapeAwkValue(value));
if (currentAttributeIndex == 0) awkValue += "\n";
else awkValue += "\t";
write(awkInputWriterFD, awkValue.c_str(), awkValue.length());
} else {
relationalWriter->writeAttribute(value);
}
}
void endOfPipe() {
cleanUp();
}
};
}
}
}