/**

 * Relational pipes

 * Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)

 *

 * This program is free software: you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program. If not, see <http://www.gnu.org/licenses/>.

 */

#pragma once

#include <memory>

#include <string>

#include <vector>

#include <iostream>

#include <sstream>

#include <locale>

#include <codecvt>

#include <regex>

#include <unistd.h>

#include <wait.h>

#include <ext/stdio_filebuf.h>

#include <relpipe/reader/typedefs.h>

#include <relpipe/reader/TypeId.h>

#include <relpipe/reader/handlers/RelationalReaderStringHandler.h>

#include <relpipe/reader/handlers/AttributeMetadata.h>

#include <relpipe/writer/Factory.h>

#include "Configuration.h"

#include "AwkException.h"

namespace relpipe {

namespace tr {

namespace awk {

using namespace std;

using namespace relpipe;

using namespace relpipe::reader;

using namespace relpipe::reader::handlers;

/**

 * This transformation consists of three processes connected together using pipes.

 * 

 * Processes:

 *	- Parent: process RelationalReaderStringHandler events (read relational data, usually from STDIN) and pass them in the special text format to the pipe1

 *  - AWK: external program (/usr/bin/awk), read from the pipe1, writes to the pipe2

 *  - Writer: reads from the pipe2 and writes to relationalWriter (relational data, usually to STDOUT)

 */

class AwkHandler : public RelationalReaderStringHandler {

private:

	Configuration configuration;

	writer::RelationalWriter* relationalWriter;

	std::function<void() > relationalWriterFlush;

	std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings

	int awkInputWriterFD = -1;

	RelationConfiguration* currentRelationConfiguration = nullptr;

	std::vector<AttributeMetadata> currentReaderMetadata;

	std::vector<writer::AttributeMetadata> currentWriterMetadata;

	std::map<string_t, string_t> currenVariablesMapping;

	integer_t currentAttributeIndex = 0;

	void createPipe(int& readerFD, int& writerFD) {

		int fds[2];

		int result = pipe(fds);

		readerFD = fds[0];

		writerFD = fds[1];

		if (result < 0) throw AwkException(L"Unable to create a pipe.");

	}

	void redirectFD(int oldfd, int newfd) {

		int result = dup2(oldfd, newfd);

		if (result < 0) throw AwkException(L"Unable redirect FD.");

	}

	void closeOrThrow(int fd) {

		int error = close(fd);

		if (error) throw AwkException(L"Unable to close FD: " + to_wstring(fd) + L" from PID: " + to_wstring(getpid()));

	}

	void execp(const std::vector<std::string>& args) {

		const char** a = new const char*[args.size() + 1];

		for (size_t i = 0; i < args.size(); i++) a[i] = args[i].c_str();

		a[args.size()] = nullptr;

		execvp(a[0], (char*const*) a);

		delete[] a;

		throw AwkException(L"Unable to do execvp().");

	}

	/* TODO: move to lib-cli when stable and used in other modules */

	void setEnv(const char * name, const string_t& value) {

		setenv(name, convertor.to_bytes(value).c_str(), true);

	}

	/* TODO: move to lib-cli when stable and used in other modules */

	void setEnv(const char * name, std::string& value) {

		setenv(name, value.c_str(), true);

	}

	/* TODO: move to lib-cli when stable and used in other modules */

	void setEnv(const char * name, const integer_t& value) {

		setenv(name, to_string(value).c_str(), true);

	}

	void addDefinition(std::vector<std::string>& awkCliArgs, DefinitionRecipe& d) {

		awkCliArgs.push_back("-v");

		awkCliArgs.push_back(convertor.to_bytes(a2v(d.name) + L"=" + d.value));

	}

	void add(vector<AttributeMetadata>& readerAttributes, vector<writer::AttributeMetadata>& writerAttributes) {

		for (AttributeMetadata readerAttributes : readerAttributes)

			writerAttributes.push_back({

				readerAttributes.getAttributeName(),

				relationalWriter->toTypeId(readerAttributes.getTypeName())

			});

	}

	void cleanUp() {

		if (awkInputWriterFD >= 0) {

			closeOrThrow(awkInputWriterFD);

			int error1;

			int error2;

			__pid_t waitPID1 = wait(&error1);

			__pid_t waitPID2 = wait(&error2);

			if (error1 || error2) throw AwkException(L"The child process returned an error exit code.");

			awkInputWriterFD = -1;

		}

		currentAttributeIndex = 0;

		currentReaderMetadata.clear();

		currentWriterMetadata.clear();

		currenVariablesMapping.clear();

		currentRelationConfiguration = nullptr;

	}

	void generateVariableMappings() {

		for (AttributeMetadata m : currentReaderMetadata) currenVariablesMapping[m.getAttributeName()] = L"";

		for (writer::AttributeMetadata m : currentWriterMetadata) currenVariablesMapping[m.attributeName] = L"";

		for (DefinitionRecipe d : configuration.definitions) currenVariablesMapping[d.name] = L"";

		for (DefinitionRecipe d : currentRelationConfiguration->definitions) currenVariablesMapping[d.name] = L"";

		for (std::pair<string_t, string_t> m : currenVariablesMapping) {

			currenVariablesMapping[m.first] = escapeAwkVariableName(m.first);

		}

	}

	string_t a2v(const string_t& attributeName) {

		if (currenVariablesMapping.find(attributeName) != currenVariablesMapping.end()) return currenVariablesMapping[attributeName];

		else throw AwkException(L"Unable to find value in currenVariablesMapping");

	}

	template <typename K, typename V> bool containsValue(std::map<K, V> map, V value) {

		for (std::pair<K, V> p : map) if (p.second == value) return true;

		return false;

	}

	string_t escapeAwkVariableName(const string_t& attributeName) {

		// cat awkgram.y | awk -v FS='\\{"|",' -v ORS='|' '/static const struct token tokentab/, /\};/ { if (/^\{/) { print $2} }'

		// cat AwkHandler.h | awk -v FS=' |\\(' -v ORS='|' '/awkScript.*"function/ { print $4; }'

		std::wregex awkReservedNames(L"FS|OFS|NR|NF|" L"BEGIN|BEGINFILE|END|ENDFILE|adump|and|asort|asorti|atan2|bindtextdomain|break|case|close|compl|continue|cos|dcgettext|dcngettext|default|delete|do|else|eval|exit|exp|fflush|for|func|function|gensub|getline|gsub|if|in|include|index|int|intdiv0|isarray|length|load|log|lshift|match|mktime|namespace|next|nextfile|or|patsplit|print|printf|rand|return|rshift|sin|split|sprintf|sqrt|srand|stopme|strftime|strtonum|sub|substr|switch|system|systime|tolower|toupper|typeof|while|xor");

		std::wregex trReservedNames(L"_escape|_unescape|_readVariables|_writeVariables|record");

		std::wregex badCharacters(L"[^a-zA-Z0-9_]|^([0-9])");

		const string_t& name = std::regex_replace(attributeName, badCharacters, L"_$1");

		bool badName = false;

		badName |= regex_match(name, awkReservedNames);

		badName |= regex_match(name, trReservedNames);

		badName |= containsValue(currenVariablesMapping, name);

		if (badName) return escapeAwkVariableName(L"_" + name);

		else return name;

	}

	string_t escapeAwkValue(const string_t& value) {

		std::wstringstream escaped;

		for (wchar_t ch : value) {

			if (ch == '\t') escaped << L"\\t";

			else if (ch == '\n') escaped << L"\\n";

			else if (ch == '\\') escaped << L"\\\\";

			else escaped << ch;

		}

		return escaped.str();

	}

	void processAwkOutput(int awkOutputReaderFD) {

		locale::global(locale("")); // needed for processing unicode texts, otherwise getline() stopped working on first line with non-ascii characters; TODO: move somewhere else?

		__gnu_cxx::stdio_filebuf<wchar_t> awkOutputReaderBuffer(awkOutputReaderFD, std::ios::in);

		std::wistream awkOutputReader(&awkOutputReaderBuffer);

		if (currentRelationConfiguration->drop) {

			for (wchar_t ch; awkOutputReader.read(&ch, 1).good();); // just eat the lines from the AWK

		} else {

			std::wstringstream currentValue;

			for (wchar_t ch; awkOutputReader.read(&ch, 1).good();) {

				if (ch == '\t' || ch == '\n') {

					relationalWriter->writeAttribute(currentValue.str());

					currentValue.str(L"");

					currentValue.clear();

				} else if (ch == '\\') {

					ch = awkOutputReader.get();

					if (ch == 't') currentValue << L'\t';

					else if (ch == 'n') currentValue << L'\n';

					else if (ch == '\\') currentValue << L'\\';

					else throw AwkException(L"Unknown escape sequence. Only \\t, \\n and \\\\ are supported.");

				} else {

					currentValue << ch;

				}

			}

		}

		closeOrThrow(awkOutputReaderFD);

	}

	void debugVariableMapping(const string_t& relationName) {

		relationalWriter->startRelation(relationName + L".variableMapping",{

			{L"attribute", writer::TypeId::STRING},

			{L"variable", writer::TypeId::STRING},

		}, true);

		for (std::pair<string_t, string_t> m : currenVariablesMapping) {

			relationalWriter->writeAttribute(m.first);

			relationalWriter->writeAttribute(m.second);

		}

	}

public:

	/**

	 * @param relationalWriter

	 * @param relationalWriterFlush the writer must be flushed before fork() in order to 

	 * avoid duplicate output (otherwise single relation might be written from two processes); 

	 * This is a little hack – if it stops working, we should reconnect the pipes 

	 * and use the writer only from a single process and avoid its effective duplication,

	 * or use different writers for each relation (or process).

	 * @param configuration

	 */

	AwkHandler(writer::RelationalWriter* relationalWriter, std::function<void() > relationalWriterFlush, Configuration& configuration) : relationalWriter(relationalWriter), relationalWriterFlush(relationalWriterFlush), configuration(configuration) {

	}

	void startRelation(string_t name, vector<AttributeMetadata> attributes) override {

		cleanUp();

		for (int i = 0; i < configuration.relationConfigurations.size(); i++) {

			if (regex_match(name, wregex(configuration.relationConfigurations[i].relation))) {

				currentRelationConfiguration = &configuration.relationConfigurations[i];

				break; // it there are multiple matches, only the first configuration is used

			}

		}

		currentReaderMetadata = attributes;

		// TODO: move to a reusable method (or use same metadata on both reader and writer side?)		

		if (currentRelationConfiguration && currentRelationConfiguration->writerMetadata.size()) {

			if (currentRelationConfiguration->inputAttributesPrepend) add(currentReaderMetadata, currentWriterMetadata);

			currentWriterMetadata.insert(currentWriterMetadata.end(), currentRelationConfiguration->writerMetadata.begin(), currentRelationConfiguration->writerMetadata.end());

			if (currentRelationConfiguration->inputAttributesAppend) add(currentReaderMetadata, currentWriterMetadata);

		} else {

			add(currentReaderMetadata, currentWriterMetadata);

		}

		if (currentRelationConfiguration) {

			generateVariableMappings();

			int awkInputReaderFD;

			int awkOutputReaderFD;

			int awkOutputWriterFD;

			createPipe(awkInputReaderFD, awkInputWriterFD);

			createPipe(awkOutputReaderFD, awkOutputWriterFD);

			relationalWriterFlush();

			__pid_t awkPid = fork();

			if (awkPid < 0) {

				throw AwkException(L"Unable to fork AWK process.");

			} else if (awkPid == 0) {

				// AWK child process

				closeOrThrow(awkInputWriterFD);

				closeOrThrow(awkOutputReaderFD);

				redirectFD(awkInputReaderFD, STDIN_FILENO);

				redirectFD(awkOutputWriterFD, STDOUT_FILENO);

				// Environment variables:

				setEnv("relationName", name);

				setEnv("inputAttributeCount", currentReaderMetadata.size());

				setEnv("outputAttributeCount", currentWriterMetadata.size());

				// TODO: better names? ENV vs. AWK variables?

				for (int i = 0; i < currentReaderMetadata.size(); i++) {

					setEnv((std::string("inputAttributeName") + std::to_string(i)).c_str(), currentReaderMetadata[i].getAttributeName());

					setEnv("inputAttributeType" + i, currentReaderMetadata[i].getTypeName());

				}

				for (int i = 0; i < currentWriterMetadata.size(); i++) {

					setEnv("outputAttributeName" + i, currentWriterMetadata[i].attributeName);

					// setEnv("outputAttributeType" + i, currentWriterMetadata[i].typeId); // TODO: type?

				}

				// AWK script:

				std::wstringstream awkScript;

				awkScript << L"BEGIN {" << std::endl;

				awkScript << L"FS=\"\\t\";" << std::endl;

				awkScript << L"OFS=\"\\t\";" << std::endl;

				awkScript << currentRelationConfiguration->awkBeforeRecords << std::endl;

				awkScript << L"};" << std::endl;

				awkScript << std::endl;

				awkScript << L"END {" << std::endl;

				awkScript << currentRelationConfiguration->awkAfterRecords << std::endl;

				awkScript << L"};" << std::endl;

				awkScript << LR"AWK(

function _escape(value,    i) {

	result = "";

	split(value, chars, "");

	for (i = 1; i <= length(chars); i++) {

		ch = chars[i];

		if (ch == "\\")      { ch = "\\\\"; }

		else if (ch == "\t") { ch = "\\t"; }

		else if (ch == "\n") { ch = "\\n"; }

		result = result ch;

	}

	return result;

};

function _unescape(value,    i) {

	result = "";

	split(value, chars, "");

	for (i = 1; i <= length(chars); i++) {

		ch = chars[i];

		if (ch == "\\") {

			ch = chars[++i];

			if (ch == "\\")     { ch = "\\"; }

			else if (ch == "t") { ch = "\t"; }

			else if (ch == "n") { ch = "\n"; }

			else {

				printf("Unsupported escape sequence: %s\n", ch) > "/dev/stderr";

				exit 70;

			}

		}

		result = result ch;

	}

	return result;

};

)AWK";

				awkScript << std::endl;

				awkScript << L"function _readVariables() {" << std::endl;

				for (int i = 0; i < currentReaderMetadata.size(); i++) {

					AttributeMetadata& a = currentReaderMetadata[i];

					awkScript << a2v(a.getAttributeName());

					if (a.getTypeId() == TypeId::INTEGER) awkScript << L"=$" << (i + 1) << L";";

					else if (a.getTypeId() == TypeId::BOOLEAN) awkScript << L"= $" << (i + 1) << L" == \"true\";";

					else awkScript << L"=_unescape($" << (i + 1) << L");";

					awkScript << std::endl;

				}

				awkScript << L"};" << std::endl;

				awkScript << std::endl;

				awkScript << L"function _writeVariables() {" << std::endl;

				awkScript << L"NF=" << currentWriterMetadata.size() << ";" << std::endl;

				for (int i = 0; i < currentWriterMetadata.size(); i++) {

					writer::AttributeMetadata& a = currentWriterMetadata[i];

					awkScript << L"$" << (i + 1);

					if (a.typeId == writer::TypeId::INTEGER) awkScript << L"=" << a2v(a.attributeName) << L";";

					else if (a.typeId == writer::TypeId::BOOLEAN) awkScript << L"= " << a2v(a.attributeName) << L" ? \"true\" : \"false\" ;";

					else awkScript << L"=_escape(" << a2v(a.attributeName) << L");";

					awkScript << std::endl;

				}

				awkScript << L"};" << std::endl;

				awkScript << std::endl;

				awkScript << L"function record() {" << std::endl;

				awkScript << L"_writeVariables();" << std::endl;

				awkScript << L"print;" << std::endl;

				awkScript << L"};" << std::endl;

				awkScript << std::endl;

				awkScript << L"{ _readVariables();  }" << std::endl; // read line (input attributes) into AWK variables

				awkScript << L"{ _writeVariables(); }" << std::endl; // write AWK variables to the line (so it matches the output attributes and can be implicitly printed without explicit record() call)

				awkScript << std::endl;

				awkScript << currentRelationConfiguration->awkForEach << std::endl; // user's code – can modify variables, filter results or explicitly call record() (can generate additional records or duplicate them)

				// CLI arguments:

				std::vector<std::string> args;

				args.push_back("awk");

				for (auto d : configuration.definitions) addDefinition(args, d);

				for (auto d : currentRelationConfiguration->definitions) addDefinition(args, d);

				args.push_back(convertor.to_bytes(awkScript.str()));

				// Runs AWK program found on $PATH → user can plug-in a custom implementation or a wrapper, but this can be also bit dangerous (however AWK itself is dangerous).

				execp(args);

			} else {

				// Parent process

				closeOrThrow(awkInputReaderFD);

				closeOrThrow(awkOutputWriterFD);

				__pid_t writerPid = fork();

				if (writerPid < 0) {

					throw AwkException(L"Unable to fork Writer process.");

				} else if (writerPid == 0) {

					// Writer child process

					closeOrThrow(awkInputWriterFD);

					if (currentRelationConfiguration->debugVariableMapping) debugVariableMapping(name);

					if (currentRelationConfiguration->drop) {

						// TODO: omit whole this process and pipe AWK output to /dev/null?

					} else {

						relationalWriter->startRelation(name, currentWriterMetadata, true);

					}

					processAwkOutput(awkOutputReaderFD);

					exit(0);

				} else {

					// Parent process

					closeOrThrow(awkOutputReaderFD);

				}

			}

		} else {

			relationalWriter->startRelation(name, currentWriterMetadata, true);

		}

	}

	void attribute(const string_t& value) override {

		if (currentRelationConfiguration) {

			currentAttributeIndex++;

			currentAttributeIndex = currentAttributeIndex % currentReaderMetadata.size();

			std::string awkValue = convertor.to_bytes(escapeAwkValue(value));

			if (currentAttributeIndex == 0) awkValue += "\n";

			else awkValue += "\t";

			write(awkInputWriterFD, awkValue.c_str(), awkValue.length());

		} else {

			relationalWriter->writeAttribute(value);

		}

	}

	void endOfPipe() {

		cleanUp();

	}

};

}

}

}