src/AwkHandler.h
author František Kučera <franta-hg@frantovo.cz>
Mon, 06 May 2019 21:57:16 +0200
branchv_0
changeset 12 8844ebce8fb4
parent 11 f515d14794e0
child 13 b74001992ec3
permissions -rw-r--r--
pass defined global variables to AWK as CLI arguments
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     1
/**
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     2
 * Relational pipes
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     3
 * Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     4
 *
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     5
 * This program is free software: you can redistribute it and/or modify
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     6
 * it under the terms of the GNU General Public License as published by
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     7
 * the Free Software Foundation, either version 3 of the License, or
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     8
 * (at your option) any later version.
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     9
 *
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    10
 * This program is distributed in the hope that it will be useful,
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    11
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    12
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    13
 * GNU General Public License for more details.
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    14
 *
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    15
 * You should have received a copy of the GNU General Public License
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    16
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    17
 */
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    18
#pragma once
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    19
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    20
#include <memory>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    21
#include <string>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    22
#include <vector>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    23
#include <iostream>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    24
#include <sstream>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    25
#include <locale>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    26
#include <codecvt>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    27
#include <regex>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    28
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    29
#include <unistd.h>
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    30
#include <wait.h>
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    31
#include <ext/stdio_filebuf.h>
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    32
0
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    33
#include <relpipe/reader/typedefs.h>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    34
#include <relpipe/reader/TypeId.h>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    35
#include <relpipe/reader/handlers/RelationalReaderStringHandler.h>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    36
#include <relpipe/reader/handlers/AttributeMetadata.h>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    37
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    38
#include <relpipe/writer/Factory.h>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    39
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    40
#include <relpipe/cli/RelpipeCLIException.h>
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    41
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    42
#include "Configuration.h"
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    43
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    44
namespace relpipe {
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    45
namespace tr {
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    46
namespace awk {
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    47
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    48
using namespace std;
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    49
using namespace relpipe;
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    50
using namespace relpipe::reader;
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    51
using namespace relpipe::reader::handlers;
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    52
5
86de8e6ab231 documentation
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
    53
/**
86de8e6ab231 documentation
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
    54
 * This transformation consists of three processes connected together using pipes.
86de8e6ab231 documentation
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
    55
 * 
86de8e6ab231 documentation
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
    56
 * Processes:
86de8e6ab231 documentation
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
    57
 *	- Parent: process RelationalReaderStringHandler events (read relational data, usually from STDIN) and pass them in the special text format to the pipe1
86de8e6ab231 documentation
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
    58
 *  - AWK: external program (/usr/bin/awk), read from the pipe1, writes to the pipe2
86de8e6ab231 documentation
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
    59
 *  - Writer: reads from the pipe2 and writes to relationalWriter (relational data, usually to STDOUT)
86de8e6ab231 documentation
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
    60
 */
0
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    61
class AwkHandler : public RelationalReaderStringHandler {
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    62
private:
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    63
	Configuration configuration;
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    64
	writer::RelationalWriter* relationalWriter;
7
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
    65
	std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings
0
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    66
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    67
	int awkInputWriterFD = -1;
7
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
    68
	std::vector<AttributeMetadata> currentReaderMetadata;
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
    69
	integer_t currentAttributeIndex = 0;
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    70
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    71
	void createPipe(int& readerFD, int& writerFD) {
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    72
		int fds[2];
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    73
		int result = pipe(fds);
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    74
		readerFD = fds[0];
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    75
		writerFD = fds[1];
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    76
		if (result < 0) throw cli::RelpipeCLIException(L"Unable to create a pipe.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    77
	}
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    78
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    79
	void redirectFD(int oldfd, int newfd) {
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    80
		int result = dup2(oldfd, newfd);
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    81
		if (result < 0) throw cli::RelpipeCLIException(L"Unable redirect FD.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    82
	}
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    83
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    84
	void closeOrThrow(int fd) {
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    85
		int error = close(fd);
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    86
		if (error) throw cli::RelpipeCLIException(L"Unable to close FD: " + to_wstring(fd) + L" from PID: " + to_wstring(getpid()), cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    87
	}
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
    88
11
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    89
	void execp(const std::vector<std::string>& args) {
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    90
		const char** a = new const char*[args.size() + 1];
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    91
		for (size_t i = 0; i < args.size(); i++) a[i] = args[i].c_str();
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    92
		a[args.size()] = nullptr;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    93
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    94
		execvp(a[0], (char*const*) a);
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    95
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    96
		delete[] a;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    97
		throw cli::RelpipeCLIException(L"Unable to do execvp().", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    98
	}
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
    99
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   100
	void cleanUp() {
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   101
		if (awkInputWriterFD >= 0) {
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   102
			closeOrThrow(awkInputWriterFD);
5
86de8e6ab231 documentation
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
   103
			// TODO: check exit codes
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   104
			__pid_t waitResult1 = wait(NULL);
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   105
			__pid_t waitResult2 = wait(NULL);
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   106
			awkInputWriterFD = -1;
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   107
		}
7
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   108
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   109
		currentAttributeIndex = 0;
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   110
		currentReaderMetadata.clear();
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   111
	}
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   112
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   113
	string_t a2v(const string_t& attributeName) {
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   114
		// FIXME: escape reserved names; prefix with _ ?
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   115
		// cat awkgram.y | awk -v FS='\\{"|",' -v ORS='|' '/static const struct token tokentab/, /\};/ { if (/^\{/) { print $2} }'
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   116
		// BEGIN|BEGINFILE|END|ENDFILE|adump|and|asort|asorti|atan2|bindtextdomain|break|case|close|compl|continue|cos|dcgettext|dcngettext|default|delete|do|else|eval|exit|exp|fflush|for|func|function|gensub|getline|gsub|if|in|include|index|int|intdiv0|isarray|length|load|log|lshift|match|mktime|namespace|next|nextfile|or|patsplit|print|printf|rand|return|rshift|sin|split|sprintf|sqrt|srand|stopme|strftime|strtonum|sub|substr|switch|system|systime|tolower|toupper|typeof|while|xor
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   117
		return attributeName;
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   118
	}
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   119
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   120
	string_t escapeAwkValue(const string_t& value) {
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   121
		// FIXME: escape field and record separators
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   122
		return value;
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   123
	}
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   124
0
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   125
public:
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   126
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   127
	AwkHandler(writer::RelationalWriter* relationalWriter, Configuration& configuration) : relationalWriter(relationalWriter), configuration(configuration) {
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   128
	}
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   129
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   130
	void startRelation(string_t name, vector<AttributeMetadata> attributes) override {
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   131
		cleanUp();
0
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   132
7
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   133
		currentReaderMetadata = attributes;
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   134
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   135
		int awkInputReaderFD;
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   136
		int awkOutputReaderFD;
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   137
		int awkOutputWriterFD;
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   138
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   139
		createPipe(awkInputReaderFD, awkInputWriterFD);
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   140
		createPipe(awkOutputReaderFD, awkOutputWriterFD);
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   141
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   142
		__pid_t awkPid = fork();
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   143
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   144
		if (awkPid < 0) {
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   145
			throw cli::RelpipeCLIException(L"Unable to fork AWK process.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   146
		} else if (awkPid == 0) {
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   147
			// AWK child process
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   148
			closeOrThrow(awkInputWriterFD);
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   149
			closeOrThrow(awkOutputReaderFD);
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   150
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   151
			redirectFD(awkInputReaderFD, STDIN_FILENO);
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   152
			redirectFD(awkOutputWriterFD, STDOUT_FILENO);
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   153
12
8844ebce8fb4 pass defined global variables to AWK as CLI arguments
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   154
			// AWK script:
11
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   155
			std::wstringstream awkScript;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   156
			awkScript << L"BEGIN {" << std::endl;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   157
			awkScript << L"FS=\"\\t\";" << std::endl;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   158
			awkScript << L"};" << std::endl;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   159
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   160
			awkScript << L"END {" << std::endl;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   161
			// awkScript << … << std::endl;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   162
			awkScript << L"};" << std::endl;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   163
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   164
			awkScript << L"{print \"AWK says: line \" NR \" '\" $0 \"' has \" NF \" fields; first field is '\" $1 \"'\";}" << std::endl;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   165
12
8844ebce8fb4 pass defined global variables to AWK as CLI arguments
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   166
			// CLI arguments:
11
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   167
			std::vector<std::string> args;
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   168
			args.push_back("awk");
12
8844ebce8fb4 pass defined global variables to AWK as CLI arguments
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   169
8844ebce8fb4 pass defined global variables to AWK as CLI arguments
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   170
			for (auto d : configuration.definitions) {
8844ebce8fb4 pass defined global variables to AWK as CLI arguments
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   171
				args.push_back("-v");
8844ebce8fb4 pass defined global variables to AWK as CLI arguments
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   172
				args.push_back(convertor.to_bytes(a2v(d.name) + L"=" + d.value));
8844ebce8fb4 pass defined global variables to AWK as CLI arguments
František Kučera <franta-hg@frantovo.cz>
parents: 11
diff changeset
   173
			}
11
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   174
			args.push_back(convertor.to_bytes(awkScript.str()));
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   175
5
86de8e6ab231 documentation
František Kučera <franta-hg@frantovo.cz>
parents: 4
diff changeset
   176
			// Runs AWK program found on $PATH → user can plug-in a custom implementation or a wrapper, but this can be also bit dangerous (however AWK itself is dangerous).
11
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   177
			execp(args);
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   178
		} else {
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   179
			// Parent process
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   180
			closeOrThrow(awkInputReaderFD);
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   181
			closeOrThrow(awkOutputWriterFD);
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   182
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   183
			__pid_t writerPid = fork();
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   184
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   185
			if (writerPid < 0) {
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   186
				throw cli::RelpipeCLIException(L"Unable to fork Writer process.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions?
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   187
			} else if (writerPid == 0) {
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   188
				// Writer child process
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   189
				closeOrThrow(awkInputWriterFD);
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   190
10
f911910fd68f support non-ascii texts
František Kučera <franta-hg@frantovo.cz>
parents: 7
diff changeset
   191
				locale::global(locale("")); // needed for processing unicode texts, otherwise getline() stopped working on first line with non-ascii characters; TODO: move somewhere else?
11
f515d14794e0 variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents: 10
diff changeset
   192
3
e086ae6a19c3 generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   193
				__gnu_cxx::stdio_filebuf<wchar_t> awkOutputReaderBuffer(awkOutputReaderFD, std::ios::in);
e086ae6a19c3 generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   194
				std::wistream awkOutputReader(&awkOutputReaderBuffer);
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   195
3
e086ae6a19c3 generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   196
				relationalWriter->startRelation(L"writer_debug",{
e086ae6a19c3 generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   197
					{L"message", writer::TypeId::STRING},
e086ae6a19c3 generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   198
				}, true);
e086ae6a19c3 generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents: 2
diff changeset
   199
6
efa96f51b308 process lines instead of characters
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
   200
				for (string_t line; getline(awkOutputReader, line).good();) {
efa96f51b308 process lines instead of characters
František Kučera <franta-hg@frantovo.cz>
parents: 5
diff changeset
   201
					relationalWriter->writeAttribute(line);
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   202
				}
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   203
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   204
				closeOrThrow(awkOutputReaderFD);
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   205
				exit(0);
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   206
			} else {
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   207
				// Parent process
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   208
				closeOrThrow(awkOutputReaderFD);
1
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   209
			}
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   210
		}
05d969cd90d4 fork() processes
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
   211
0
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   212
	}
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   213
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   214
	void attribute(const string_t& value) override {
7
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   215
		string_t variableName = a2v(currentReaderMetadata[currentAttributeIndex].getAttributeName());
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   216
		string_t variableValue = escapeAwkValue(value);
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   217
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   218
		currentAttributeIndex++;
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   219
		currentAttributeIndex = currentAttributeIndex % currentReaderMetadata.size();
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   220
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   221
		// TODO: just the value – move name to the AWK function
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   222
		std::string variablePair = convertor.to_bytes(variableName + L"=" + variableValue);
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   223
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   224
		if (currentAttributeIndex == 0) variablePair += "\n";
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   225
		else variablePair += "\t";
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   226
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   227
		write(awkInputWriterFD, variablePair.c_str(), variablePair.length());
46db0e6e548b pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents: 6
diff changeset
   228
0
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   229
	}
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   230
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   231
	void endOfPipe() {
2
580ccb511301 for each attribute
František Kučera <franta-hg@frantovo.cz>
parents: 1
diff changeset
   232
		cleanUp();
0
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   233
	}
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   234
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   235
};
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   236
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   237
}
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   238
}
644fd2ce2580 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   239
}