author | František Kučera <franta-hg@frantovo.cz> |
Mon, 06 May 2019 20:38:17 +0200 | |
branch | v_0 |
changeset 11 | f515d14794e0 |
parent 10 | f911910fd68f |
child 12 | 8844ebce8fb4 |
permissions | -rw-r--r-- |
0 | 1 |
/** |
2 |
* Relational pipes |
|
3 |
* Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info) |
|
4 |
* |
|
5 |
* This program is free software: you can redistribute it and/or modify |
|
6 |
* it under the terms of the GNU General Public License as published by |
|
7 |
* the Free Software Foundation, either version 3 of the License, or |
|
8 |
* (at your option) any later version. |
|
9 |
* |
|
10 |
* This program is distributed in the hope that it will be useful, |
|
11 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
12 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
13 |
* GNU General Public License for more details. |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License |
|
16 |
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
17 |
*/ |
|
18 |
#pragma once |
|
19 |
||
20 |
#include <memory> |
|
21 |
#include <string> |
|
22 |
#include <vector> |
|
23 |
#include <iostream> |
|
24 |
#include <sstream> |
|
25 |
#include <locale> |
|
26 |
#include <codecvt> |
|
27 |
#include <regex> |
|
28 |
||
1 | 29 |
#include <unistd.h> |
30 |
#include <wait.h> |
|
31 |
#include <ext/stdio_filebuf.h> |
|
32 |
||
0 | 33 |
#include <relpipe/reader/typedefs.h> |
34 |
#include <relpipe/reader/TypeId.h> |
|
35 |
#include <relpipe/reader/handlers/RelationalReaderStringHandler.h> |
|
36 |
#include <relpipe/reader/handlers/AttributeMetadata.h> |
|
37 |
||
38 |
#include <relpipe/writer/Factory.h> |
|
39 |
||
40 |
#include <relpipe/cli/RelpipeCLIException.h> |
|
41 |
||
42 |
#include "Configuration.h" |
|
43 |
||
44 |
namespace relpipe { |
|
45 |
namespace tr { |
|
46 |
namespace awk { |
|
47 |
||
48 |
using namespace std; |
|
49 |
using namespace relpipe; |
|
50 |
using namespace relpipe::reader; |
|
51 |
using namespace relpipe::reader::handlers; |
|
52 |
||
5 | 53 |
/** |
54 |
* This transformation consists of three processes connected together using pipes. |
|
55 |
* |
|
56 |
* Processes: |
|
57 |
* - Parent: process RelationalReaderStringHandler events (read relational data, usually from STDIN) and pass them in the special text format to the pipe1 |
|
58 |
* - AWK: external program (/usr/bin/awk), read from the pipe1, writes to the pipe2 |
|
59 |
* - Writer: reads from the pipe2 and writes to relationalWriter (relational data, usually to STDOUT) |
|
60 |
*/ |
|
0 | 61 |
class AwkHandler : public RelationalReaderStringHandler { |
62 |
private: |
|
63 |
Configuration configuration; |
|
64 |
writer::RelationalWriter* relationalWriter; |
|
7
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
65 |
std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings |
0 | 66 |
|
2 | 67 |
int awkInputWriterFD = -1; |
7
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
68 |
std::vector<AttributeMetadata> currentReaderMetadata; |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
69 |
integer_t currentAttributeIndex = 0; |
2 | 70 |
|
1 | 71 |
void createPipe(int& readerFD, int& writerFD) { |
72 |
int fds[2]; |
|
73 |
int result = pipe(fds); |
|
74 |
readerFD = fds[0]; |
|
75 |
writerFD = fds[1]; |
|
76 |
if (result < 0) throw cli::RelpipeCLIException(L"Unable to create a pipe.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions? |
|
77 |
} |
|
78 |
||
79 |
void redirectFD(int oldfd, int newfd) { |
|
80 |
int result = dup2(oldfd, newfd); |
|
81 |
if (result < 0) throw cli::RelpipeCLIException(L"Unable redirect FD.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions? |
|
82 |
} |
|
83 |
||
84 |
void closeOrThrow(int fd) { |
|
85 |
int error = close(fd); |
|
2 | 86 |
if (error) throw cli::RelpipeCLIException(L"Unable to close FD: " + to_wstring(fd) + L" from PID: " + to_wstring(getpid()), cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions? |
87 |
} |
|
88 |
||
11
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
89 |
void execp(const std::vector<std::string>& args) { |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
90 |
const char** a = new const char*[args.size() + 1]; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
91 |
for (size_t i = 0; i < args.size(); i++) a[i] = args[i].c_str(); |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
92 |
a[args.size()] = nullptr; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
93 |
|
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
94 |
execvp(a[0], (char*const*) a); |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
95 |
|
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
96 |
delete[] a; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
97 |
throw cli::RelpipeCLIException(L"Unable to do execvp().", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions? |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
98 |
} |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
99 |
|
2 | 100 |
void cleanUp() { |
101 |
if (awkInputWriterFD >= 0) { |
|
102 |
closeOrThrow(awkInputWriterFD); |
|
5 | 103 |
// TODO: check exit codes |
2 | 104 |
__pid_t waitResult1 = wait(NULL); |
105 |
__pid_t waitResult2 = wait(NULL); |
|
106 |
awkInputWriterFD = -1; |
|
107 |
} |
|
7
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
108 |
|
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
109 |
currentAttributeIndex = 0; |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
110 |
currentReaderMetadata.clear(); |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
111 |
} |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
112 |
|
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
113 |
string_t a2v(const string_t& attributeName) { |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
114 |
// FIXME: escape reserved names; prefix with _ ? |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
115 |
// cat awkgram.y | awk -v FS='\\{"|",' -v ORS='|' '/static const struct token tokentab/, /\};/ { if (/^\{/) { print $2} }' |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
116 |
// BEGIN|BEGINFILE|END|ENDFILE|adump|and|asort|asorti|atan2|bindtextdomain|break|case|close|compl|continue|cos|dcgettext|dcngettext|default|delete|do|else|eval|exit|exp|fflush|for|func|function|gensub|getline|gsub|if|in|include|index|int|intdiv0|isarray|length|load|log|lshift|match|mktime|namespace|next|nextfile|or|patsplit|print|printf|rand|return|rshift|sin|split|sprintf|sqrt|srand|stopme|strftime|strtonum|sub|substr|switch|system|systime|tolower|toupper|typeof|while|xor |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
117 |
return attributeName; |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
118 |
} |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
119 |
|
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
120 |
string_t escapeAwkValue(const string_t& value) { |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
121 |
// FIXME: escape field and record separators |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
122 |
return value; |
1 | 123 |
} |
124 |
||
0 | 125 |
public: |
126 |
||
127 |
AwkHandler(writer::RelationalWriter* relationalWriter, Configuration& configuration) : relationalWriter(relationalWriter), configuration(configuration) { |
|
128 |
} |
|
129 |
||
130 |
void startRelation(string_t name, vector<AttributeMetadata> attributes) override { |
|
2 | 131 |
cleanUp(); |
0 | 132 |
|
7
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
133 |
currentReaderMetadata = attributes; |
1 | 134 |
|
135 |
int awkInputReaderFD; |
|
136 |
int awkOutputReaderFD; |
|
137 |
int awkOutputWriterFD; |
|
138 |
||
139 |
createPipe(awkInputReaderFD, awkInputWriterFD); |
|
140 |
createPipe(awkOutputReaderFD, awkOutputWriterFD); |
|
141 |
||
142 |
__pid_t awkPid = fork(); |
|
143 |
||
144 |
if (awkPid < 0) { |
|
145 |
throw cli::RelpipeCLIException(L"Unable to fork AWK process.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions? |
|
146 |
} else if (awkPid == 0) { |
|
147 |
// AWK child process |
|
2 | 148 |
closeOrThrow(awkInputWriterFD); |
149 |
closeOrThrow(awkOutputReaderFD); |
|
1 | 150 |
|
2 | 151 |
redirectFD(awkInputReaderFD, STDIN_FILENO); |
152 |
redirectFD(awkOutputWriterFD, STDOUT_FILENO); |
|
1 | 153 |
|
11
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
154 |
std::wstringstream awkScript; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
155 |
awkScript << L"BEGIN {" << std::endl; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
156 |
awkScript << L"FS=\"\\t\";" << std::endl; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
157 |
awkScript << L"};" << std::endl; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
158 |
|
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
159 |
awkScript << L"END {" << std::endl; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
160 |
// awkScript << … << std::endl; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
161 |
awkScript << L"};" << std::endl; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
162 |
|
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
163 |
awkScript << L"{print \"AWK says: line \" NR \" '\" $0 \"' has \" NF \" fields; first field is '\" $1 \"'\";}" << std::endl; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
164 |
|
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
165 |
std::vector<std::string> args; |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
166 |
args.push_back("awk"); |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
167 |
args.push_back(convertor.to_bytes(awkScript.str())); |
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
168 |
|
5 | 169 |
// Runs AWK program found on $PATH → user can plug-in a custom implementation or a wrapper, but this can be also bit dangerous (however AWK itself is dangerous). |
11
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
170 |
execp(args); |
1 | 171 |
} else { |
172 |
// Parent process |
|
2 | 173 |
closeOrThrow(awkInputReaderFD); |
174 |
closeOrThrow(awkOutputWriterFD); |
|
1 | 175 |
|
176 |
__pid_t writerPid = fork(); |
|
177 |
||
178 |
if (writerPid < 0) { |
|
179 |
throw cli::RelpipeCLIException(L"Unable to fork Writer process.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: better exceptions? |
|
180 |
} else if (writerPid == 0) { |
|
181 |
// Writer child process |
|
2 | 182 |
closeOrThrow(awkInputWriterFD); |
1 | 183 |
|
10
f911910fd68f
support non-ascii texts
František Kučera <franta-hg@frantovo.cz>
parents:
7
diff
changeset
|
184 |
locale::global(locale("")); // needed for processing unicode texts, otherwise getline() stopped working on first line with non-ascii characters; TODO: move somewhere else? |
11
f515d14794e0
variable execvp() arguments
František Kučera <franta-hg@frantovo.cz>
parents:
10
diff
changeset
|
185 |
|
3
e086ae6a19c3
generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
186 |
__gnu_cxx::stdio_filebuf<wchar_t> awkOutputReaderBuffer(awkOutputReaderFD, std::ios::in); |
e086ae6a19c3
generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
187 |
std::wistream awkOutputReader(&awkOutputReaderBuffer); |
1 | 188 |
|
3
e086ae6a19c3
generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
189 |
relationalWriter->startRelation(L"writer_debug",{ |
e086ae6a19c3
generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
190 |
{L"message", writer::TypeId::STRING}, |
e086ae6a19c3
generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
191 |
}, true); |
e086ae6a19c3
generate (some) relational output
František Kučera <franta-hg@frantovo.cz>
parents:
2
diff
changeset
|
192 |
|
6
efa96f51b308
process lines instead of characters
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
193 |
for (string_t line; getline(awkOutputReader, line).good();) { |
efa96f51b308
process lines instead of characters
František Kučera <franta-hg@frantovo.cz>
parents:
5
diff
changeset
|
194 |
relationalWriter->writeAttribute(line); |
1 | 195 |
} |
196 |
||
2 | 197 |
closeOrThrow(awkOutputReaderFD); |
198 |
exit(0); |
|
1 | 199 |
} else { |
200 |
// Parent process |
|
2 | 201 |
closeOrThrow(awkOutputReaderFD); |
1 | 202 |
} |
203 |
} |
|
204 |
||
0 | 205 |
} |
206 |
||
207 |
void attribute(const string_t& value) override { |
|
7
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
208 |
string_t variableName = a2v(currentReaderMetadata[currentAttributeIndex].getAttributeName()); |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
209 |
string_t variableValue = escapeAwkValue(value); |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
210 |
|
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
211 |
currentAttributeIndex++; |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
212 |
currentAttributeIndex = currentAttributeIndex % currentReaderMetadata.size(); |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
213 |
|
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
214 |
// TODO: just the value – move name to the AWK function |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
215 |
std::string variablePair = convertor.to_bytes(variableName + L"=" + variableValue); |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
216 |
|
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
217 |
if (currentAttributeIndex == 0) variablePair += "\n"; |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
218 |
else variablePair += "\t"; |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
219 |
|
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
220 |
write(awkInputWriterFD, variablePair.c_str(), variablePair.length()); |
46db0e6e548b
pass name-value pair to AWK
František Kučera <franta-hg@frantovo.cz>
parents:
6
diff
changeset
|
221 |
|
0 | 222 |
} |
223 |
||
224 |
void endOfPipe() { |
|
2 | 225 |
cleanUp(); |
0 | 226 |
} |
227 |
||
228 |
}; |
|
229 |
||
230 |
} |
|
231 |
} |
|
232 |
} |