/**
* Relational pipes
* Copyright © 2018 František Kučera (Frantovo.cz, GlobalCode.info)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <memory>
#include <string>
#include <vector>
#include <iostream>
#include <sstream>
#include <locale>
#include <codecvt>
#include <regex>
#include <Python.h>
#include <relpipe/reader/typedefs.h>
#include <relpipe/reader/TypeId.h>
#include <relpipe/reader/handlers/RelationalReaderStringHandler.h>
#include <relpipe/reader/handlers/AttributeMetadata.h>
#include <relpipe/writer/Factory.h>
#include <relpipe/cli/RelpipeCLIException.h>
namespace relpipe {
namespace tr {
namespace python {
using namespace std;
using namespace relpipe;
using namespace relpipe::reader;
using namespace relpipe::reader::handlers;
class PythonHandler : public RelationalReaderStringHadler {
private:
const wregex ATTRIBUTE_NAMES_ALLOWED = wregex(L"[a-zA-Z0-9_]+");
const wregex ATTRIBUTE_NAMES_DISALLOWED = wregex(L"WHERE|[0-9_].*|False|None|True|and|as|assert|break|class|continue|def|del|elif|else|except|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|raise|return|try|while|with|yield"); // Python keywords from: import keyword; keyword.kwlist
shared_ptr<writer::RelationalWriter> relationalWriter;
wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings.
wchar_t* pythonProgramName;
wregex relationNameRegEx;
string_t pythonCode;
vector<writer::AttributeMetadata> currentWriterMetadata;
vector<string_t> currentRecord;
integer_t currentAttributeIndex = 0;
boolean_t includeCurrentRecord = true;
boolean_t filterCurrentRelation = false;
public:
PythonHandler(ostream& output, const vector<string_t>& arguments) {
relationalWriter.reset(writer::Factory::create(output));
pythonProgramName = Py_DecodeLocale("relpipe-tr-python", NULL);
Py_SetProgramName(pythonProgramName);
Py_Initialize();
if (arguments.size() == 2) {
relationNameRegEx = wregex(arguments[0]);
pythonCode = arguments[1];
// TODO: allow relation structure changes: if there are more arguments, the describe the output relation
// TODO: allow also other modes:
// - insert additional records
// - process whole relation at once (can do e.g. some aggregations)
// - process whole stream at once (can do e.g. some joins or unions)
} else {
PyMem_RawFree(pythonProgramName);
throw cli::RelpipeCLIException(L"Usage: relpipe-tr-python <relationNameRegExp> <pythonCode>", cli::CLI::EXIT_CODE_UNKNOWN_COMMAND);
}
}
virtual ~PythonHandler() {
Py_FinalizeEx();
PyMem_RawFree(pythonProgramName);
}
void startRelation(string_t name, vector<AttributeMetadata> attributes) override {
// TODO: move to a reusable method (or use same metadata on both reader and writer side?)
currentWriterMetadata.clear();
for (AttributeMetadata readerMetadata : attributes) {
currentWriterMetadata.push_back({readerMetadata.getAttributeName(), relationalWriter->toTypeId(readerMetadata.getTypeName())});
}
currentRecord.resize(attributes.size());
filterCurrentRelation = regex_match(name, relationNameRegEx);
relationalWriter->startRelation(name, currentWriterMetadata, true);
}
void attribute(const string_t& value) override {
if (filterCurrentRelation) {
currentRecord[currentAttributeIndex] = value;
currentAttributeIndex++;
if (currentAttributeIndex > 0 && currentAttributeIndex % currentRecord.size() == 0) {
PyObject* pyModule;
PyObject* pyDict;
PyObject* pyWhere;
PyObject* pyRecord = PyList_New(currentRecord.size());
//PyUnicode_FromWideChar()
pyModule = PyImport_AddModule((char*) "__main__"); // FIXME: variable and Py_DecodeLocale ?
pyDict = PyModule_GetDict(pyModule);
for (int i = 0; i < currentRecord.size(); i++) {
// TODO: pass particular data-types to Python, not only strings
PyObject* pyValue = PyUnicode_FromString(convertor.to_bytes(currentRecord[i]).c_str());
PyList_SetItem(pyRecord, i, pyValue);
string_t attributeName = currentWriterMetadata[i].attributeName;
if (regex_match(attributeName, ATTRIBUTE_NAMES_ALLOWED)) {
if (regex_match(attributeName, ATTRIBUTE_NAMES_DISALLOWED)) attributeName = L"a_" + attributeName;
PyDict_SetItemString(pyDict, convertor.to_bytes(attributeName).c_str(), pyValue);
}
}
PyDict_SetItemString(pyDict, "r", pyRecord);
PyRun_SimpleString(convertor.to_bytes(pythonCode).c_str());
// FIXME: check Python error and throw exception
// if (PyErr_Occurred()) throw cli::RelpipeCLIException(L"Python code failed.", cli::CLI::EXIT_CODE_UNEXPECTED_ERROR); // TODO: review exit code
// if (PyErr_Occurred()) { PyErr_Print(); PyErr_Clear(); }
pyWhere = PyDict_GetItemString(pyDict, "WHERE");
if (pyWhere) includeCurrentRecord = PyLong_AsLong(pyWhere);
if (includeCurrentRecord) {
for (int i = 0; i < currentRecord.size(); i++) {
Py_ssize_t l;
currentRecord[i] = wstring(PyUnicode_AsWideCharString(PyList_GetItem(pyRecord, i), &l));
}
}
if (includeCurrentRecord) for (string_t v : currentRecord) relationalWriter->writeAttribute(v);
includeCurrentRecord = true;
}
currentAttributeIndex = currentAttributeIndex % currentRecord.size();
} else {
relationalWriter->writeAttribute(value);
}
}
void endOfPipe() {
}
};
}
}
}