parallel processing: refactoring: ParallelFilesystemWorker inherits FilesystemWorker
--- a/src/FilesystemCommand.h Sat Jan 25 16:37:20 2020 +0100
+++ b/src/FilesystemCommand.h Sat Jan 25 21:28:37 2020 +0100
@@ -39,6 +39,7 @@
#include "FileAttributeFinder.h"
#include "XattrAttributeFinder.h"
#include "StreamletAttributeFinder.h"
+#include "FilesystemWorker.h"
namespace relpipe {
namespace in {
@@ -47,65 +48,7 @@
namespace fs = std::filesystem;
using namespace relpipe::writer;
-class FilesystemCommand {
-protected:
- std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings.
-
- std::map<string_t, std::shared_ptr<AttributeFinder>> createAttributeFinders() {
- return {
- {RequestedField::GROUP_FILE, std::make_shared<FileAttributeFinder>()},
- {RequestedField::GROUP_STREAMLET, std::make_shared<StreamletAttributeFinder>()},
- {RequestedField::GROUP_XATTR, std::make_shared<XattrAttributeFinder>()}};
- }
-
- void reset(std::stringstream& stream) {
- stream.str("");
- stream.clear();
- }
-
- bool readNext(std::istream& input, std::stringstream& originalName) {
- for (char ch; input.get(ch);) {
- if (ch == 0) return true;
- else originalName << ch;
- }
- return originalName.tellp();
- }
-
- string_t fetchRelationName(Configuration* configuration) {
- return configuration->relation.empty() ? L"filesystem" : configuration->relation;
- }
-
- void writeHeader(RelationalWriter* writer, std::map<string_t, std::shared_ptr < AttributeFinder>> attributeFinders, string_t relationName, std::vector<RequestedField>* fields, bool writeHeader = true) {
- std::vector<AttributeMetadata> attributesMetadata;
- for (RequestedField field : *fields) {
- std::shared_ptr<AttributeFinder> finder = attributeFinders[field.group];
- if (finder) for (AttributeMetadata m : finder->toMetadata(writer, relationName, field)) attributesMetadata.push_back(m);
- else throw RelpipeWriterException(L"Unsupported field group: " + field.group);
- }
-
- writer->startRelation(relationName, attributesMetadata, writeHeader);
- }
-
- void processSingleFile(std::shared_ptr<RelationalWriter> writer, std::stringstream& originalName, std::map<string_t, std::shared_ptr < AttributeFinder>>&attributeFinders, Configuration& configuration, string_t relationName) {
- fs::path file(originalName.str().empty() ? "." : originalName.str()); // interpret empty string as current directory (e.g. result of: find -printf '%P\0')
- bool exists = false;
-
- try {
- exists = fs::exists(file);
- } catch (const fs::filesystem_error& e) {
- // we probably do not have permissions to given directory → pretend that the file does not exist
- }
-
- for (auto& finder : attributeFinders) finder.second->startFile(file, originalName.str(), exists);
-
- for (RequestedField field : configuration.fields) {
- std::shared_ptr<AttributeFinder> finder = attributeFinders[field.group]; // should not be nullptr, because already checked while writing the relation metadata
- finder->writeField(writer.get(), relationName, field);
- }
-
- for (auto& finder : attributeFinders) finder.second->endFile();
- }
-
+class FilesystemCommand : public FilesystemWorker {
public:
virtual ~FilesystemCommand() = default;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/FilesystemWorker.h Sat Jan 25 21:28:37 2020 +0100
@@ -0,0 +1,116 @@
+/**
+ * Relational pipes
+ * Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <cstdlib>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+#include <map>
+#include <memory>
+#include <algorithm>
+#include <filesystem>
+
+#include <pwd.h>
+#include <grp.h>
+#include <sys/stat.h>
+
+#include <sys/xattr.h>
+
+#include <relpipe/writer/typedefs.h>
+
+#include "Configuration.h"
+#include "AttributeFinder.h"
+#include "FileAttributeFinder.h"
+#include "XattrAttributeFinder.h"
+#include "StreamletAttributeFinder.h"
+
+namespace relpipe {
+namespace in {
+namespace filesystem {
+
+namespace fs = std::filesystem;
+using namespace relpipe::writer;
+
+class FilesystemWorker {
+protected:
+ std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings.
+
+ std::map<string_t, std::shared_ptr<AttributeFinder>> createAttributeFinders() {
+ return {
+ {RequestedField::GROUP_FILE, std::make_shared<FileAttributeFinder>()},
+ {RequestedField::GROUP_STREAMLET, std::make_shared<StreamletAttributeFinder>()},
+ {RequestedField::GROUP_XATTR, std::make_shared<XattrAttributeFinder>()}};
+ }
+
+ void reset(std::stringstream& stream) {
+ stream.str("");
+ stream.clear();
+ }
+
+ bool readNext(std::istream& input, std::stringstream& originalName) {
+ for (char ch; input.get(ch);) {
+ if (ch == 0) return true;
+ else originalName << ch;
+ }
+ return originalName.tellp();
+ }
+
+ string_t fetchRelationName(Configuration* configuration) {
+ return configuration->relation.empty() ? L"filesystem" : configuration->relation;
+ }
+
+ void writeHeader(RelationalWriter* writer, std::map<string_t, std::shared_ptr < AttributeFinder>> attributeFinders, string_t relationName, std::vector<RequestedField>* fields, bool writeHeader = true) {
+ std::vector<AttributeMetadata> attributesMetadata;
+ for (RequestedField field : *fields) {
+ std::shared_ptr<AttributeFinder> finder = attributeFinders[field.group];
+ if (finder) for (AttributeMetadata m : finder->toMetadata(writer, relationName, field)) attributesMetadata.push_back(m);
+ else throw RelpipeWriterException(L"Unsupported field group: " + field.group);
+ }
+
+ writer->startRelation(relationName, attributesMetadata, writeHeader);
+ }
+
+ void processSingleFile(std::shared_ptr<RelationalWriter> writer, std::stringstream& originalName, std::map<string_t, std::shared_ptr < AttributeFinder>>&attributeFinders, Configuration& configuration, string_t relationName) {
+ fs::path file(originalName.str().empty() ? "." : originalName.str()); // interpret empty string as current directory (e.g. result of: find -printf '%P\0')
+ bool exists = false;
+
+ try {
+ exists = fs::exists(file);
+ } catch (const fs::filesystem_error& e) {
+ // we probably do not have permissions to given directory → pretend that the file does not exist
+ }
+
+ for (auto& finder : attributeFinders) finder.second->startFile(file, originalName.str(), exists);
+
+ for (RequestedField field : configuration.fields) {
+ std::shared_ptr<AttributeFinder> finder = attributeFinders[field.group]; // should not be nullptr, because already checked while writing the relation metadata
+ finder->writeField(writer.get(), relationName, field);
+ }
+
+ for (auto& finder : attributeFinders) finder.second->endFile();
+ }
+
+public:
+
+ virtual ~FilesystemWorker() = default;
+};
+
+}
+}
+}
--- a/src/ParallelFilesystemCommand.h Sat Jan 25 16:37:20 2020 +0100
+++ b/src/ParallelFilesystemCommand.h Sat Jan 25 21:28:37 2020 +0100
@@ -187,7 +187,7 @@
}
};
-class ParallelFilesystemWorker : FilesystemCommand {
+class ParallelFilesystemWorker : FilesystemWorker {
private:
std::string queueName;
NamedMutex& stdoutMutex;
@@ -230,11 +230,6 @@
}
}
-
- void process(int inputFD, int outputFD, Configuration& configuration) override {
- // FIXME: refactoring, not used
- }
-
};
class ParallelFilesystemProcess {
--- a/src/StreamletAttributeFinder.h Sat Jan 25 16:37:20 2020 +0100
+++ b/src/StreamletAttributeFinder.h Sat Jan 25 21:28:37 2020 +0100
@@ -74,7 +74,7 @@
if (streamletPathChars) {
for (string_t path : splitBySeparator(convertor.from_bytes(streamletPathChars))) {
fs::path file = fs::path(path) / fs::path(field.name);
- if (fs::exists(file) && ::access(file.c_str(), X_OK) == 0) return file.wstring(); // n.b. must be set executable using e.g. chmod – files executable through only ACL, are actually not executable
+ if (fs::exists(file) && ::access(file.c_str(), X_OK) == 0) return file.wstring(); // n.b. must be set executable using e.g. chmod – files executable through only ACL, are actually not executable
}
throw RelpipeWriterException(L"Streamlet „" + field.name + L"“ was not found at $RELPIPE_IN_FILESYSTEM_STREAMLET_PATH");
} else {
@@ -104,7 +104,6 @@
SubProcess::Message m = subProcesses[field.id]->read();
if (m.code != StreamletMsg::WAITING_FOR_INPUT_ATTRIBUTES) throw RelpipeWriterException(L"Protocol violation from exec sub-process. Expected WAITING_FOR_INPUT_ATTRIBUTES but got: " + m.toString());
- // FIXME: generic protocol violation error messages / method for checking responses
}
}