# HG changeset patch
# User František Kučera <franta-hg@frantovo.cz>
# Date 1579984117 -3600
# Node ID 640ba8948d69607e307c9bcc4762b6dfc0c64cdd
# Parent  bb7ca589175570e25674f06a35c3c97aa1e5d756
parallel processing: refactoring: ParallelFilesystemWorker inherits FilesystemWorker

diff -r bb7ca5891755 -r 640ba8948d69 src/FilesystemCommand.h
--- a/src/FilesystemCommand.h	Sat Jan 25 16:37:20 2020 +0100
+++ b/src/FilesystemCommand.h	Sat Jan 25 21:28:37 2020 +0100
@@ -39,6 +39,7 @@
 #include "FileAttributeFinder.h"
 #include "XattrAttributeFinder.h"
 #include "StreamletAttributeFinder.h"
+#include "FilesystemWorker.h"
 
 namespace relpipe {
 namespace in {
@@ -47,65 +48,7 @@
 namespace fs = std::filesystem;
 using namespace relpipe::writer;
 
-class FilesystemCommand {
-protected:
-	std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings.	
-
-	std::map<string_t, std::shared_ptr<AttributeFinder>> createAttributeFinders() {
-		return {
-			{RequestedField::GROUP_FILE, std::make_shared<FileAttributeFinder>()},
-			{RequestedField::GROUP_STREAMLET, std::make_shared<StreamletAttributeFinder>()},
-			{RequestedField::GROUP_XATTR, std::make_shared<XattrAttributeFinder>()}};
-	}
-
-	void reset(std::stringstream& stream) {
-		stream.str("");
-		stream.clear();
-	}
-
-	bool readNext(std::istream& input, std::stringstream& originalName) {
-		for (char ch; input.get(ch);) {
-			if (ch == 0) return true;
-			else originalName << ch;
-		}
-		return originalName.tellp();
-	}
-
-	string_t fetchRelationName(Configuration* configuration) {
-		return configuration->relation.empty() ? L"filesystem" : configuration->relation;
-	}
-
-	void writeHeader(RelationalWriter* writer, std::map<string_t, std::shared_ptr < AttributeFinder>> attributeFinders, string_t relationName, std::vector<RequestedField>* fields, bool writeHeader = true) {
-		std::vector<AttributeMetadata> attributesMetadata;
-		for (RequestedField field : *fields) {
-			std::shared_ptr<AttributeFinder> finder = attributeFinders[field.group];
-			if (finder) for (AttributeMetadata m : finder->toMetadata(writer, relationName, field)) attributesMetadata.push_back(m);
-			else throw RelpipeWriterException(L"Unsupported field group: " + field.group);
-		}
-
-		writer->startRelation(relationName, attributesMetadata, writeHeader);
-	}
-
-	void processSingleFile(std::shared_ptr<RelationalWriter> writer, std::stringstream& originalName, std::map<string_t, std::shared_ptr < AttributeFinder>>&attributeFinders, Configuration& configuration, string_t relationName) {
-		fs::path file(originalName.str().empty() ? "." : originalName.str()); // interpret empty string as current directory (e.g. result of: find -printf '%P\0')
-		bool exists = false;
-
-		try {
-			exists = fs::exists(file);
-		} catch (const fs::filesystem_error& e) {
-			// we probably do not have permissions to given directory → pretend that the file does not exist
-		}
-
-		for (auto& finder : attributeFinders) finder.second->startFile(file, originalName.str(), exists);
-
-		for (RequestedField field : configuration.fields) {
-			std::shared_ptr<AttributeFinder> finder = attributeFinders[field.group]; // should not be nullptr, because already checked while writing the relation metadata
-			finder->writeField(writer.get(), relationName, field);
-		}
-
-		for (auto& finder : attributeFinders) finder.second->endFile();
-	}
-
+class FilesystemCommand : public FilesystemWorker {
 public:
 
 	virtual ~FilesystemCommand() = default;
diff -r bb7ca5891755 -r 640ba8948d69 src/FilesystemWorker.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/FilesystemWorker.h	Sat Jan 25 21:28:37 2020 +0100
@@ -0,0 +1,116 @@
+/**
+ * Relational pipes
+ * Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <cstdlib>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+#include <map>
+#include <memory>
+#include <algorithm>
+#include <filesystem>
+
+#include <pwd.h>
+#include <grp.h>
+#include <sys/stat.h>
+
+#include <sys/xattr.h>
+
+#include <relpipe/writer/typedefs.h>
+
+#include "Configuration.h"
+#include "AttributeFinder.h"
+#include "FileAttributeFinder.h"
+#include "XattrAttributeFinder.h"
+#include "StreamletAttributeFinder.h"
+
+namespace relpipe {
+namespace in {
+namespace filesystem {
+
+namespace fs = std::filesystem;
+using namespace relpipe::writer;
+
+class FilesystemWorker {
+protected:
+	std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings.	
+
+	std::map<string_t, std::shared_ptr<AttributeFinder>> createAttributeFinders() {
+		return {
+			{RequestedField::GROUP_FILE, std::make_shared<FileAttributeFinder>()},
+			{RequestedField::GROUP_STREAMLET, std::make_shared<StreamletAttributeFinder>()},
+			{RequestedField::GROUP_XATTR, std::make_shared<XattrAttributeFinder>()}};
+	}
+
+	void reset(std::stringstream& stream) {
+		stream.str("");
+		stream.clear();
+	}
+
+	bool readNext(std::istream& input, std::stringstream& originalName) {
+		for (char ch; input.get(ch);) {
+			if (ch == 0) return true;
+			else originalName << ch;
+		}
+		return originalName.tellp();
+	}
+
+	string_t fetchRelationName(Configuration* configuration) {
+		return configuration->relation.empty() ? L"filesystem" : configuration->relation;
+	}
+
+	void writeHeader(RelationalWriter* writer, std::map<string_t, std::shared_ptr < AttributeFinder>> attributeFinders, string_t relationName, std::vector<RequestedField>* fields, bool writeHeader = true) {
+		std::vector<AttributeMetadata> attributesMetadata;
+		for (RequestedField field : *fields) {
+			std::shared_ptr<AttributeFinder> finder = attributeFinders[field.group];
+			if (finder) for (AttributeMetadata m : finder->toMetadata(writer, relationName, field)) attributesMetadata.push_back(m);
+			else throw RelpipeWriterException(L"Unsupported field group: " + field.group);
+		}
+
+		writer->startRelation(relationName, attributesMetadata, writeHeader);
+	}
+
+	void processSingleFile(std::shared_ptr<RelationalWriter> writer, std::stringstream& originalName, std::map<string_t, std::shared_ptr < AttributeFinder>>&attributeFinders, Configuration& configuration, string_t relationName) {
+		fs::path file(originalName.str().empty() ? "." : originalName.str()); // interpret empty string as current directory (e.g. result of: find -printf '%P\0')
+		bool exists = false;
+
+		try {
+			exists = fs::exists(file);
+		} catch (const fs::filesystem_error& e) {
+			// we probably do not have permissions to given directory → pretend that the file does not exist
+		}
+
+		for (auto& finder : attributeFinders) finder.second->startFile(file, originalName.str(), exists);
+
+		for (RequestedField field : configuration.fields) {
+			std::shared_ptr<AttributeFinder> finder = attributeFinders[field.group]; // should not be nullptr, because already checked while writing the relation metadata
+			finder->writeField(writer.get(), relationName, field);
+		}
+
+		for (auto& finder : attributeFinders) finder.second->endFile();
+	}
+
+public:
+
+	virtual ~FilesystemWorker() = default;
+};
+
+}
+}
+}
diff -r bb7ca5891755 -r 640ba8948d69 src/ParallelFilesystemCommand.h
--- a/src/ParallelFilesystemCommand.h	Sat Jan 25 16:37:20 2020 +0100
+++ b/src/ParallelFilesystemCommand.h	Sat Jan 25 21:28:37 2020 +0100
@@ -187,7 +187,7 @@
 	}
 };
 
-class ParallelFilesystemWorker : FilesystemCommand {
+class ParallelFilesystemWorker : FilesystemWorker {
 private:
 	std::string queueName;
 	NamedMutex& stdoutMutex;
@@ -230,11 +230,6 @@
 		}
 
 	}
-
-	void process(int inputFD, int outputFD, Configuration& configuration) override {
-		// FIXME: refactoring, not used
-	}
-
 };
 
 class ParallelFilesystemProcess {
diff -r bb7ca5891755 -r 640ba8948d69 src/StreamletAttributeFinder.h
--- a/src/StreamletAttributeFinder.h	Sat Jan 25 16:37:20 2020 +0100
+++ b/src/StreamletAttributeFinder.h	Sat Jan 25 21:28:37 2020 +0100
@@ -74,7 +74,7 @@
 		if (streamletPathChars) {
 			for (string_t path : splitBySeparator(convertor.from_bytes(streamletPathChars))) {
 				fs::path file = fs::path(path) / fs::path(field.name);
-				if (fs::exists(file) && ::access(file.c_str(), X_OK) == 0) return file.wstring();  // n.b. must be set executable using e.g. chmod – files executable through only ACL, are actually not executable
+				if (fs::exists(file) && ::access(file.c_str(), X_OK) == 0) return file.wstring(); // n.b. must be set executable using e.g. chmod – files executable through only ACL, are actually not executable
 			}
 			throw RelpipeWriterException(L"Streamlet „" + field.name + L"“ was not found at $RELPIPE_IN_FILESYSTEM_STREAMLET_PATH");
 		} else {
@@ -104,7 +104,6 @@
 
 			SubProcess::Message m = subProcesses[field.id]->read();
 			if (m.code != StreamletMsg::WAITING_FOR_INPUT_ATTRIBUTES) throw RelpipeWriterException(L"Protocol violation from exec sub-process. Expected WAITING_FOR_INPUT_ATTRIBUTES but got: " + m.toString());
-			// FIXME: generic protocol violation error messages / method for checking responses
 		}
 	}