src/FilesystemWorker.h
author František Kučera <franta-hg@frantovo.cz>
Sun, 25 Apr 2021 18:47:57 +0200
branchv_0
changeset 89 25a11859975b
parent 61 640ba8948d69
permissions -rw-r--r--
streamlet examples: QR: rename qr to qr-decode + simplify Makefile
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     1
/**
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     2
 * Relational pipes
4
d44ed75822e7 modular design
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
     3
 * Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)
0
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     4
 *
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     5
 * This program is free software: you can redistribute it and/or modify
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     6
 * it under the terms of the GNU General Public License as published by
24
4353cd19a6b5 fix license version: GNU GPLv3
František Kučera <franta-hg@frantovo.cz>
parents: 16
diff changeset
     7
 * the Free Software Foundation, version 3 of the License.
0
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     8
 *
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
     9
 * This program is distributed in the hope that it will be useful,
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    10
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    11
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    12
 * GNU General Public License for more details.
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    13
 *
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    14
 * You should have received a copy of the GNU General Public License
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    15
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    16
 */
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    17
#pragma once
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    18
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    19
#include <cstdlib>
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    20
#include <iostream>
4
d44ed75822e7 modular design
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    21
#include <sstream>
0
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    22
#include <string>
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    23
#include <vector>
4
d44ed75822e7 modular design
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    24
#include <map>
52
fea625f0a096 parallel processing: prepare infrastructure
František Kučera <franta-hg@frantovo.cz>
parents: 32
diff changeset
    25
#include <memory>
0
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    26
#include <algorithm>
2
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    27
#include <filesystem>
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    28
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    29
#include <pwd.h>
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    30
#include <grp.h>
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    31
#include <sys/stat.h>
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    32
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    33
#include <sys/xattr.h>
0
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    34
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    35
#include <relpipe/writer/typedefs.h>
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    36
4
d44ed75822e7 modular design
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    37
#include "Configuration.h"
d44ed75822e7 modular design
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    38
#include "AttributeFinder.h"
d44ed75822e7 modular design
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    39
#include "FileAttributeFinder.h"
d44ed75822e7 modular design
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    40
#include "XattrAttributeFinder.h"
31
c64e1588f428 rename --exec to --streamlet
František Kučera <franta-hg@frantovo.cz>
parents: 29
diff changeset
    41
#include "StreamletAttributeFinder.h"
4
d44ed75822e7 modular design
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    42
0
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    43
namespace relpipe {
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    44
namespace in {
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    45
namespace filesystem {
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
    46
2
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    47
namespace fs = std::filesystem;
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    48
using namespace relpipe::writer;
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    49
61
640ba8948d69 parallel processing: refactoring: ParallelFilesystemWorker inherits FilesystemWorker
František Kučera <franta-hg@frantovo.cz>
parents: 59
diff changeset
    50
class FilesystemWorker {
52
fea625f0a096 parallel processing: prepare infrastructure
František Kučera <franta-hg@frantovo.cz>
parents: 32
diff changeset
    51
protected:
58
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    52
	std::wstring_convert<codecvt_utf8<wchar_t>> convertor; // TODO: support also other encodings.	
2
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    53
52
fea625f0a096 parallel processing: prepare infrastructure
František Kučera <franta-hg@frantovo.cz>
parents: 32
diff changeset
    54
	std::map<string_t, std::shared_ptr<AttributeFinder>> createAttributeFinders() {
fea625f0a096 parallel processing: prepare infrastructure
František Kučera <franta-hg@frantovo.cz>
parents: 32
diff changeset
    55
		return {
fea625f0a096 parallel processing: prepare infrastructure
František Kučera <franta-hg@frantovo.cz>
parents: 32
diff changeset
    56
			{RequestedField::GROUP_FILE, std::make_shared<FileAttributeFinder>()},
fea625f0a096 parallel processing: prepare infrastructure
František Kučera <franta-hg@frantovo.cz>
parents: 32
diff changeset
    57
			{RequestedField::GROUP_STREAMLET, std::make_shared<StreamletAttributeFinder>()},
fea625f0a096 parallel processing: prepare infrastructure
František Kučera <franta-hg@frantovo.cz>
parents: 32
diff changeset
    58
			{RequestedField::GROUP_XATTR, std::make_shared<XattrAttributeFinder>()}};
fea625f0a096 parallel processing: prepare infrastructure
František Kučera <franta-hg@frantovo.cz>
parents: 32
diff changeset
    59
	}
4
d44ed75822e7 modular design
František Kučera <franta-hg@frantovo.cz>
parents: 3
diff changeset
    60
2
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    61
	void reset(std::stringstream& stream) {
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    62
		stream.str("");
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    63
		stream.clear();
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    64
	}
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    65
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    66
	bool readNext(std::istream& input, std::stringstream& originalName) {
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    67
		for (char ch; input.get(ch);) {
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    68
			if (ch == 0) return true;
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    69
			else originalName << ch;
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    70
		}
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    71
		return originalName.tellp();
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    72
	}
f07ed604a0ab read filenames separated by a null-byte; fetch their size, owner, group… and some xattr (extended attribute); a preview version
František Kučera <franta-hg@frantovo.cz>
parents: 0
diff changeset
    73
58
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    74
	string_t fetchRelationName(Configuration* configuration) {
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    75
		return configuration->relation.empty() ? L"filesystem" : configuration->relation;
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    76
	}
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    77
59
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    78
	void writeHeader(RelationalWriter* writer, std::map<string_t, std::shared_ptr < AttributeFinder>> attributeFinders, string_t relationName, std::vector<RequestedField>* fields, bool writeHeader = true) {
58
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    79
		std::vector<AttributeMetadata> attributesMetadata;
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    80
		for (RequestedField field : *fields) {
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    81
			std::shared_ptr<AttributeFinder> finder = attributeFinders[field.group];
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    82
			if (finder) for (AttributeMetadata m : finder->toMetadata(writer, relationName, field)) attributesMetadata.push_back(m);
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    83
			else throw RelpipeWriterException(L"Unsupported field group: " + field.group);
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    84
		}
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
    85
59
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    86
		writer->startRelation(relationName, attributesMetadata, writeHeader);
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    87
	}
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    88
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    89
	void processSingleFile(std::shared_ptr<RelationalWriter> writer, std::stringstream& originalName, std::map<string_t, std::shared_ptr < AttributeFinder>>&attributeFinders, Configuration& configuration, string_t relationName) {
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    90
		fs::path file(originalName.str().empty() ? "." : originalName.str()); // interpret empty string as current directory (e.g. result of: find -printf '%P\0')
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    91
		bool exists = false;
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    92
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    93
		try {
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    94
			exists = fs::exists(file);
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    95
		} catch (const fs::filesystem_error& e) {
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    96
			// we probably do not have permissions to given directory → pretend that the file does not exist
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    97
		}
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    98
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
    99
		for (auto& finder : attributeFinders) finder.second->startFile(file, originalName.str(), exists);
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
   100
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
   101
		for (RequestedField field : configuration.fields) {
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
   102
			std::shared_ptr<AttributeFinder> finder = attributeFinders[field.group]; // should not be nullptr, because already checked while writing the relation metadata
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
   103
			finder->writeField(writer.get(), relationName, field);
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
   104
		}
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
   105
7471529c0d11 parallel processing: first working version
František Kučera <franta-hg@frantovo.cz>
parents: 58
diff changeset
   106
		for (auto& finder : attributeFinders) finder.second->endFile();
58
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
   107
	}
4679f67a8324 parallel processing: put some common code in FilesystemCommand + use POSIX semaphores for STDOUT synchronization across sub-processes
František Kučera <franta-hg@frantovo.cz>
parents: 57
diff changeset
   108
0
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   109
public:
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   110
61
640ba8948d69 parallel processing: refactoring: ParallelFilesystemWorker inherits FilesystemWorker
František Kučera <franta-hg@frantovo.cz>
parents: 59
diff changeset
   111
	virtual ~FilesystemWorker() = default;
0
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   112
};
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   113
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   114
}
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   115
}
467d09b62a12 project skeleton
František Kučera <franta-hg@frantovo.cz>
parents:
diff changeset
   116
}