relpipe-data/examples/parallel-hashes-1.sh
branchv_0
changeset 294 abbc9bcfbcc4
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/parallel-hashes-1.sh	Mon Feb 03 22:10:07 2020 +0100
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+findFiles() {
+	find /bin/ -print0;
+}
+
+fetchAttributes() {
+	relpipe-in-filesystem \
+		--parallel 4 \
+		--file path \
+		--file type \
+		--file size \
+		--streamlet hash;
+}
+
+aggregate() {
+	relpipe-tr-sql \
+		--relation "file_hashes" \
+		"SELECT
+			path,
+			type,
+			size,
+			sha256,
+			count(*) OVER (PARTITION BY sha256) AS same_hash_count
+		FROM filesystem
+		ORDER BY same_hash_count, sha256, path, type";
+}
+
+findFiles | fetchAttributes | aggregate | relpipe-out-tabular