diff -r b862d16a2e9f -r abbc9bcfbcc4 relpipe-data/examples/parallel-hashes-1.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/relpipe-data/examples/parallel-hashes-1.sh Mon Feb 03 22:10:07 2020 +0100 @@ -0,0 +1,29 @@ +#!/bin/bash + +findFiles() { + find /bin/ -print0; +} + +fetchAttributes() { + relpipe-in-filesystem \ + --parallel 4 \ + --file path \ + --file type \ + --file size \ + --streamlet hash; +} + +aggregate() { + relpipe-tr-sql \ + --relation "file_hashes" \ + "SELECT + path, + type, + size, + sha256, + count(*) OVER (PARTITION BY sha256) AS same_hash_count + FROM filesystem + ORDER BY same_hash_count, sha256, path, type"; +} + +findFiles | fetchAttributes | aggregate | relpipe-out-tabular