--- a/relpipe-data/big-picture/relpipe-1.tmx Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/big-picture/relpipe-1.tmx Mon Feb 21 00:43:11 2022 +0100
@@ -1,134 +1,172 @@
<?xml version="1.0" encoding="UTF-8"?>
-<map version="1.0" tiledversion="1.0.3" orientation="orthogonal" renderorder="right-down" width="15" height="12" tilewidth="48" tileheight="48" nextobjectid="57">
+<map version="1.0" tiledversion="1.0.3" orientation="orthogonal" renderorder="right-down" width="19" height="13" tilewidth="48" tileheight="48" nextobjectid="69">
<tileset firstgid="1" source="tiles48-relpipe.tsx"/>
- <layer name="background" width="15" height="12">
+ <layer name="background" width="19" height="13">
<data encoding="csv">
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
</data>
</layer>
- <layer name="pipes" width="15" height="12">
+ <layer name="pipes" width="19" height="13">
<data encoding="csv">
-48,42,60,25,0,26,6,12,30,25,0,0,0,0,0,
-48,42,60,28,0,31,6,12,30,28,0,26,36,42,54,
-48,42,60,33,34,33,6,12,30,28,0,31,36,42,54,
-48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
-48,42,60,28,0,31,6,12,30,33,34,33,36,42,54,
-48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
-48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
-48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
-48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
-48,42,60,28,0,27,34,34,34,23,0,27,36,42,54,
-48,42,60,28,0,0,0,0,0,0,0,0,0,0,0,
-48,42,60,23,0,0,0,0,0,0,0,0,0,0,0
+0,0,0,0,48,42,60,25,0,26,6,12,30,25,0,0,0,0,0,
+0,0,0,0,48,42,60,28,0,31,6,12,30,28,0,26,36,42,54,
+48,42,60,25,48,42,60,33,34,33,6,12,30,28,0,31,36,42,54,
+48,42,60,33,36,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
+48,42,60,28,48,42,60,28,0,31,6,12,30,33,34,33,36,42,54,
+48,42,60,28,48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
+48,42,60,28,48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
+48,42,60,23,48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
+0,0,0,0,48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
+0,0,0,0,48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
+0,0,0,0,48,42,60,28,0,31,6,12,30,28,0,31,36,42,54,
+0,0,0,0,48,42,60,28,0,31,6,12,30,28,0,27,36,42,54,
+0,0,0,0,48,42,60,23,0,27,34,34,34,23,0,0,0,0,0
</data>
</layer>
<objectgroup name="texts">
- <object id="8" x="31.813" y="14.5" width="92.375" height="19">
+ <object id="8" x="223.813" y="14.5" width="92.375" height="19">
<text wrap="1">filesystem</text>
</object>
- <object id="10" x="52.146" y="62.5" width="92.375" height="19">
+ <object id="10" x="244.146" y="62.5" width="92.375" height="19">
<text wrap="1">fstab</text>
</object>
- <object id="11" x="56.479" y="110.5" width="92.375" height="19">
+ <object id="11" x="248.479" y="110.5" width="92.375" height="19">
<text wrap="1">XML</text>
</object>
- <object id="12" x="57.479" y="350.5" width="92.375" height="19">
+ <object id="12" x="249.479" y="255.5" width="92.375" height="19">
<text wrap="1">CSV</text>
</object>
- <object id="13" x="62.146" y="448.167" width="116.375" height="19">
+ <object id="13" x="254.146" y="497.167" width="116.375" height="19">
<text wrap="1">CLI</text>
</object>
- <object id="18" x="341.479" y="14.167" width="92.375" height="19">
+ <object id="18" x="533.479" y="14.167" width="92.375" height="19">
<text wrap="1">grep</text>
</object>
- <object id="19" x="347.146" y="62.833" width="92.375" height="19">
+ <object id="19" x="539.146" y="62.833" width="92.375" height="19">
<text wrap="1">cut</text>
</object>
- <object id="20" x="346.146" y="110.833" width="92.375" height="19">
+ <object id="20" x="538.146" y="110.833" width="92.375" height="19">
<text wrap="1">sed</text>
</object>
- <object id="21" x="326.146" y="158.167" width="92.375" height="19">
+ <object id="21" x="518.146" y="158.167" width="92.375" height="19">
<text wrap="1">validator</text>
</object>
- <object id="22" x="328.146" y="254.5" width="92.375" height="19">
+ <object id="22" x="520.146" y="351.5" width="92.375" height="19">
<text wrap="1">Scheme</text>
</object>
- <object id="24" x="345.146" y="207.5" width="92.375" height="19">
+ <object id="24" x="537.146" y="304.5" width="92.375" height="19">
<text wrap="1">SQL</text>
</object>
- <object id="25" x="332.146" y="302.5" width="92.375" height="19">
+ <object id="25" x="524.146" y="446.5" width="92.375" height="19">
<text wrap="1">Python</text>
</object>
- <object id="26" x="331.479" y="351.167" width="92.375" height="19">
- <text wrap="1">XQuery</text>
+ <object id="26" x="529.479" y="495.167" width="92.375" height="19">
+ <text wrap="1">XPath</text>
</object>
- <object id="27" x="616.146" y="62.167" width="92.375" height="19">
+ <object id="27" x="808.146" y="62.167" width="92.375" height="19">
<text wrap="1">tabular</text>
</object>
- <object id="28" x="630.479" y="110.833" width="92.375" height="19">
+ <object id="28" x="822.479" y="110.833" width="92.375" height="19">
<text wrap="1">XML</text>
</object>
- <object id="29" x="631.479" y="158.833" width="92.375" height="19">
+ <object id="29" x="823.479" y="158.833" width="92.375" height="19">
<text wrap="1">CSV</text>
</object>
- <object id="30" x="613.479" y="254.5" width="92.375" height="19">
+ <object id="30" x="805.479" y="254.5" width="92.375" height="19">
<text wrap="1">nullbyte</text>
</object>
- <object id="31" x="632.146" y="302.5" width="92.375" height="19">
+ <object id="31" x="824.146" y="302.5" width="92.375" height="19">
<text wrap="1">ODS</text>
</object>
- <object id="32" x="633.479" y="351.833" width="92.375" height="19">
+ <object id="32" x="825.479" y="351.833" width="92.375" height="19">
<text wrap="1">GUI</text>
</object>
- <object id="36" x="210.146" y="110.5" width="92.375" height="19">
+ <object id="36" x="402.146" y="110.5" width="92.375" height="19">
<text wrap="1">→</text>
</object>
- <object id="37" x="497.479" y="206.167" width="92.375" height="19">
+ <object id="37" x="689.479" y="206.167" width="92.375" height="19">
<text wrap="1">→</text>
</object>
- <object id="39" x="47.8125" y="301.5" width="92.375" height="19">
+ <object id="39" x="239.813" y="206.5" width="92.375" height="19">
<text wrap="1">recfile</text>
</object>
- <object id="40" x="37.8125" y="158.5" width="92.375" height="19">
+ <object id="40" x="229.813" y="158.5" width="92.375" height="19">
<text wrap="1">XMLTable</text>
</object>
- <object id="41" x="342.813" y="399.5" width="92.375" height="19">
+ <object id="41" x="534.813" y="543.5" width="92.375" height="19">
<text wrap="1">AWK</text>
</object>
- <object id="42" x="623.813" y="205.5" width="92.375" height="19">
+ <object id="42" x="815.813" y="205.5" width="92.375" height="19">
<text wrap="1">recfile</text>
</object>
- <object id="43" x="606.813" y="398.5" width="92.375" height="19">
+ <object id="43" x="798.813" y="398.5" width="92.375" height="19">
<text wrap="1">ASN.1 BER</text>
</object>
- <object id="51" x="33.8125" y="494.5" width="116.375" height="19">
+ <object id="51" x="225.813" y="543.5" width="116.375" height="19">
<text wrap="1">SQL script</text>
</object>
- <object id="52" x="62.8125" y="398.5" width="92.375" height="19">
+ <object id="52" x="254.813" y="447.5" width="92.375" height="19">
+ <text wrap="1">INI</text>
+ </object>
+ <object id="53" x="243.813" y="351.5" width="92.375" height="19">
+ <text wrap="1">YAML</text>
+ </object>
+ <object id="54" x="53.8125" y="207.5" width="92.375" height="19">
+ <text wrap="1">JSON</text>
+ </object>
+ <object id="55" x="800.813" y="446.5" width="92.375" height="19">
+ <text wrap="1">JACK/MIDI</text>
+ </object>
+ <object id="56" x="226.813" y="591.5" width="92.375" height="19">
+ <text wrap="1">JACK/MIDI</text>
+ </object>
+ <object id="57" x="59.8125" y="111.5" width="92.375" height="19">
<text wrap="1">INI</text>
</object>
- <object id="53" x="31.8125" y="207.5" width="92.375" height="19">
- <text wrap="1">YAMLTable</text>
+ <object id="58" x="30.8125" y="350.5" width="92.375" height="19">
+ <text wrap="1">ASN.1 BER</text>
+ </object>
+ <object id="59" x="48.8125" y="255.5" width="92.375" height="19">
+ <text wrap="1">CBOR</text>
</object>
- <object id="54" x="34.8125" y="254.5" width="92.375" height="19">
- <text wrap="1">JSONTable</text>
+ <object id="60" x="247.813" y="303.5" width="92.375" height="19">
+ <text wrap="1">DSV</text>
+ </object>
+ <object id="61" x="48.8125" y="303.5" width="92.375" height="19">
+ <text wrap="1">TOML</text>
+ </object>
+ <object id="62" x="51.8125" y="158.5" width="92.375" height="19">
+ <text wrap="1">YAML</text>
</object>
- <object id="55" x="608.813" y="446.5" width="92.375" height="19">
- <text wrap="1">JACK/MIDI</text>
+ <object id="63" x="246.813" y="399.5" width="92.375" height="19">
+ <text wrap="1">JSON</text>
+ </object>
+ <object id="64" x="827.813" y="495.5" width="92.375" height="19">
+ <text wrap="1">INI</text>
+ </object>
+ <object id="65" x="532.813" y="399.5" width="92.375" height="19">
+ <text wrap="1">Janet</text>
</object>
- <object id="56" x="34.8125" y="542.5" width="92.375" height="19">
- <text wrap="1">JACK/MIDI</text>
+ <object id="66" x="530.813" y="205.5" width="92.375" height="19">
+ <text wrap="1">types</text>
+ </object>
+ <object id="67" x="530.813" y="254.5" width="92.375" height="19">
+ <text wrap="1">union</text>
+ </object>
+ <object id="68" x="817.813" y="543.5" width="92.375" height="19">
+ <text wrap="1">YAML</text>
</object>
</objectgroup>
</map>
--- a/relpipe-data/classic-example.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/classic-example.xml Mon Feb 21 00:43:11 2022 +0100
@@ -61,7 +61,7 @@
<p>
Each program used in the pipeline can be written in different programming language and they will work together.
- Tools written in C, C++, Java, Lisp, Perl, Python, Rust or any other language can be combined together.
+ Tools written in C, C++, D, Java, Lisp, Perl, Python, Rust or any other language can be combined together.
Thus optimal language can be used for each task.
</p>
--- a/relpipe-data/contact.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/contact.xml Mon Feb 21 00:43:11 2022 +0100
@@ -49,6 +49,7 @@
<li>BTC: <code>1APShtP1D4iBNTqX1MhY9uDpeyibarWJjP</code></li>
</ul>
+ <!--
<h2>Mailing lists</h2>
<ul>
@@ -58,6 +59,7 @@
</ul>
<p>Please see the <a href="https://globalcode.info/mailing-lists.xhtml">Mailing lists rules</a>.</p>
+ -->
</text>
--- a/relpipe-data/download.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/download.xml Mon Feb 21 00:43:11 2022 +0100
@@ -14,15 +14,26 @@
<!--
ssh hg@hg.globalcode.info ls -1d data/relpipe/* | xargs -n1 basename | sort | while read r; do echo "hg clone https://hg.globalcode.info/relpipe/$r;"; done
-->
- <pre><![CDATA[hg clone https://hg.globalcode.info/relpipe/relpipe-in-cli.cpp;
+ <pre><![CDATA[hg clone https://hg.globalcode.info/relpipe/relpipe-in-asn1.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-asn1table.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-barcode.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-cbortable.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-cli.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-in-csv.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-in-filesystem.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-in-filesystem.doc;
hg clone https://hg.globalcode.info/relpipe/relpipe-in-fstab.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-htmltable.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-ini.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-initable.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-in-jack.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-mimetable.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-in-recfile.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-in-xml.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-in-xmltable.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-x11.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-yaml.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-in-yamltable.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-lib-cli.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-lib-common.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-lib-reader.cpp;
@@ -31,20 +42,25 @@
hg clone https://hg.globalcode.info/relpipe/relpipe-out-asn1.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-out-csv.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-out-gui.qt.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-out-ini.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-out-jack.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-out-nullbyte.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-out-ods.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-out-recfile.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-out-tabular.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-out-xml.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-out-x11.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-out-yaml.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-tr-awk.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-tr-cut.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-tr-grep.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-tr-infertypes.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-tr-python.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-tr-sed.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-tr-scheme.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-tr-sql.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-tr-validator.cpp;
+hg clone https://hg.globalcode.info/relpipe/relpipe-tr-xpath.cpp;
hg clone https://hg.globalcode.info/relpipe/relpipe-web;]]></pre>
<p>
@@ -59,11 +75,13 @@
<p>
Build instructions are in release notes below.
+ (including the script that downloads all sources and builds them)
</p>
<h2>Released versions</h2>
<ul>
+ <li>2021-12-04: <m:a href="release-v0.18">v0.18</m:a></li>
<li>2020-10-24: <m:a href="release-v0.17.1">v0.17.1</m:a></li>
<li>2020-10-20: <m:a href="release-v0.17">v0.17</m:a></li>
<li>2020-06-06: <m:a href="release-v0.16">v0.16</m:a></li>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples-asn1-x509.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,121 @@
+<stránka
+ xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+ xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+
+ <nadpis>Exploring content of X.509 certificates</nadpis>
+ <perex>open and query common SSL/TLS certificates or other ASN.1 data encoded in BER/DER/CER</perex>
+ <m:pořadí-příkladu>05000</m:pořadí-příkladu>
+
+ <text xmlns="http://www.w3.org/1999/xhtml">
+
+ <p>
+ X.509 certificates and keys used for SSL/TLS (HTTPS, POP3S, IMAPS etc.) are usually distributed as files either with <code>.pem</code> or <code>.der</code> extension.
+ Or bundled together in a PKCS#12 container as a <code>.p12</code> file.
+ The „text“ PEM format is often considered more „accessible“ or „friendly“ than the binary DER.
+ However PEM is just Base64 encoded DER original and is actually less legible to the naked eye than DER,
+ because in DER we can spot at least some strings like common and domain names or validity/expiration dates or recognize certain data structures in a HEX editor.
+ Base64 just obfuscates everything. PEM can be easily copied through clipboard, which is probably the only advantage of this format (but it can also more likely leak).
+ </p>
+
+ <!-- Indeed, Base64 is evil. Use hexadecimal encoding where ASCII representaion of binary data is necessary. -->
+
+ <p>
+ So our first step is to get rid of the annoying Base64 pseudo-plain-text encoding – we use one of these commands:
+ </p>
+
+ <m:pre jazyk="text"><![CDATA[cat certificate.pem | grep -v ^--- | base64 -d > certificate.der
+cat certificate.pem | openssl x509 -inform PEM -outform DER > certificate.der]]></m:pre>
+
+ <p>
+ Telco veterans could now start reading the DER file with <code>hd</code> or <code>xxd</code>, jumping over the offsets and traversing the sequences and sets…
+ However most people would appreciate some software that helps them parsing the ASN.1 BER encoding (the superset of DER and CER).
+ Such software is e.g. Wireshark or dumpasn1. These programs are good for ad-hoc inspection or quick check.
+ </p>
+
+ <p>
+ In <m:name/> <m:a href="release-v0.18">v0.18</m:a> we have (early and bit raw) support for ASN.1 BER encoding and thus we can get the structured data in a machine-readable form
+ – which is good for further processing, conversion to other formats or use in scripts.
+ Because the ASN.1 data model is not relational – actually it is a tree – this format is supported in the <code>relpipe-in-asn1table</code>
+ command that is modelled after the well-known <code>XMLTable()</code> database function that allows translating arbitrary tree structures to relations using the XPath expressions.
+ So in <code>relpipe-in-asn1table</code> we can write XPath expressions to query the ASN.1 tree data structures and extract relations, records and attributes
+ from X.509 certificates, keys or other cryptographic artifacts, LDAP or SNMP packets or any other ASN.1 BER data.
+ </p>
+
+ <p>
+ But how do we know what XPath expressions should we run?
+ It is useful to see the XML representation of whole source data.
+ There is a simple trick to do this – use <code>"/"</code> as the XPath for selecting records (is always selects the single record, single node – the root)
+ and use <code>"."</code> as the XPath to select a single attribute (it always select the root element)
+ and add <code>--mode raw-xml</code>, so we get the raw XML source instead of the text content of given elements.
+ We do not have to write this routine by hand – just create a symlink to the example script:
+ </p>
+ <m:pre jazyk="bash"><![CDATA[ln -s …/relpipe-in-xmltable.cpp/examples/2xml.sh asn12xml # in ~/bin or somewhere]]></m:pre>
+ <p>
+ This example is generic and works also for other formats supported by the <code>relpipe-in-*table</code> commands.
+ </p>
+
+ <p>
+ Then we can analyze X.509 DER certificates stored on our disk or we can fetch some from live servers.
+ The <code>openssl</code> command helps us with that:
+ </p>
+
+
+ <m:pre jazyk="bash"><![CDATA[fetch_x509_certificate() {
+ echo \
+ | openssl s_client -connect $1:${2:-443} 2>/dev/null \
+ | openssl x509 -inform PEM -outform DER;
+}]]></m:pre>
+
+ <p>Now put both commands together in a pipeline:</p>
+
+ <m:pre jazyk="bash"><![CDATA[fetch_x509_certificate "gnu.org" | asn12xml # HTTPS port (443) is used as default]]></m:pre>
+
+ <p>and get this XML representation of the ASN.1 X.509 tree:</p>
+
+ <m:pre jazyk="xml" src="examples/x509-gnu.org.xml"/>
+
+
+ <p>Once we know the structure, we can easily hack together a function that extracts parts of the tree as relations:</p>
+
+ <m:pre jazyk="bash"><![CDATA[parse_x509_certificate() {
+ relpipe-in-asn1table \
+ --relation 'validity' \
+ --records '//sequence[date-time][1]' \
+ --attribute 'from' string 'date-time[1]' \
+ --attribute 'to' string 'date-time[2]' \
+ --relation 'alternative_name' \
+ --records '//sequence[oid="2.5.29.17"][1]/encapsulated/sequence/specific' \
+ --attribute 'name' string '.';
+}]]></m:pre>
+
+ <p>Everything put together:</p>
+
+ <m:pre jazyk="bash"><![CDATA[fetch_x509_certificate "gnu.org" | parse_x509_certificate | relpipe-out-tabular]]></m:pre>
+
+ <p>will print:</p>
+
+ <m:pre jazyk="text" src="examples/x509-gnu.org.txt"/>
+
+ <p>
+ The function above is just a „hello world“ example.
+ Please note that the XPath expressions need to be carefully crafted with respect to the given format in order to match exactly what we want.
+ </p>
+
+ <p>
+ Instead of printing a table, we can use the <code>relpipe-out-nullbyte</code> tool + the <code>read_nullbyte</code> function
+ and shell loop over the records (alternative names) and e.g. <code>ping</code> each domain or fetch given root web page using <code>wget</code> or <code>curl</code>.
+ We can also write a simple script that checks the validity of our own certificates and notifies us in advance when some of them are going to expire.
+ </p>
+
+ <p>
+ Later versions of <code>relpipe-in-asn1table</code> will probably support OID names, so it will not be necessary to use the numeric object identifiers.
+ </p>
+
+ <p>
+ n.b. there is also the <code>relpipe-in-asn1</code> – this tool reads data generated by its counterpart, the <code>relpipe-out-asn1</code> (or other ASN.1 BER capable software)
+ i.e. it is not as universal as <code>relpipe-in-asn1table</code>, it has simpler interface, needs no configuration and expects certain ASN.1 structures (relations serialized in BER format).
+ </p>
+
+ </text>
+
+</stránka>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples-barcode.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,86 @@
+<stránka
+ xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+ xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+
+ <nadpis>Reading barcodes and QR</nadpis>
+ <perex>recognize barcodes (including QR) in image files or streams</perex>
+ <m:pořadí-příkladu>05400</m:pořadí-příkladu>
+
+ <text xmlns="http://www.w3.org/1999/xhtml">
+
+ <p>
+ Either ubiquitous 1D barcodes (procudct labels, ISBN in form of EAN-13 etc.) or 2D barcodes (QR containing hyperlinks, vCards etc.)
+ encode numbers, texts or other data into images that can be printed and scanned or photographed later.
+ Since <m:a href="release-v0.18">v0.18</m:a>, <m:name/> can interact with this technology.
+ </p>
+
+ <p>
+ The <code>relpipe-in-barcode</code> tool reads image data from the standard input, so we can feed a file containing a barcode into it or we can generate one on-the-fly.
+ We can also use <code>tee</code> command to store generated image data in a file.
+ </p>
+ <m:pre jazyk="shell"><![CDATA[echo -n "Big Fun" | qrencode -o - | tee qr.png | relpipe-in-barcode | relpipe-out-tabular]]></m:pre>
+
+ <p>This pipeline generates this relation:</p>
+ <m:pre jazyk="text"><![CDATA[symbol:
+ ╭──────────────┬───────────────┬────────────────┬─────────────┬─────────────┬─────────────────┬──────────────────╮
+ │ id (integer) │ type (string) │ value (string) │ x (integer) │ y (integer) │ width (integer) │ height (integer) │
+ ├──────────────┼───────────────┼────────────────┼─────────────┼─────────────┼─────────────────┼──────────────────┤
+ │ 0 │ QR-Code │ Big Fun │ 12 │ 12 │ 63 │ 63 │
+ ╰──────────────┴───────────────┴────────────────┴─────────────┴─────────────┴─────────────────┴──────────────────╯
+Record count: 1]]></m:pre>
+
+ <p>
+ There might be multiple barcodes (symbols) in the image file and we obtain also their sizes and positions.
+ We may read a <a href="img/barcode-qr-IMG_5758.jpeg">photo</a>:
+ </p>
+
+ <m:pre jazyk="shell"><![CDATA[cat barcode-qr-IMG_5758.jpeg | relpipe-in-barcode | relpipe-out-tabular]]></m:pre>
+
+ <p>and get plenty of symbols:</p>
+ <m:pre jazyk="text" src="examples/barcode-qr-IMG_5758.txt"/>
+
+ <p>
+ There is also (a bit experimental) example of barcode <m:a href="release-v0.15">streamlet</m:a> that is called from <code>relpipe-in-filesystem</code>
+ and works with several files at once (can even run in parallel, like any other streamlet):
+ </p>
+ <m:pre jazyk="shell"><![CDATA[find -print0 | relpipe-in-filesystem --parallel 8 --file name --streamlet barcode-reader | …]]></m:pre>
+
+
+ <p>
+ When we are generating QR codes and use Unicode characters (e.g. the <code>→</code> arrow):
+ </p>
+ <m:pre jazyk="shell"><![CDATA[echo -n "extreme→impression" | qrencode -o - | relpipe-in-barcode | relpipe-out-csv]]></m:pre>
+
+ <p>we may get unexpected result:</p>
+ <m:pre jazyk="text"><![CDATA["id","type","value","x","y","width","height"
+"0","QR-Code","extreme竊段mpression","12","12","74","74"]]></m:pre>
+
+
+ <p>
+ QR codes use the ISO 8859-1 encoding as default and readers usually do some heuristics to guess actual encoding.
+ Sometimes it works, sometimes not.
+ In this particular case, the reader accidentally thought that the encoding was SJIS.
+ So we can <i>fix</i> it by adding <code>| iconv -t SJIS</code> at the end of the pipeline.
+ </p>
+
+ <p>
+ But this is just a workaround.
+ We rather want to add UTF-8 BOM or ECI (Extended Channel Interpretations) to make the reader use the same encoding as we did (UTF-8):
+ </p>
+
+ <m:pre jazyk="shell"><![CDATA[echo -ne "\xEF\xBB\xBFextreme→impression" | qrencode -o - | relpipe-in-barcode | relpipe-out-csv]]></m:pre>
+
+ <p>and get desired result:</p>
+ <m:pre jazyk="text"><![CDATA["id","type","value","x","y","width","height"
+"0","QR-Code","extreme→impression","12","12","74","74"]]></m:pre>
+
+ <p>
+ Because the <code>relpipe-in-barcode</code> tool and the <code>barcode-reader</code> streamlet produce data in machine readable form,
+ we can use them not only for manual ah-hoc reading but also in scripts or batch processing –
+ e.g. extract payment information from invoices or contact information from scanned business cards, catalogize books or read package labels.
+ </p>
+
+
+ </text>
+
+</stránka>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples-csv-data-types.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,169 @@
+<stránka
+ xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+ xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+
+ <nadpis>CSV and data types</nadpis>
+ <perex>declare or recognize integers and booleans in a typeless format</perex>
+ <m:pořadí-příkladu>04800</m:pořadí-příkladu>
+
+ <text xmlns="http://www.w3.org/1999/xhtml">
+
+ <p>
+ CSV (<m:a href="4180" typ="rfc">RFC 4180</m:a>) is quite good solution when we want to store or share relational data in a simple text format –
+ both, human-readable and well supported by many existing applications and libraries.
+ We have even ready-to-use GUI editors, so called spreadsheets (e.g. LibreOffice Calc).
+ However, such simple formats have usually some drawbacks.
+ CSV may contain only a single relation (<i>table</i>, <i>sheet</i>). This is not a big issue – we can use several files.
+ A more serious problem is the absence of data types – in CSV, everything is just a text string.
+ Thus it was impossible to have loss-less conversion to CSV and back.
+ </p>
+
+ <m:pre jazyk="text"><![CDATA[$ find license/ -print0 | relpipe-in-filesystem | relpipe-out-tabular
+filesystem:
+ ╭─────────────────┬───────────────┬────────────────┬────────────────┬────────────────╮
+ │ path (string) │ type (string) │ size (integer) │ owner (string) │ group (string) │
+ ├─────────────────┼───────────────┼────────────────┼────────────────┼────────────────┤
+ │ license/ │ d │ 0 │ hacker │ hacker │
+ │ license/gpl.txt │ f │ 35147 │ hacker │ hacker │
+ ╰─────────────────┴───────────────┴────────────────┴────────────────┴────────────────╯
+Record count: 2]]></m:pre>
+
+ <p>Data types are missing in CSV by default:</p>
+ <m:pre jazyk="text"><![CDATA[$ find license/ -print0 | relpipe-in-filesystem | relpipe-out-csv
+"path","type","size","owner","group"
+"license/","d","0","hacker","hacker"
+"license/gpl.txt","f","35147","hacker","hacker"]]></m:pre>
+
+ <p>The <code>size</code> attribute was integer and now it is mere string:</p>
+ <m:pre jazyk="text"><![CDATA[$ find license/ -print0 | relpipe-in-filesystem | relpipe-out-csv | relpipe-in-csv | relpipe-out-tabular
+csv:
+ ╭─────────────────┬───────────────┬───────────────┬────────────────┬────────────────╮
+ │ path (string) │ type (string) │ size (string) │ owner (string) │ group (string) │
+ ├─────────────────┼───────────────┼───────────────┼────────────────┼────────────────┤
+ │ license/ │ d │ 0 │ hacker │ hacker │
+ │ license/gpl.txt │ f │ 35147 │ hacker │ hacker │
+ ╰─────────────────┴───────────────┴───────────────┴────────────────┴────────────────╯
+Record count: 2]]></m:pre>
+
+
+ <h2>Declare data types in the CSV header</h2>
+
+ <p>
+ Since <m:name/> <m:a href="release-v0.18">v0.18</m:a> we can encode the data types (currently strings, integers and booleans) in the CSV header and then recover them while reading.
+ Such „CSV with data types“ is valid CSV according to the RFC specification and can be viewed or edited in any CSV-capable software.
+ </p>
+
+ <p>
+ The attribute name and data type are separated by the <code>::</code> symbol e.g. <code>name::string,age::integer,member::boolean</code>.
+ Attribute names may contain <code>::</code> (unlike the data type names).
+ </p>
+
+ <p>The data type declarations may be added simply by hand or automatically using <code>relpipe-out-csv</code>.</p>
+
+ <m:pre jazyk="text"><![CDATA[$ find license/ -print0 | relpipe-in-filesystem | relpipe-out-csv --write-types true
+"path::string","type::string","size::integer","owner::string","group::string"
+"license/","d","0","hacker","hacker"
+"license/gpl.txt","f","35147","hacker","hacker"]]></m:pre>
+
+ <p>The <code>relpipe-out-csv</code> + <code>relpipe-in-csv</code> round-trip now does not degrade the data quality:</p>
+ <m:pre jazyk="text"><![CDATA[$ find license/ -print0 | relpipe-in-filesystem | relpipe-out-csv --write-types true | relpipe-in-csv | relpipe-out-tabular
+csv:
+ ╭─────────────────┬───────────────┬────────────────┬────────────────┬────────────────╮
+ │ path (string) │ type (string) │ size (integer) │ owner (string) │ group (string) │
+ ├─────────────────┼───────────────┼────────────────┼────────────────┼────────────────┤
+ │ license/ │ d │ 0 │ hacker │ hacker │
+ │ license/gpl.txt │ f │ 35147 │ hacker │ hacker │
+ ╰─────────────────┴───────────────┴────────────────┴────────────────┴────────────────╯
+Record count: 2]]></m:pre>
+
+
+ <p>
+ So we can put e.g. a CSV editor between them while storing and versioning the data in a different format (like XML or Recfile).
+ Such workflow can be effectively managed by <code>make</code> –
+ <code>make edit</code> will convert versioned data to CSV and launch the editor,
+ <code>make commit</code> will convert data back from the CSV and commit them in Mercurial, Git or other version control system (VCS).
+ </p>
+
+ <p>
+ Why put into VCS data in different format than CSV?
+ Formats like XML or Recfile may have each attribute on a separate line which leads to more readable diffs.
+ At a glance we can see which attributes have been changed.
+ While in CSV we see just a changed long line and even with a better tools we need to count the comas to know which attribute it was.
+ </p>
+
+ <p>
+ The <code>relpipe-out-csv</code> tool generates data types only when explicitly asked for: <code>--write-types true</code>.
+ The <code>relpipe-in-csv</code> tool automatically looks for these type declarations
+ and if all attributes have valid type declarations, they are used, otherwise they are considered to be a part of the attribute name.
+ This behavior can be disabled by <code>--read-types false</code> (<code>true</code> will require valid type declarations).
+ </p>
+
+
+ <h2>Recognize data types using relpipe-tr-infertypes</h2>
+
+ <p>
+ Sometimes we may also want to infer data types from the values automatically without any explicit declaration.
+ Then we put the <code>relpipe-tr-infertypes</code> tool in our pipeline.
+ It buffers whole relations and checks all values of each attribute.
+ If they contain all integers or all booleans they are converted to given type.
+ </p>
+
+ <m:pre jazyk="text"><![CDATA[$ find license/ -print0 | relpipe-in-filesystem | relpipe-out-csv | relpipe-in-csv | relpipe-tr-infertypes | relpipe-out-tabular
+csv:
+ ╭─────────────────┬───────────────┬────────────────┬────────────────┬────────────────╮
+ │ path (string) │ type (string) │ size (integer) │ owner (string) │ group (string) │
+ ├─────────────────┼───────────────┼────────────────┼────────────────┼────────────────┤
+ │ license/ │ d │ 0 │ hacker │ hacker │
+ │ license/gpl.txt │ f │ 35147 │ hacker │ hacker │
+ ╰─────────────────┴───────────────┴────────────────┴────────────────┴────────────────╯
+Record count: 2]]></m:pre>
+
+ <p>
+ This approach is inefficient and contradicts streaming, however it is sometimes useful and convenient for small data coming from external sources.
+ We can e.g. download some data set from network and pipe it through <code>relpipe-in-csv</code> + <code>relpipe-tr-infertypes</code> and improve the data quality a bit.
+ </p>
+
+ <p>
+ We may apply the type inference only on certain relations: <code>--relation "my_relation"</code>
+ or chose different mode: <code>--mode data</code> or <code>metadata</code> or <code>auto</code>.
+ The <code>data</code> mode is described above.
+ In the <code>metadata</code> mode the <code>relpipe-tr-infertypes</code> works similar to <code>relpipe-in-csv --read-types true</code>.
+ The <code>auto</code> mode checks for the metadata in attribute names first and if not found, it fallbacks to the <code>data</code> mode.
+ This tool works with any relational data regardless their original format or source (not only with CSV).
+ </p>
+
+
+ <h2>No header? Specify types as CLI parameters</h2>
+
+ <p>
+ Some CSV files contain just data – have no header line containing the column names.
+ Then we specify the attribute names and data types as CLI parameters of <code>relpipe-in-csv</code>:
+ </p>
+
+ <m:pre jazyk="text"><![CDATA[$ echo -e "a,b,c\nA,B,C" \
+ | relpipe-in-csv \
+ --relation 'just_data' \
+ --attribute 'x' string \
+ --attribute 'y' string \
+ --attribute 'z' string \
+ | relpipe-out-tabular
+
+just_data:
+ ╭────────────┬────────────┬────────────╮
+ │ x (string) │ y (string) │ z (string) │
+ ├────────────┼────────────┼────────────┤
+ │ a │ b │ c │
+ │ A │ B │ C │
+ ╰────────────┴────────────┴────────────╯
+Record count: 2]]></m:pre>
+
+ <p>
+ We may also skip existing header line: <code>tail -n +2</code> and force our own names and types.
+ However this will not work if there are multiline values in the header – which is not common –
+ in such cases we should use some <code>relpipe-tr-*</code> tool to rewrite the names or types
+ (these tools work with relational data instead of plain text).
+ </p>
+
+ </text>
+
+</stránka>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples-csv-sql-join.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,258 @@
+<stránka
+ xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+ xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+
+ <nadpis>Running SQL JOINs on multiple CSV files</nadpis>
+ <perex>query a collection of (not only) CSV files using SQL</perex>
+ <m:pořadí-příkladu>05100</m:pořadí-příkladu>
+
+ <text xmlns="http://www.w3.org/1999/xhtml">
+
+ <p>
+ CSV (<m:a href="4180" typ="rfc">RFC 4180</m:a>) is quite good solution when we want to store or share relational data in a simple text format –
+ both, human-readable and well supported by many existing applications and libraries.
+ We have even ready-to-use GUI editors, so called spreadsheets e.g. LibreOffice Calc.
+ (on the other hand, such simple formats have usually some drawbacks…)
+ </p>
+ <p>
+ In this example, we will show how to query a set of CSV files like it was a relational database.
+ </p>
+
+ <p>Suppose we have a CSV file describing our network interfaces:</p>
+ <m:pre jazyk="text"><![CDATA[address,name
+00:00:00:00:00:00,lo
+00:D0:D8:00:26:00,eth0
+00:01:02:01:33:70,eth1]]></m:pre>
+
+
+ <p>and another CSV file with IP addresses assigned to them:</p>
+ <m:pre jazyk="text"><![CDATA[address,mask,version,interface
+127.0.0.1,8,4,lo
+::1,128,6,lo
+192.168.1.2,24,4,eth0
+192.168.1.8,24,4,eth0
+10.21.12.24,24,4,eth0
+75.748.86.91,95,4,eth1
+23.75.345.200,95,4,eth1
+2a01:430:2e::cafe:babe,64,6,eth1]]></m:pre>
+
+
+ <h2>Loading a CSV file and running basic queries</h2>
+
+ <p>
+ Simplest task is to parse the file and print it as a table in our terminal or convert it to another format (XML, Recfile, ODS, YAML, XHTML, ASN.1 etc.)
+ We can also add <code>relpipe-tr-sql</code> in the middle of our pipeline and run some SQL queries –
+ transform data on-the-fly and send the query result to the <code>relpipe-out-tabular</code> (or other output filter) in place of the original data.
+ For now, we will filter just the IPv6 addresses:
+ </p>
+ <m:pre jazyk="bash"><![CDATA[cat ip.csv \
+ | relpipe-in-csv --relation 'ip' \
+ | relpipe-tr-sql \
+ --relation 'ipv6' "SELECT * FROM ip WHERE version = 6" \
+ | relpipe-out-tabular]]></m:pre>
+ <p>and get them printed:</p>
+ <m:pre jazyk="text"><![CDATA[ipv6:
+ ╭────────────────────────┬───────────────┬──────────────────┬────────────────────╮
+ │ address (string) │ mask (string) │ version (string) │ interface (string) │
+ ├────────────────────────┼───────────────┼──────────────────┼────────────────────┤
+ │ ::1 │ 128 │ 6 │ lo │
+ │ 2a01:430:2e::cafe:babe │ 64 │ 6 │ eth1 │
+ ╰────────────────────────┴───────────────┴──────────────────┴────────────────────╯
+Record count: 2]]></m:pre>
+
+ <p>
+ It is alo possible to run several queries at once
+ and thanks to the <m:name/> format, the result sets are not mixed together, their boundaries are retained and everything is safely passed to the next stage of the pipeline:
+ </p>
+ <m:pre jazyk="bash"><![CDATA[cat ip.csv \
+ | relpipe-in-csv --relation 'ip' \
+ | relpipe-tr-sql \
+ --relation 'ipv4' "SELECT * FROM ip WHERE version = 4" \
+ --relation 'ipv6' "SELECT * FROM ip WHERE version = 6" \
+ | relpipe-out-tabular]]></m:pre>
+ <p>resulting in two nice tables:</p>
+ <m:pre jazyk="text"><![CDATA[ipv4:
+ ╭──────────────────┬───────────────┬──────────────────┬────────────────────╮
+ │ address (string) │ mask (string) │ version (string) │ interface (string) │
+ ├──────────────────┼───────────────┼──────────────────┼────────────────────┤
+ │ 127.0.0.1 │ 8 │ 4 │ lo │
+ │ 192.168.1.2 │ 24 │ 4 │ eth0 │
+ │ 192.168.1.8 │ 24 │ 4 │ eth0 │
+ │ 10.21.12.24 │ 24 │ 4 │ eth0 │
+ │ 75.748.86.91 │ 95 │ 4 │ eth1 │
+ │ 23.75.345.200 │ 95 │ 4 │ eth1 │
+ ╰──────────────────┴───────────────┴──────────────────┴────────────────────╯
+Record count: 6
+
+ipv6:
+ ╭────────────────────────┬───────────────┬──────────────────┬────────────────────╮
+ │ address (string) │ mask (string) │ version (string) │ interface (string) │
+ ├────────────────────────┼───────────────┼──────────────────┼────────────────────┤
+ │ ::1 │ 128 │ 6 │ lo │
+ │ 2a01:430:2e::cafe:babe │ 64 │ 6 │ eth1 │
+ ╰────────────────────────┴───────────────┴──────────────────┴────────────────────╯
+Record count: 2]]></m:pre>
+
+ <h2>Using parametrized queries to avoid SQL injection</h2>
+ <p>
+ When <code>"4"</code> and <code>"6"</code> are not fixed values, we should not glue them to the query string like <code>version = $version</code>,
+ because it is a dangerous practice that may lead to SQL injection.
+ We have parametrized queries for such tasks:
+ </p>
+ <m:pre jazyk="bash"><![CDATA[--relation 'ipv6' "SELECT * FROM ip WHERE version = ?" --parameter "6"]]></m:pre>
+
+
+ <h2>Running SQL JOINs, UNIONs etc. on multiple CSV files</h2>
+
+ <p>
+ To load multiple CSV files into our <i>in-memory database</i>, we just concatenate the relational streams
+ using the means of our shell – the semicolons and parenthesis:
+ </p>
+ <m:pre jazyk="bash"><![CDATA[(relpipe-in-csv --relation 'ip' < ip.csv; relpipe-in-csv --relation 'nic' < nic.csv) \
+ | relpipe-tr-sql \
+ --relation 'ip_nic' "SELECT * FROM ip JOIN nic ON nic.name = ip.interface" \
+ | relpipe-out-tabular]]></m:pre>
+
+ <p>Generic version that loads all <code>*.csv</code> files:</p>
+ <m:pre jazyk="bash"><![CDATA[for csv in *.csv; do relpipe-in-csv --relation "$(basename "$csv" .csv)" < "$csv"; done \
+ | relpipe-tr-sql \
+ --relation 'ip_nic' "SELECT * FROM ip JOIN nic ON nic.name = ip.interface" \
+ | relpipe-out-tabular]]></m:pre>
+
+ <p>Then we can JOIN data from multiple CSV files or do UNIONs, INTERSECTions etc.</p>
+ <m:pre jazyk="text"><![CDATA[ip_nic:
+ ╭────────────────────────┬───────────────┬──────────────────┬────────────────────┬───────────────────┬───────────────╮
+ │ address (string) │ mask (string) │ version (string) │ interface (string) │ address (string) │ name (string) │
+ ├────────────────────────┼───────────────┼──────────────────┼────────────────────┼───────────────────┼───────────────┤
+ │ 127.0.0.1 │ 8 │ 4 │ lo │ 00:00:00:00:00:00 │ lo │
+ │ ::1 │ 128 │ 6 │ lo │ 00:00:00:00:00:00 │ lo │
+ │ 192.168.1.2 │ 24 │ 4 │ eth0 │ 00:D0:D8:00:26:00 │ eth0 │
+ │ 192.168.1.8 │ 24 │ 4 │ eth0 │ 00:D0:D8:00:26:00 │ eth0 │
+ │ 10.21.12.24 │ 24 │ 4 │ eth0 │ 00:D0:D8:00:26:00 │ eth0 │
+ │ 75.748.86.91 │ 95 │ 4 │ eth1 │ 00:01:02:01:33:70 │ eth1 │
+ │ 23.75.345.200 │ 95 │ 4 │ eth1 │ 00:01:02:01:33:70 │ eth1 │
+ │ 2a01:430:2e::cafe:babe │ 64 │ 6 │ eth1 │ 00:01:02:01:33:70 │ eth1 │
+ ╰────────────────────────┴───────────────┴──────────────────┴────────────────────┴───────────────────┴───────────────╯
+Record count: 8]]></m:pre>
+
+
+ <h2>Leveraging shell functions</h2>
+
+ <p>
+ Good practice is to wrap common code blocks into functions and thus make them reusable.
+ In shell, the function still works with input and output streams and we can use them when building our pipelines.
+ Shell functions can be seen as named reusable parts of a pipeline.
+ </p>
+
+ <m:pre jazyk="bash"><![CDATA[csv2relation() { for file; do relpipe-in-csv --relation "$(basename "$file" .csv)" < "$file"; done }
+do_query() { relpipe-tr-sql --relation 'ip_nic' "SELECT * FROM ip JOIN nic ON nic.name = ip.interface"; }
+format_result() { [[ -t 1 ]] && relpipe-out-tabular || cat; }
+
+csv2relation *.csv | do_query | format_result]]></m:pre>
+
+ <p>
+ The <code>format_result()</code> function checks whether the STDOUT is a terminal or not.
+ and when printing to the terminal, it generates a table.
+ When writing to a regular file or STDIN of another process, it passes through original relational data.
+ Thus <code>./our-script.sh</code> will print a nice table in the terminal, while <code>./our-script.sh > data.rp</code> will create a file containing machine-readable data
+ and <code>./our-script.sh | relpipe-out-xhtml > report.xhtml</code> will create an XHTML report and <code>./our-script.sh | relpipe-out-gui</code> will show a GUI window full of tables and maybe also charts.
+ </p>
+
+ <m:img src="img/csv-sql-gui-ip-address-counts.png"/>
+
+ <m:pre jazyk="sql"><![CDATA[SELECT
+ nic.name || ' IPv' || ip.version AS label,
+ nic.name AS interface,
+ ip.version AS ip_version,
+ count(*) AS address_count
+FROM nic
+ LEFT JOIN ip ON (ip.interface = nic.name)
+GROUP BY nic.name, ip.version
+ORDER BY count(*) DESC]]></m:pre>
+
+
+ <h2>Makefile version</h2>
+
+ <p>
+ Shell scripts are not the only way to structure and organize our pipelines or generally our data-processing code.
+ We can also use Make (the tool intended mainly for building sofware), write a <i>Makefile</i> and organize our code around some temporary files and other targets instead of functions.
+ </p>
+
+ <m:pre jazyk="Makefile"><![CDATA[all: print_summary
+
+.PHONY: clean print_summary run_commands
+
+clean:
+ rm -rf *.rp
+
+%.rp: %.csv
+ relpipe-in-csv --relation "$(basename $(<))" < $(<) > $(@)
+
+define SQL_IP_NIC
+ SELECT
+ ip.address AS ip_address,
+ nic.name AS interface,
+ nic.address AS mac_address
+ FROM ip
+ JOIN nic ON (nic.name = ip.interface)
+endef
+export SQL_IP_NIC
+
+define SQL_COUNT_VERSIONS
+ SELECT
+ interface,
+ count(CASE WHEN version=4 THEN 1 ELSE NULL END) AS ipv4_count,
+ count(CASE WHEN version=6 THEN 1 ELSE NULL END) AS ipv6_count
+ FROM ip
+ GROUP BY interface
+ ORDER BY interface
+endef
+export SQL_COUNT_VERSIONS
+
+# Longer SQL queries are better kept in separate .sql files,
+# because we can enjoy syntax highlighting and other support in our editors.
+# Then we use it like this: --relation "ip_nic" "$$(cat ip_nic.sql)"
+
+summary.rp: nic.rp ip.rp
+ cat $(^) \
+ | relpipe-tr-sql \
+ --relation "ip_nic" "$$SQL_IP_NIC" \
+ --relation "counts" "$$SQL_COUNT_VERSIONS" \
+ > $(@)
+
+print_summary: summary.rp
+ cat $(<) | relpipe-out-tabular
+]]></m:pre>
+
+ <p>
+ We can even combine advantages of Make and Bash together (without calling or including Bash scripts from Make)
+ and have reusable shell functions available in the Makefile:
+ </p>
+
+<m:pre jazyk="text"><![CDATA[
+SHELL=bash
+BASH_FUNC_read_nullbyte%%=() { local IFS=; for v in "$$@"; do export "$$v"; read -r -d '' "$$v"; done }
+export BASH_FUNC_read_nullbyte%%]]></m:pre>
+
+ <p>usage example:</p>
+
+<m:pre jazyk="Makefile"><![CDATA[
+run_commands: summary.rp
+ cat $(<) \
+ | relpipe-tr-cut --relation 'ip_nic' --invert-match relation true \
+ | relpipe-out-nullbyte \
+ | while read_nullbyte ip_address interface mac_address; do\
+ echo "network interface $$interface ($$mac_address) has IP address $$ip_address"; \
+ done;
+]]></m:pre>
+
+ <p>
+ Both approaches – the shell script and the Makefile – have pros and cons.
+ With Makefile, we usually create some temporary files containing intermediate results.
+ That avoids streaming. But on the other hand, we process (parse, transform, filter, format etc.) only data that have been changed.
+ </p>
+
+
+ </text>
+
+</stránka>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples-ini.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,333 @@
+<stránka
+ xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+ xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+
+ <nadpis>Reading and writing INI and unix configs</nadpis>
+ <perex>work with INI, classic key=value configurations, Java .properties or Manifests</perex>
+ <m:pořadí-příkladu>05300</m:pořadí-příkladu>
+
+ <text xmlns="http://www.w3.org/1999/xhtml">
+
+ <p>
+ INI is very common simple text format used for configuration files.
+ It extends classic <i>key=value</i> config files and adds one more level – sections – for structuring.
+ </p>
+
+ <p>
+ Unfortunatelly, there is no single INI standard and we have to deal with various INI dialects.
+ Some organizations specify their own INI dialect, but often INI files lacking any formal specification are used.
+ Our tools support various specifics and dialects through the <code>--parser-option</code> options.
+ Like any other CLI options, they are supported in our Bash completion scripts.
+ </p>
+
+ <!--
+ <p>(please note that OS/2 uses binary INI files that are currently not supported in <m:name/> and it is unsure whether they will ever be)</p>
+ -->
+
+ <p>
+ Classic (<i>unix</i>) <i>key=value</i> config files can be considered a dialect or a subset of the INI format
+ and thus be processed by INI input and output filters.
+ It is like INI without any sections or INI having only global properties (<i>key=value</i> at the beginning of the file).
+ Java <i>.properties</i> and <i>MANIFEST.MF</i> files are similar case. They also can be processed by our tools.
+ </p>
+
+ <h3>Simple INI example</h3>
+
+ <p>This is a simple INI file:</p>
+
+ <m:pre jazyk="ini" src="examples/ini-simple.ini"/>
+
+ <p>We read such INI file using this command:</p>
+
+ <m:pre jazyk="text"><![CDATA[cat ini-simple.ini | relpipe-in-ini | relpipe-out-tabular]]></m:pre>
+
+ <p>and get:</p>
+
+ <m:pre jazyk="text" src="examples/ini-simple.tabular"/>
+
+ <p>
+ Of course, we can do more that listing the entries as a table.
+ We can convert them to other formats or e.g. run some command on each entry.
+ Or modify certain values or add/remove entries and save as new INI file.
+ </p>
+
+ <h3>More complex INI example</h3>
+
+ <p>
+ The INI format is just seemingly simple. There is much more than <i>section/key/value</i>.
+ Our parser is by default configured to eat as much INI as possible.
+ It can be tuned by <code>--parser-option</code> CLI options.
+ Sometimes, such tuning is necessary, because some INI features are conflicting – some INI dialects are mutually exclusive.
+ </p>
+
+
+ <p>(please note that syntax highlighting does not support advanced INI features and is sometimes broken)</p>
+
+ <m:pre jazyk="ini" src="examples/ini-complex.ini"/>
+
+ <p>We read such INI file using this command:</p>
+
+ <m:pre jazyk="text"><![CDATA[cat ini-simple.ini | relpipe-in-ini --enable-sub-keys true | relpipe-out-tabular]]></m:pre>
+
+ <p>and get:</p>
+
+ <m:pre jazyk="text" src="examples/ini-complex.tabular"/>
+
+ <p>
+ If we omit the <code>--enable-sub-keys true</code> option, the <code>sub_key</code> attribute disappears
+ and we get <code>a[x]</code> and <code>a[y]</code> in the <code>key</code> attribute
+ i.e. brackets are not interpreted in any special way – they are considered just part of the key.
+ </p>
+
+ <p>See Bash completion for complete list of options and dialects.</p>
+
+
+ <h3>Keeping comments and whitespace while modifying the INI files</h3>
+
+ <p>
+ Comments and empty lines are ignored by default and does not generate any output.
+ This is desired behavior in most cases.
+ But sometimes we want to pass them through.
+ We may e.g. generate some XHTML report or other user-friendly output containing the comments,
+ or we may want to modify existing INI without losing original comments and formatting.
+ The tool offers the <code>--enable-comments</code> and <code>--enable-whitespace</code> options.
+ They allows lossless or almost lossless round-trip conversion from INI to relations and back.
+ </p>
+
+ <p>
+ Besides that, we have also the <code>--enable-event-numbers</code> and <code>--enable-line-numbers</code> options.
+ But they are useful mostly for debugging purposes.
+ </p>
+
+ <p>If we enable all that options, we get this output:</p>
+
+ <m:pre jazyk="text" src="examples/ini-complex.full.tabular"/>
+
+ <p>
+ So we can take the simple INI file (see above) and pass it through a filter
+ that changes (on-the-fly) the <code>x</code> value in the <code>first-section</code>:
+ </p>
+
+ <m:pre jazyk="bash"><![CDATA[cat ini-simple.ini \
+ | relpipe-in-ini --enable-comments true \
+ | relpipe-tr-scheme \
+ --relation 'ini' \
+ --for-each '
+ (if (and (string= $section "first-section") (string= $key "x"))
+ (set! $value "256"))' \
+ | relpipe-out-ini]]></m:pre>
+
+ <p>and get an INI file that contains original comments and modified values:</p>
+
+ <m:pre jazyk="ini" src="examples/ini-simple.modified.ini"/>
+
+ <p>
+ Both the comment at the beginning of the file
+ and the <code>x</code> value in <code>second-section</code>
+ were kept intact.
+ </p>
+
+ <m:pre jazyk="bash"><![CDATA[cat ini-simple.ini \
+ | relpipe-in-ini --enable-comments true \
+ | relpipe-tr-awk \
+ --relation '.*' \
+ --for-each '(section == "first-section" && key == "x") { value = "256"; }; record();' \
+ | relpipe-out-ini]]></m:pre>
+
+ <p>
+ This AWK filter will generate the same output.
+ And we can simply add comments just by adding e.g. <code>comment = "the script changed this value";</code> after the <code>value = "256";</code>
+ which results in:
+ </p>
+
+ <m:pre jazyk="ini"><![CDATA[x = 256 ; the script changed this value]]></m:pre>
+
+ <p>This way we can automatically modify our configuration files.</p>
+
+
+ <h3>Generating INI files</h3>
+
+ <p>
+ The <code>relpipe-out-ini</code> tool serves not only for recreating already existing INI files but also for creating new ones.
+ Data may come from various sources – CSV editor, script, SQL query etc. – and may come in diferent shapes.
+ While on the INI side we use the term „dialect“, on the relational side we use the term „style“.
+ </p>
+
+
+ <p>So in <code>relpipe-out-ini</code> we can chose from several <i>styles</i>:</p>
+
+ <ul>
+ <li><code>standard</code>: expects same structure as generated by <code>relpipe-in-ini</code>; can generate also comments and whitespace</li>
+ <li><code>literal</code>: relation name → section name; attribute name → key; attribute value → value; multiple records → duplicate section names</li>
+ <li><code>section-first</code>: like <code>literal</code>, just section names are taken from the first attribute</li>
+ <li><code>dropped</code>: this relation is ignored/skipped</li>
+ <li><code>auto</code>: use <code>standard</code> style if the relation has <code>key</code> and <code>value</code> attributes; otherwise use <code>literal</code> style</li>
+ </ul>
+
+ <p>Each relation might be processed in different way – use the <code>--relation</code> and regular expression describing the relation(s) name.</p>
+
+ <p>
+ We may for example run:
+ </p>
+
+ <m:pre jazyk="shell"><![CDATA[relpipe-in-x11 --list-input-devices true \
+ | relpipe-tr-awk \
+ --relation '.*' \
+ --output-attribute 'type' string \
+ --output-attribute 'id' integer \
+ --output-attribute 'name' string \
+ --where 'type == "mouse" || type == "keyboard"' \
+ | relpipe-out-ini --style section-first]]></m:pre>
+
+ <p>and get INI like this:</p>
+
+
+ <m:pre jazyk="ini"><![CDATA[[keyboard]
+id = 6
+name = Power Button
+
+[keyboard]
+id = 7
+name = Video Bus
+
+[keyboard]
+id = 8
+name = Power Button
+
+[mouse]
+id = 10
+name = Logitech USB Trackball
+
+[keyboard]
+id = 16
+name = AT Translated Set 2 keyboard
+
+[keyboard]
+id = 12
+name = ZSA Technology Labs Inc ErgoDox EZ Shine Keyboard
+
+[mouse]
+id = 9
+name = 3Dconnexion 3Dconnexion Universal Receiver Mouse]]></m:pre>
+
+ <p>
+ This way, we can generate INI files to be loaded as configuration files by other programs.
+ We can also use <code>relpipe-out-ini</code> for displaying data rather vertically than horizontally (like in common <code>relpipe-out-tabular</code>).
+ This is useful for data with long values or many attributes.
+ The <code>relpipe-out-recfile</code> tool can also server this purpose.
+ </p>
+
+ <h3>Generating XHTML report from several INI files</h3>
+
+ <p>
+ INI format is used by many programs like Mercurial, Git, KDE, Gnome… and also by <m:a href="examples-tr-sql-odbc">ODBC</m:a>.
+ Its configuration is spread across several INI files (system-wide driver configuration and connections and user-specific connections).
+ We can collect all these INI files, use their names as relation names and generate a nice XHTML report:
+ </p>
+
+ <m:pre jazyk="bash"><![CDATA[for file in /etc/odbcinst.ini /etc/odbc.ini ~/.odbc.ini ; do
+ cat "$file" | relpipe-in-ini --relation "$(basename $file)";
+done | relpipe-out-xhtml > odbc-report.xhtml]]></m:pre>
+
+ <p>
+ n.b. It might contain also sensitive information like passwords.
+ We may use some <code>relpipe-tr-*</code> filter to remove such records or replace such values by <code>********</code>.
+ </p>
+
+
+ <h3>Java .properties and MANIFEST.MF</h3>
+
+ <p>
+ Java has built-in support for simple key=value format – so called <i>.properties files</i>.
+ They are often used for storing flat data where more advanced format (like XML) is not necessary.
+ This format supports comments and some escaping sequences.
+ Escaping of non-ASCII characters was mandatory but since Java 9 the UTF-8 encoding is used by default (no need to call <code>native2ascii</code>).
+ In <m:name/> we consider .properties files to be a dialect of the INI format so it is supported in <code>relpipe-in-ini</code> and <code>relpipe-out-ini</code>.
+ The same applies to the MANIFEST.MF which is another key=value format used in the Java world.
+ Thus we can read and write e.g. Java resource bundles (localization), Java EE or Spring application configs or MANIFEST.MF containing OSGi metadata and other information.
+ </p>
+
+ <m:pre jazyk="bash"><![CDATA[# read .properties or MANIFEST.MF:
+cat example.properties | relpipe-in-ini --parser-option dialect java-properties | …
+cat MANIFEST.MF | relpipe-in-ini --parser-option dialect java-manifest-mf | …
+
+# write .properties or MANIFEST.MF:
+… | relpipe-out-ini --writer-option dialect java-properties > example.properties
+… | relpipe-out-ini --writer-option dialect java-manifest-mf > MANIFEST.MF]]></m:pre>
+
+ <p>
+ We can pass such data through various transformation (SQL, AWK, Scheme, XPath etc.)
+ and do conversions to/from various formats (CSV, INI, XML, XHTML, YAML, Recfile, ASN.1 etc.)
+ or convert .properties to MANIFEST.MF and vice versa.
+ </p>
+
+
+ <h3>Classic <i>unix</i> key=value config files</h3>
+
+ <p>
+ Many <i>unix</i> or GNU/Linux programs use simple key=value configuration files with <code>#</code> comments
+ and some basic escaping (like <code>\\</code> → <code>\</code>, <code>\n</code> → <i>new line</i> etc.)
+ It is very similar to Java .properties mentioned above.
+ </p>
+
+ <p>
+ The configuration is often spread across many small files heavily stuffed with comments.
+ If we want to collect just the valid configuration entries from all the files,
+ we may somehow misuse the INI input filter and the fact that given files contain no sections – turn the filenames into section names
+ and feed the result to the <code>relpipe-in-ini</code>:
+ </p>
+
+
+ <m:pre jazyk="bash"><![CDATA[head -n -0 /etc/sysctl.d/*.conf \
+ | sed -E 's/==> (.*) <==/[\1]/g' \
+ | relpipe-in-ini --relation 'sysctl' \
+ | relpipe-out-tabular]]></m:pre>
+
+ <p>This gives us nice overview of our system configuration:</p>
+ <m:pre jazyk="text"><![CDATA[sysctl:
+ ╭─────────────────────────────────────────┬────────────────────────────────────┬────────────────╮
+ │ section (string) │ key (string) │ value (string) │
+ ├─────────────────────────────────────────┼────────────────────────────────────┼────────────────┤
+ │ /etc/sysctl.d/uhd-usrp2.conf │ net.core.rmem_max │ 50000000 │
+ │ /etc/sysctl.d/uhd-usrp2.conf │ net.core.wmem_max │ 1048576 │
+ │ /etc/sysctl.d/10-console-messages.conf │ kernel.printk │ 4 4 1 7 │
+ │ /etc/sysctl.d/10-ipv6-privacy.conf │ net.ipv6.conf.all.use_tempaddr │ 2 │
+ │ /etc/sysctl.d/10-ipv6-privacy.conf │ net.ipv6.conf.default.use_tempaddr │ 2 │
+ │ /etc/sysctl.d/10-kernel-hardening.conf │ kernel.kptr_restrict │ 1 │
+ │ /etc/sysctl.d/10-link-restrictions.conf │ fs.protected_hardlinks │ 1 │
+ │ /etc/sysctl.d/10-link-restrictions.conf │ fs.protected_symlinks │ 1 │
+ │ /etc/sysctl.d/10-magic-sysrq.conf │ kernel.sysrq │ 176 │
+ │ /etc/sysctl.d/10-network-security.conf │ net.ipv4.conf.default.rp_filter │ 1 │
+ │ /etc/sysctl.d/10-network-security.conf │ net.ipv4.conf.all.rp_filter │ 1 │
+ │ /etc/sysctl.d/10-network-security.conf │ net.ipv4.tcp_syncookies │ 1 │
+ │ /etc/sysctl.d/10-ptrace.conf │ kernel.yama.ptrace_scope │ 1 │
+ │ /etc/sysctl.d/10-zeropage.conf │ vm.mmap_min_addr │ 65536 │
+ │ /etc/sysctl.d/99-sysctl.conf │ kernel.sysrq │ 1 │
+ ╰─────────────────────────────────────────┴────────────────────────────────────┴────────────────╯
+Record count: 15]]></m:pre>
+
+
+ <p>We can also do this:</p>
+ <m:pre jazyk="bash"><![CDATA[find /etc/sysctl.d/ -name '*.conf' \
+ | while read f; do echo "[$f]"; cat "$f"; done \
+ | relpipe-in-ini \
+ | relpipe-out-tabular]]></m:pre>
+
+ <p>Or create a reusable function:</p>
+ <m:pre jazyk="bash"><![CDATA[list-config() {
+ head -n -0 "$@" \
+ | sed -E 's/==> (.*) <==/[\1]/g' \
+ | relpipe-in-ini --relation "configs" \
+ | relpipe-out-tabular --write-types false;
+}]]></m:pre>
+
+ <p>and call it this way:</p>
+ <m:pre jazyk="bash"><![CDATA[list-config /etc/sysctl.d/*.conf]]></m:pre>
+
+ <p>It is a bit dirty hack (what if the file name, key or value contains <code><==</code> or <code>[…]</code>?) but it will work quite well.</p>
+
+ <p>And like with Java .properties and MANIFEST-MF, we can use the <code>relpipe-out-ini</code> to generate the configuration files.</p>
+
+ </text>
+
+</stránka>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples-reading-querying-uniform-way.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,170 @@
+<stránka
+ xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+ xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+
+ <nadpis>Reading and querying JSON, YAML, CBOR, HTML, MIME, INI, ASN.1 and XML in a uniform way</nadpis>
+ <perex>run XPath queries and turn data from various sources to relations</perex>
+ <m:pořadí-příkladu>04600</m:pořadí-příkladu>
+
+ <text xmlns="http://www.w3.org/1999/xhtml">
+
+ <p>
+ Data come in different shapes and formats.
+ We can distinguish several main logical models:
+ relational,
+ tree
+ and graph
+ (a tree is an undirected graph with no cycles)
+ Arbitrary trees or even graphs are more flexible, but they are also harder to comprehend and work with.
+ Relational model is somehow limited and easier to grasp, however still flexible enough to describe almost anything.
+ (actually it can describe anything, it is just a question of how nice and native it should look)
+ Unsurprisingly, <m:name/> are build around the relational model.
+ However, sometimes we have to interact with the tree/graph world and deal with data that have other than relational shape.
+ So we need to bridge the gap between trees/graphs and relations.
+ </p>
+
+ <p>
+ While we have just few logical models, there is abundance of serialization formats i.e. mappings of given logical model to a sequence of octets (bytes).
+ Relations might be serialized as CSV, ODS, tables in a database, Recfiles etc.
+ Trees might be serialized as XML, YAML, ASN.1, CBOR, JSON etc.
+ </p>
+
+ <p>
+ Why reinvent the wheel and repeat the same work for each format?
+ </p>
+
+ <p>
+ We already have reusable code for relational data – this is given by the design of <m:name/>, because it separates: <em>inputs</em>, <em>transformations</em> and <em>outputs</em>.
+ Once the data (e.g. CSV) passes through the input filter, it becomes relational data and can be processed in a uniform way by any transformation(s) or output filter.
+ </p>
+
+ <p>
+ But what about the tree data? We have created a set of tools (input filters) that support various serialization formats, in <m:a href="release-v0.18">v0.18</m:a>:
+ </p>
+
+ <ul>
+ <li>XML: <code>relpipe-in-xmltable</code></li>
+ <li>ASN.1: <code>relpipe-in-asn1table</code></li>
+ <li>CBOR: <code>relpipe-in-cbortable</code></li>
+ <li>HTML: <code>relpipe-in-htmltable</code></li>
+ <li>INI: <code>relpipe-in-initable</code></li>
+ <li>MIME: <code>relpipe-in-mimetable</code></li>
+ <li>YAML: <code>relpipe-in-yamltable</code></li>
+ </ul>
+
+ <p>
+ These tools follow the same design principle and offer the same user interface.
+ So once the user learns one tool, he can use this knowledge also while working with other formats.
+ The principle is:
+ </p>
+
+ <ul>
+ <li>We are converting the tree structure to one or more relations.</li>
+ <li>For each relation, define the expression that selects record nodes from the tree.</li>
+ <li>For each attribute, define the expression (relative to the record node) that selects the attribute value.</li>
+ <li>If anything can not (or is not desired to) be mapped to relations, keep is as a tree, so we can process it later – these (sub)trees might be embedded in normal records or reside in a separate relation.</li>
+ <li>We may do a full (lossless) conversion, but we may also extract just a single value from the whole tree (generate a single relation with single record and single attribute). Or anything in between. Anyway, the tool and the logic used is still the same.</li>
+ </ul>
+
+ <p>
+ This is nothing new – and experienced SQL users should already know where the inspiration comes from:
+ the <code>XMLTable()</code> SQL function that converts XML tree to a result set (relation).
+ We just implemented the same functionality as a separate CLI tool, without dependency on any SQL engine and with support for not only XML but also for alternative serialization formats.
+ And for all of them, we use the same query language: XPath.
+ </p>
+
+ <p>
+ Despite this sounds so <i>XML-ish</i>, we do not translate the alternative formats to the XML markup. There is no <i>text full of angle brackets and ampersands</i> in the middle of the process.
+ In our case, we should see XML not as a markup text (meta)format, but rather as an in-memory model – a generic tree of node objects stored in the RAM that allows us doing various tree operations (queries, modifications).
+ </p>
+
+
+ <h2 id="yamlToRelations">Converting a YAML tree to a set of relations</h2>
+
+ <p>
+ Flat key-value lists become sooner or later insufficient for software configuration and it is necessary to somehow manage trees of configuration items (or relations, of course).
+ YAML is quite good tree-serialization format.
+ It is used e.g. for configuring Java Spring applications or for Netplan network configuration in the Ubuntu GNU/Linux distribution:
+ </p>
+
+ <m:pre jazyk="yaml" src="examples/netplan-1.yaml"/>
+
+ <p>We can use following command to convert the tree to a set of relations:</p>
+
+ <m:pre jazyk="bash" src="examples/netplan-1.sh"/>
+
+ <p>
+ So we can do a full relational conversion of the original tree structure or extract just few desired values (e.g. the gateway IP address).
+ We can also pipe a relation to a shell loop and execute some command for each record (e.g. DNS server or IP address).
+ </p>
+
+ <m:img src="img/wmaker-yaml-xml-tabular-1.png"/>
+
+ <p>
+ n.b. YAML is considered to be a superset of JSON, thus tools that can read YAML, can also read JSON.
+ In current version (v0.18) of <m:name/> the <code>relpipe-in-json</code> and <code>relpipe-in-jsontable</code> are just symbolic links to their YAML counterparts.
+ </p>
+
+ <p>
+ There is also similar example: <m:a href="examples-in-xmltable-libvirt">Reading Libvirt XML files using XMLTable</m:a>
+ where we build relations from a XML tree.
+ The principles are the same for all input formats.
+ </p>
+
+ <h2 id="htmlTagSoup">Dealing with the HTML tagsoup</h2>
+
+ <p>
+ With <code>relpipe-in-htmltable</code> we can extract structured information from poor HTML pages.
+ And unlike <code>relpipe-in-xmltable</code>, this tool does not require valid XML/XHTML, so it is good for the dirty work.
+ Processing such invalid data is always bit unreliable, but still better than nothing.
+ </p>
+
+ <m:pre jazyk="bash" src="examples/html-tagsoup-1.sh"/>
+
+ <p>Although Mr. Ryszczyks is unable to create a valid document, this script will print:</p>
+
+ <m:pre jazyk="text" src="examples/html-tagsoup-1.txt"/>
+
+ <p>
+ And thanks to the terminal autodetection in the <code>format_result()</code> function,
+ we can even pipe the result of this script to any <code>relpipe-tr-*</code> or <code>relpipe-out-*</code>
+ and get machine-readable data instead of the ANSI-colored tables –
+ so we can do some further processing or conversion to a different format (XHTML, GUI, ODS, Recfile etc.).
+ </p>
+
+ <h2 id="the2xmlTool">The <code>2xml</code> helper script: <code>yaml2xml</code>, <code>json2xml</code>, <code>asn12xml</code>, <code>mime2xml</code> etc.</h2>
+
+ <p>
+ Mapping from the original syntax to the tree structure is usually quite intuitive and straightforward.
+ However, sometimes it is useful to see the XML serialization of this in-memory model.
+ In the <code>relpipe-in-xmltable.cpp</code> repository we have a helper script called <code>
+ <a href="http://hg.globalcode.info/relpipe/relpipe-in-xmltable.cpp/file/tip/examples/2xml.sh">2xml</a>
+ </code>
+ – this script is not intended to be called directly – instead the user should create a symlink e.g. <code>ini2xml</code>, <code>yaml2xml</code>, <code>asn12xml</code> etc.
+ The <code>2xml</code> script choses the right input filter according to the symlink name and uses it for conversion from the source tree-serialization format to the XML tree-serialization format.
+ </p>
+
+ <p>
+ If we want to do the same thing without the helper script, it is quite simple.
+ We use appropriate <code>relpipe-in-*table</code> tool and extract a single relation with single attribute and single record.
+ The <code>--records</code> expression is <code>'/'</code> i.e. the root node.
+ The <code>--attribute</code> expression is <code>'.'</code> i.e. still the root node.
+ And then we just add the <code>--mode raw-xml</code> to this attribute, so we get the XML serialization of given node (root) instead of the text content.
+ </p>
+
+ <p>
+ In addition to this, the <code>2xml</code> script does also formatting/indentation and syntax highlighting,
+ if given tools (<code>xmllint</code> and <code>pygmentize</code>) are available and the STDOUT is a terminal.
+ </p>
+
+ <p>
+ This script is useful when writing the expressions for <code>relpipe-in-*table</code>,
+ but also as a pipeline filter that allows us to use the whole XML ecosystem also for other formats.
+ We can read YAML, JSON, INI, MIME or even some binary formats etc. and apply a XSLT transformation on such data and generate e.g. some XHTML report or a DocBook document,
+ or validate such structures using XSD or Relax NG schema or we can process such data using XQuery functional language.
+ </p>
+
+
+ </text>
+
+</stránka>
--- a/relpipe-data/examples-rename-groups-backreferences.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/examples-rename-groups-backreferences.xml Mon Feb 21 00:43:11 2022 +0100
@@ -12,8 +12,15 @@
This tool also support regex groups and backreferences. Thus we can use parts of the matched string in our replacement string:
</p>
- <m:pre jazyk="bash"><![CDATA[relpipe-in-cli generate r 1 a string "some string xxx_123 some zzz_456 other" \
- | relpipe-tr-sed 'r' 'a' '([a-z]{3})_([0-9]+)' '$2:$1' \
+ <m:pre jazyk="bash"><![CDATA[relpipe-in-cli \
+ --relation 'r' \
+ --attribute 'a' string \
+ --record 'some string xxx_123 some zzz_456 other' \
+ | relpipe-tr-sed \
+ --relation 'r' \
+ --attribute 'a' \
+ --value '([a-z]{3})_([0-9]+)' \
+ --replacement '$2:$1' \
| relpipe-out-tabular]]></m:pre>
<p>Which would convert this:</p>
--- a/relpipe-data/examples-rename-vg-fstab.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/examples-rename-vg-fstab.xml Mon Feb 21 00:43:11 2022 +0100
@@ -21,7 +21,11 @@
</p>
<m:pre jazyk="bash"><![CDATA[relpipe-in-fstab \
- | relpipe-tr-sed 'fstab' 'device' '^/dev/alpha/' '/dev/beta/' \
+ | relpipe-tr-sed \
+ --relation 'fstab' \
+ --attribute 'device' \
+ --value '^/dev/alpha/' \
+ --replacement '/dev/beta/' \
| relpipe-out-fstab]]></m:pre>
<p>
@@ -35,7 +39,13 @@
For example we can put zeroes in both <code>dump</code> and <code>pass</code> attributes:
</p>
- <m:pre jazyk="bash"><![CDATA[relpipe-in-fstab | relpipe-tr-sed 'fstab' 'dump|pass' '.+' '0' | relpipe-out-fstab]]></m:pre>
+ <m:pre jazyk="bash"><![CDATA[relpipe-in-fstab \
+ | relpipe-tr-sed \
+ --relation 'fstab' \
+ --attribute 'dump|pass' \
+ --value '.+' \
+ --replacement '0' \
+ | relpipe-out-fstab]]></m:pre>
<p>
n.b. the data types must be respected, we can not e.g. put <code>abc</code> in the <code>pass</code> attribute because it is declared as <code>integer</code>.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples-x11-basics.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,358 @@
+<stránka
+ xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+ xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+
+ <nadpis>Exploring X11 windows and devices and emulating mouse movements and keystrokes</nadpis>
+ <perex>connect to X11 server and have some fun</perex>
+ <m:pořadí-příkladu>04900</m:pořadí-příkladu>
+
+ <text xmlns="http://www.w3.org/1999/xhtml">
+
+ <p>
+ The X Window System or X11 comes from 1984 and is still widely used (despite we have some other options like Wayland).
+ This protocol and set of libraries and interfaces gives us GUI (graphical user interface), manages our displays, windows, keyboards and <a href="https://mouse.globalcode.info/v_0/index.xhtml">mice</a>.
+ </p>
+
+ <p>
+ Since <m:name/>
+ <m:a href="release-v0.18">v0.18</m:a> we can interact with this wonderful technology through the <code>relpipe-in-x11</code> and <code>relpipe-out-x11</code> tools.
+ In this example we will show how to acquire information about the input devices, screens and windows or capture and emit X11 events (key presses and mouse movements).
+ </p>
+
+ <h2>Listing windows</h2>
+
+ <p>
+ In X11 almost everything is a <i>window</i>. Especially classic applications like XCalc are composed from many <i>windows</i>.
+ We list all windows of the current display this way:
+ </p>
+
+ <m:pre jazyk="bash"><![CDATA[relpipe-in-x11 --list-windows true | relpipe-out-tabular | less -RSi
+relpipe-in-x11 --list-windows true | relpipe-out-recfile | less]]></m:pre>
+
+ <p>and realize that there are so many windows (hundreds on a common desktop) with so much metadata (long names, titles etc.).</p>
+
+
+ <m:pre jazyk="text"><![CDATA[%rec: x11_window
+%type: id int
+%type: root int
+%type: parent int
+%type: level int
+%type: process int
+%type: x int
+%type: y int
+%type: width int
+%type: height int
+
+id: 1254
+root: 1254
+parent: 0
+level: 0
+process: -1
+name:
+res_class:
+res_name:
+x: 0
+y: 0
+width: 3840
+height: 2160
+
+…
+
+id: 125829135
+root: 1254
+parent: 155189297
+level: 3
+process: 10046
+name: Exploring X11 windows and devices and emulating mouse movements and keystrokes – Relational pipes - Falkon
+res_class: Falkon
+res_name: Falkon Browser
+x: 0
+y: 0
+width: 3840
+height: 2044
+
+…]]></m:pre>
+
+ <p>
+ Windows are identified by their <code>id</code> and organized hierarchically – have <code>parent</code> attribute that contains ID of their ancestor in the tree.
+ There is also the <code>process</code> attribute that contains the PID (process ID) of the running program that created this window.
+ However we should be aware that this information is voluntarily provided by the X11 clients, not managed by the X11 server, and thus might be missing (quite often) or even misleading (rarely).
+ </p>
+
+ <p>
+ In order to not get lost in so many windows, we can run a separate X11 server and put only some programs/windows on its display:
+ </p>
+
+
+ <m:pre jazyk="bash"><![CDATA[# Run a nested X11 server:
+Xephyr :8 -screen 640x480
+
+# Run an application on its display:
+DISPLAY=:8 xcalc
+
+# List the windows:
+DISPLAY=:8 relpipe-in-x11 --list-windows true | relpipe-out-tabular
+]]></m:pre>
+
+ <p>This classic calculator consist of just 51 windows:</p>
+
+ <m:pre jazyk="text"><![CDATA[x11_window:
+ ╭──────────────┬────────────────┬──────────────────┬─────────────────┬───────────────────┬───────────────┬────────────────────┬───────────────────┬─────────────┬─────────────┬─────────────────┬──────────────────╮
+ │ id (integer) │ root (integer) │ parent (integer) │ level (integer) │ process (integer) │ name (string) │ res_class (string) │ res_name (string) │ x (integer) │ y (integer) │ width (integer) │ height (integer) │
+ ├──────────────┼────────────────┼──────────────────┼─────────────────┼───────────────────┼───────────────┼────────────────────┼───────────────────┼─────────────┼─────────────┼─────────────────┼──────────────────┤
+ │ 681 │ 681 │ 0 │ 0 │ -1 │ │ │ │ 0 │ 0 │ 640 │ 480 │
+ │ 2097174 │ 681 │ 681 │ 1 │ -1 │ Calculator │ XCalc │ xcalc │ 0 │ 0 │ 226 │ 304 │
+ │ 2097175 │ 681 │ 2097174 │ 2 │ -1 │ │ │ │ 0 │ 0 │ 226 │ 304 │
+ │ 2097180 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 180 │ 272 │ 40 │ 26 │
+ │ 2097181 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 136 │ 272 │ 40 │ 26 │
+ │ 2097182 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 92 │ 272 │ 40 │ 26 │
+ │ 2097183 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 48 │ 272 │ 40 │ 26 │
+ │ 2097184 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 4 │ 272 │ 40 │ 26 │
+ │ 2097185 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 180 │ 242 │ 40 │ 26 │
+ │ 2097186 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 136 │ 242 │ 40 │ 26 │
+ │ 2097187 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 92 │ 242 │ 40 │ 26 │
+ │ 2097188 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 48 │ 242 │ 40 │ 26 │
+ │ 2097189 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 4 │ 242 │ 40 │ 26 │
+ │ 2097190 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 180 │ 212 │ 40 │ 26 │
+ │ 2097191 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 136 │ 212 │ 40 │ 26 │
+ │ 2097192 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 92 │ 212 │ 40 │ 26 │
+ │ 2097193 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 48 │ 212 │ 40 │ 26 │
+ │ 2097194 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 4 │ 212 │ 40 │ 26 │
+ │ 2097195 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 180 │ 182 │ 40 │ 26 │
+ │ 2097196 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 136 │ 182 │ 40 │ 26 │
+ │ 2097197 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 92 │ 182 │ 40 │ 26 │
+ │ 2097198 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 48 │ 182 │ 40 │ 26 │
+ │ 2097199 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 4 │ 182 │ 40 │ 26 │
+ │ 2097200 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 180 │ 152 │ 40 │ 26 │
+ │ 2097201 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 136 │ 152 │ 40 │ 26 │
+ │ 2097202 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 92 │ 152 │ 40 │ 26 │
+ │ 2097203 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 48 │ 152 │ 40 │ 26 │
+ │ 2097204 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 4 │ 152 │ 40 │ 26 │
+ │ 2097205 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 180 │ 122 │ 40 │ 26 │
+ │ 2097206 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 136 │ 122 │ 40 │ 26 │
+ │ 2097207 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 92 │ 122 │ 40 │ 26 │
+ │ 2097208 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 48 │ 122 │ 40 │ 26 │
+ │ 2097209 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 4 │ 122 │ 40 │ 26 │
+ │ 2097210 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 180 │ 92 │ 40 │ 26 │
+ │ 2097211 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 136 │ 92 │ 40 │ 26 │
+ │ 2097212 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 92 │ 92 │ 40 │ 26 │
+ │ 2097213 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 48 │ 92 │ 40 │ 26 │
+ │ 2097214 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 4 │ 92 │ 40 │ 26 │
+ │ 2097215 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 180 │ 62 │ 40 │ 26 │
+ │ 2097216 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 136 │ 62 │ 40 │ 26 │
+ │ 2097217 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 92 │ 62 │ 40 │ 26 │
+ │ 2097218 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 48 │ 62 │ 40 │ 26 │
+ │ 2097219 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 4 │ 62 │ 40 │ 26 │
+ │ 2097220 │ 681 │ 2097175 │ 3 │ -1 │ │ │ │ 4 │ 2 │ 216 │ 46 │
+ │ 2097221 │ 681 │ 2097220 │ 4 │ -1 │ │ │ │ 6 │ 2 │ 204 │ 38 │
+ │ 2097222 │ 681 │ 2097221 │ 5 │ -1 │ │ │ │ 127 │ 21 │ 18 │ 15 │
+ │ 2097223 │ 681 │ 2097221 │ 5 │ -1 │ │ │ │ 91 │ 21 │ 34 │ 15 │
+ │ 2097224 │ 681 │ 2097221 │ 5 │ -1 │ │ │ │ 61 │ 21 │ 26 │ 15 │
+ │ 2097225 │ 681 │ 2097221 │ 5 │ -1 │ │ │ │ 31 │ 21 │ 26 │ 15 │
+ │ 2097226 │ 681 │ 2097221 │ 5 │ -1 │ │ │ │ 4 │ 23 │ 26 │ 15 │
+ │ 2097227 │ 681 │ 2097221 │ 5 │ -1 │ │ │ │ 18 │ 2 │ 186 │ 17 │
+ │ 2097228 │ 681 │ 2097221 │ 5 │ -1 │ │ │ │ 4 │ 2 │ 10 │ 15 │
+ ╰──────────────┴────────────────┴──────────────────┴─────────────────┴───────────────────┴───────────────┴────────────────────┴───────────────────┴─────────────┴─────────────┴─────────────────┴──────────────────╯
+Record count: 52]]></m:pre>
+
+
+ <h2>Filtering windows with SQL and recursive CTE</h2>
+
+ <p>
+ Another way to deal with the abundance of windows is SQL, especially the recursive CTE (common table expressions).
+ This kind of SQL expressions allows us to work with tree structures and filter all descendants of given parent window.
+ </p>
+
+
+ <m:pre jazyk="bash"><![CDATA[#!/bin/bash
+
+get_x11_child_windows() {
+ SQL="
+WITH
+ RECURSIVE w AS (
+ SELECT * FROM x11_window WHERE name = ?
+ UNION ALL
+ SELECT ch.* FROM x11_window ch JOIN w ON (w.id = ch.parent)
+ )
+SELECT * FROM w
+ORDER BY x, y
+";
+ relpipe-in-x11 --list-windows true \
+ | relpipe-tr-sql \
+ --relation 'window' "$SQL" \
+ --parameter "$1"
+}
+
+format_result() { [[ -t 1 ]] && relpipe-out-tabular || cat; }
+
+get_x11_child_windows "Calculator" | format_result]]></m:pre>
+
+ <p>
+ When we run this script on a desktop with many windows and running XCalc, we will get a listing of the only 51 windows – similar to the listing above.
+ The root window (screen) is not included in this case, but we can also add it using another <code>UNION ALL</code>.
+ </p>
+
+
+ <p>
+ Thanks to <m:name/> modular design we can use also other transformation tools and languages – e.g. Scheme (<code>relpipe-tr-scheme</code>), AWK (<code>relpipe-tr-awk</code>) or XPath (<code>relpipe-tr-xpath</code>).
+ </p>
+
+ <h2>Embedding and composing windows</h2>
+
+ <p>
+ Once we know the ID of a particular window, we can have some fun and run another application inside it:
+ </p>
+
+ <m:pre jazyk="bash"><![CDATA[/usr/lib/xscreensaver/euphoria --regular -window-id 148897953
+mpv --wid=148897935 "film.mkv"]]></m:pre>
+
+ <p>
+ So we can e.g. watch a film<!--, monitor a live camera--> or enjoy a screen saver while calculating:
+ </p>
+
+ <m:img src="img/xcalc-x11-embedding-1.png"/>
+
+ <p>And we can setup everything automatically whithout typing the IDs by hand: </p>
+
+ <m:pre jazyk="bash"><![CDATA[read_nullbyte() { local IFS=; for v in "$@"; do export "$v"; read -r -d '' "$v"; done }
+
+./the-x11-script-listed-above.sh \
+ | relpipe-tr-xpath \
+ --relation '.*' \
+ --where 'level >= 5' \
+ --output-attribute 'wid' integer 'id' \
+ | relpipe-out-nullbyte | while read_nullbyte wid; do
+ sleep 1;
+ /usr/lib/xscreensaver/euphoria --regular -window-id "$wid" &
+ done]]></m:pre>
+
+ <p>and experience pure euphoria:</p>
+
+ <m:img src="img/xcalc-x11-embedding-2.png"/>
+
+ <p>
+ This X11 feature can be used also in much more useful way for composing GUI from several separate programs that might be even written in different programming languages.
+ </p>
+
+
+ <h2>Listing input devices</h2>
+
+ <p>In order to list our keyboards, mice and similar devices we will invoke:</p>
+
+ <m:pre jazyk="bash"><![CDATA[relpipe-in-x11 | relpipe-out-tabular]]></m:pre>
+
+ <p>or we can explicitly say <code>--list-input-devices true</code>; in both cases we will get a listing similar to this one:</p>
+
+ <m:pre jazyk="text"><![CDATA[x11_input_device:
+ ╭──────────────┬───────────────────────────────────────────────────────────┬───────────────╮
+ │ id (integer) │ name (string) │ type (string) │
+ ├──────────────┼───────────────────────────────────────────────────────────┼───────────────┤
+ │ 2 │ Virtual core pointer │ │
+ │ 3 │ Virtual core keyboard │ │
+ │ 4 │ Virtual core XTEST pointer │ │
+ │ 5 │ Virtual core XTEST keyboard │ │
+ │ 6 │ Power Button │ keyboard │
+ │ 7 │ Video Bus │ keyboard │
+ │ 8 │ Power Button │ keyboard │
+ │ 10 │ Logitech USB Trackball │ mouse │
+ │ 11 │ ZSA Technology Labs Inc ErgoDox EZ Shine │ keyboard │
+ │ 12 │ ZSA Technology Labs Inc ErgoDox EZ Shine │ mouse │
+ │ 13 │ ZSA Technology Labs Inc ErgoDox EZ Shine System Control │ keyboard │
+ │ 14 │ ZSA Technology Labs Inc ErgoDox EZ Shine Consumer Control │ mouse │
+ │ 15 │ ZSA Technology Labs Inc ErgoDox EZ Shine Keyboard │ keyboard │
+ │ 16 │ AT Translated Set 2 keyboard │ keyboard │
+ │ 17 │ ZSA Technology Labs Inc ErgoDox EZ Shine Consumer Control │ keyboard │
+ │ 9 │ 3Dconnexion 3Dconnexion Universal Receiver Mouse │ mouse │
+ ╰──────────────┴───────────────────────────────────────────────────────────┴───────────────╯
+Record count: 16]]></m:pre>
+
+ <p>Device IDs can be used to identify the source of input events and their filtering.</p>
+
+ <h2>Capturing input events</h2>
+
+ <p>
+ Once we connect to an X11 server (through the <code>relpipe-in-x11</code> X11 client in our case)
+ we can listen to the input events and monitor the key codes and mouse movements.
+ </p>
+
+ <m:pre jazyk="bash">relpipe-in-x11 --list-input-events true | relpipe-out-csv</m:pre>
+
+ <p>
+ To see the events immediatelly, it is important to use an output filter that does no buffering (e.g. <code>relpipe-out-csv</code> or <code>relpipe-out-gui</code>).
+ </p>
+
+ <m:pre jazyk="text"><![CDATA["device","type","state","key","button","x","y"
+"11","key","released","36","-1","1405","1416"
+"11","key","pressed","38","-1","1405","1416"
+"11","key","released","38","-1","1405","1416"
+"11","key","pressed","43","-1","1405","1416"
+"11","key","released","43","-1","1405","1416"
+"11","key","pressed","32","-1","1405","1416"
+"11","key","released","32","-1","1405","1416"
+"11","key","pressed","44","-1","1405","1416"
+"11","key","released","44","-1","1405","1416"
+"9","motion",,"-1","-1","1405","1416"
+"9","motion",,"-1","-1","1405","1417"
+"9","motion",,"-1","-1","1406","1418"
+"9","motion",,"-1","-1","1406","1419"
+"9","button","pressed","-1","1","1406","1420"
+"9","button","released","-1","1","1406","1420"
+"9","motion",,"-1","-1","1406","1420"
+"9","motion",,"-1","-1","1405","1420"]]></m:pre>
+
+ <p>
+ In order to capture events from a different X11 server, we set the <code>DISPLAY</code> environment variable.
+ This way, we can also capture events remotely over SSH.
+ </p>
+
+ <m:pre jazyk="bash">ssh example.com DISPLAY=:0 relpipe-in-x11 --list-input-events true | relpipe-out-csv</m:pre>
+
+ <p>
+ We can log the events to a file or forward them to another X11 server through <code>relpipe-out-x11</code>.
+ Or we can multiplex the events and forward them to several X11 servers e.g. to run simultaneous tests of different versions or configurations or certain GUI application.
+ </p>
+
+
+ <h2>Simulating input events</h2>
+
+ <p>
+ With the <code>relpipe-out-x11</code> client we can connect to an X11 server and emit input events like they come from a keyboard or mouse
+ (actually they would come from a virtual/simulated one).
+ This command reads relational data with same structure as produced by its counterpart <code>relpipe-in-x11</code>.
+ We can use data captured earlier or create some new one using any tool like
+ <code>relpipe-in-cli</code> or other <code>relpipe-in-*</code> optionally transformed through a <code>relpipe-tr-*</code> filter.
+ We may also translate other signals like <m:a href="examples-jack-midi-monitoring">MIDI</m:a> events to X11 ones (<code>relpipe-in-jack</code>).
+ </p>
+
+ <p>
+ For example, the following command moves the cursor to the coordinates 100,200 and then to 640,480.
+ Such events might be produced also in a shell loop and passed through the pipe to a single <code>relpipe-out-x11</code> process
+ (i.e. without connecting and disconnecting the X11 server on each event).
+ </p>
+
+ <m:pre jazyk="bash"><![CDATA[relpipe-in-cli \
+ --relation "x11_input_event" \
+ --attribute "type" string \
+ --attribute "x" integer \
+ --attribute "y" integer \
+ --records \
+ "motion" "100" "200" \
+ "motion" "640" "480" \
+ | relpipe-out-x11]]></m:pre>
+
+ <p>
+ The <code>relpipe-out-x11</code> tool has two options:
+ <code>--dry-run true</code> that suppresses the events and just tests validity of the input; and
+ <code>--debug true</code> that prints the events in a text form (XML) on STDERR (so we can see that something comes through).
+ </p>
+
+ <m:pre jazyk="bash">relpipe-in-x11 --list-input-events true | relpipe-out-x11 --dry-run true --debug true</m:pre>
+
+ <p>
+ Using SSH, DDC, <code>relpipe-in-x11</code> and <code>relpipe-out-x11</code>,
+ we can build a software replacement of a KVM switch and control several computers from one seat (keyboard, video, mouse).
+ </p>
+
+ </text>
+
+</stránka>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples-xpath-filtering-transforming.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,202 @@
+<stránka
+ xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+ xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+
+ <nadpis>Filtering and transforming relational data with XPath</nadpis>
+ <perex>do simple restrictions and projections using a well-established query language</perex>
+ <m:pořadí-příkladu>04700</m:pořadí-příkladu>
+
+ <text xmlns="http://www.w3.org/1999/xhtml">
+
+ <p>
+ In <m:name/> <m:a href="release-v0.18">v0.18</m:a> we got a new powerful language for filtering and transformations: XPath.
+ It is now part of the toolset consisting of SQL, AWK, Scheme and others.
+ However XPath is originally a language designed for XML, in <m:name/> we can use it for relational data coming from various sources, not only XML,
+ and also for data that violates the rules of normal forms.
+ We can process quite complex tree structures entangled in records but we can also write simple and intuitive expressions like <code>x = "a" or y = 123</code>.
+ </p>
+
+
+ <h2>Basic filtering</h2>
+
+ <p>Let us have some CSV data:</p>
+ <m:pre jazyk="text" src="examples/film-1.csv"/>
+
+ <p>That look like this formatted as a table:</p>
+ <m:pre jazyk="text" src="examples/film-1.tabular"/>
+
+
+ <p>Attributes of particular relations are available in XPath under their names, so we can directly reference them in our queries:</p>
+ <m:pre jazyk="bash"><![CDATA[cat relpipe-data/examples/film-1.csv \
+ | relpipe-in-csv --relation "film" \
+ | relpipe-tr-xpath \
+ --relation '.*' \
+ --where 'year >= 1980 and year < 1990' \
+ | relpipe-out-tabular]]></m:pre>
+
+ <p>filtered result:</p>
+ <m:pre jazyk="text" src="examples/film-1.filtered-1.tabular"/>
+
+ <p>
+ n.b. If there were any characters that are not valid XML name, they would be escaped in the same way as <code>relpipe-in-*table</code> commands do it
+ i.e. by adding underscores and unicode codepoints of given characters – e.g. the <code>weird:field</code> attribute will be available as <code>weird_3a_field</code> in XPath.
+ </p>
+
+
+ <h2>Filtering records with tree structures</h2>
+
+ <p>
+ The CSV above is not a best example of data modeling.
+ Actually, it is quite terrible.
+ But in the real world, we often have to deal with such data – either work with them directly or give them some better shape before we start doing our job.
+ </p>
+
+ <p>
+ Usually the best way is to normalize the model – follow the rules of <a href="https://en.wikipedia.org/wiki/Database_normalization#Normal_forms">Normal forms</a>.
+ In this case, we would break this denormalized CSV table into several relations:
+ <code>film</code>, <code>director</code>, <code>screenwriter</code>…
+ or rather <code>film</code>, <code>role</code>, <code>person</code>, <code>film_person_role</code>…
+ </p>
+
+ <p>
+ But for now, we will keep the data denormalized and just give them a better and machine-readable structure instead of limited and ambiguous notation of <code>screenwriter = name1 + name2</code>
+ (that makes trouble when the value contains certain characters and requires writing a parser for <em>never-specified syntax</em>).
+ So, we will keep some data in classic relational attributes and some in nested XML structure.
+ This approach allows us to combine rigid attributes with free-form rich tree structures.
+ </p>
+
+ <m:pre jazyk="text" src="examples/film-2.tabular"/>
+
+ <p>
+ The <code>relpipe-tr-xpath</code> seamlessly integrates the schema-backed (<code>year</code>) and schema-free (<code>metadata/film</code>) parts of our data model.
+ We use the same language syntax and principles for both kinds of attributes:
+ </p>
+
+
+ <m:pre jazyk="bash"><![CDATA[cat relpipe-data/examples/film-2.csv \
+ | relpipe-in-csv --relation "film" \
+ | relpipe-tr-xpath \
+ --relation '.*' \
+ --xml-attribute 'metadata' \
+ --where 'year = 1986 or metadata/film/screenwriter = "John Hughes"' \
+ | relpipe-out-tabular]]></m:pre>
+
+ <p>Filtered result:</p>
+
+ <m:pre jazyk="text" src="examples/film-2.filtered-1.tabular"/>
+
+ <p>
+ n.b. In current version, we have to mark the attributes containing XML: <code>--xml-attribute 'metadata'</code>.
+ In later versions, there will be a dedicated data type for XML, so these hints will not be necessary.
+ </p>
+
+ <p>
+ This way, we can work with free-form attributes containing multiple values or run various functions on them.
+ We can e.g. list films that have more than one screenwriter:
+ </p>
+
+ <m:pre jazyk="bash">--where 'count(metadata/film/screenwriter) > 1'</m:pre>
+
+ <p>Well, well… here we are:</p>
+
+ <m:pre jazyk="text" src="examples/film-2.filtered-2.tabular"/>
+
+ <p>
+ We can also run XPath from SQL queries (<code>relpipe-tr-sql</code>) e.g. in PostgreSQL.
+ </p>
+
+ <!--
+ cat relpipe-data/examples/film-2.csv \
+ | relpipe-in-csv -\-relation 'film' \
+ | relpipe-tr-sql \
+ -\-data-source-name myPostgreSQL \
+ -\-relation film_1 "SELECT * FROM film WHERE (xpath('count(/film/screenwriter)', metadata::xml))[1]::text::integer > 1" \
+ -\-relation film_2 "SELECT * FROM film WHERE (xpath('count(/film/screenwriter) > 1', metadata::xml))[1]::text::boolean" \
+ | relpipe-out-tabular
+ -->
+
+
+ <h2>Adding new attributes and transforming data</h2>
+
+ <p>
+ The <code>relpipe-tr-xpath</code> does not only restriction but also projection.
+ It can add, remove or modify the attributes while converting the input to the result set.
+ </p>
+
+ <m:pre jazyk="bash"><![CDATA[cat relpipe-data/examples/film-2.csv \
+ | relpipe-in-csv --relation "film" \
+ | relpipe-tr-xpath \
+ --relation '.*' \
+ --xml-attribute 'metadata' \
+ --output-attribute 'title' string 'title' \
+ --output-attribute 'director' string 'metadata/film/director' \
+ --output-attribute 'screenwriter_count' integer 'count(metadata/film/screenwriter)' \
+ | relpipe-out-tabular]]></m:pre>
+
+ <p>We removed some attributes and created new ones:</p>
+ <m:pre jazyk="text" src="examples/film-2.filtered-3.tabular"/>
+
+
+ <p>Or we may concatenate the values:</p>
+
+ <m:pre jazyk="bash"><![CDATA[cat relpipe-data/examples/film-2.csv \
+ | relpipe-in-csv \
+ | relpipe-tr-xpath \
+ --relation '.*' \
+ --xml-attribute 'metadata' \
+ --output-attribute 'sentence' string 'concat("The film ", title, " was directed by ", metadata/film/director, " in year ", year, ".")' \
+ | relpipe-out-nullbyte | tr \\0 \\n]]></m:pre>
+ <!-- alias relpipe-out-lines='relpipe-out-nullbyte | tr \\0 \\n' -->
+
+ <p>and build some sentences:</p>
+ <m:pre jazyk="text" src="examples/film-2.filtered-4.txt"/>
+
+ <h2>Exctracting values from multiple XML files</h2>
+
+ <p>
+ Input data may come not only from some kind of database or some carefully designed data set,
+ they may be e.g. scattered on our filesystem in some already defined file format never intended for use as a database…
+ despite this fact, we can still collect and query such data in a relational way.
+ </p>
+
+ <p>
+ For example, Maven (a build system for Java) describe its modules in XML format in <code>pom.xml</code> files.
+ Using the <code>find</code> and <code>relpipe-in-filesystem</code> we collect them and create a relation containing names and contents of such files:
+ </p>
+
+ <m:pre jazyk="bash"><![CDATA[find -type f -name 'pom.xml' -print0 \
+ | relpipe-in-filesystem \
+ --relation 'module' \
+ --file path \
+ --file content \
+ | relpipe-tr-xpath \
+ --namespace 'm' 'http://maven.apache.org/POM/4.0.0' \
+ --relation '.*' \
+ --xml-attribute 'content' \
+ --output-attribute 'path' string 'path' \
+ --output-attribute 'group_id' string 'content/m:project/m:groupId' \
+ --output-attribute 'artifact_id' string 'content/m:project/m:artifactId' \
+ --output-attribute 'version' string 'content/m:project/m:version' \
+ | relpipe-out-tabular]]></m:pre>
+ <!-- see also relpipe-in-filesystem -\-streamlet xpath -->
+
+ <p>Then we extract desired values using <code>relpipe-tr-xpath</code> and get:</p>
+ <m:pre jazyk="text" src="examples/xpath-maven-1.tabular"/>
+
+ <p>
+ This way we can harvest useful values from XML files – and not only XML files, also from various alternative formats, after we convert them (on-the-fly) to XML.
+ Such conversions are already available for formats like <m:a href="examples-reading-querying-uniform-way">INI, ASN.1, MIME, HTML JSON, YAML etc.</m:a>
+ </p>
+
+
+ <h2>Post scriptum</h2>
+
+ <p>
+ The abovementioned combination of classic relational attributes with free-form XML structures is definitely not a design of first choice.
+ But sometimes it makes sense and sometimes we have to work with data not designed by us and need some tools to deal with them.
+ When we are designing the data model ourselves, we should always pursue the normalized form …and break the rules only if we have really good reason to do so.
+ </p>
+
+ </text>
+
+</stránka>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples-yaml.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,79 @@
+<stránka
+ xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+ xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+
+ <nadpis>Reading and writing YAML</nadpis>
+ <perex>convert relations to YAML or YAML/JSON to relations</perex>
+ <m:pořadí-příkladu>05200</m:pořadí-příkladu>
+
+ <text xmlns="http://www.w3.org/1999/xhtml">
+
+ <p>
+ YAML is a text format for serializing tree structures into text (like XML).
+ It can be sometimes tricky, but basic scenarios are intuitive and straightforward.
+ We can use YAML to describe relational data and load them using the <code>relpipe-in-yaml</code> input filter:
+ </p>
+
+ <m:pre src="examples/film-1-no-types.yaml" jazyk="yaml"/>
+ <p>This YAML notation is almost as simple as <m:a href="examples-recfile">Recfiles</m:a> and like Recfiles it can be easily written by hand.</p>
+
+ <p>
+ We can also use the JSON notation (it can be read by the same tool – <code>relpipe-in-json</code> is currently just a symlink to <code>relpipe-in-yaml</code>).
+ JSON is inferior to YAML and XML, but the JSON input filter might be useful when interacting with software that can produce only JSON.
+ The <code>film</code> relation would look like this in JSON format (shortened):
+ </p>
+ <m:pre jazyk="json"><![CDATA[{
+ "film": [
+ {
+ "title": "Beetlejuice",
+ "year": "1988",
+ "director": "Tim Burton"
+ },
+ {
+ "title": "…",
+ "year": "…",
+ "director": "…"
+ },
+
+ ]
+}]]></m:pre>
+
+ <p>
+ When we want to declare data types
+ – which is usually a good idea –
+ we add the <code>attribute-metadata</code> header and move the data under the <code>record</code> node:
+ </p>
+ <m:pre src="examples/film-1.yaml" jazyk="yaml"/>
+ <p>
+ This form (with data types and quotes) is also produced by the <code>relpipe-out-yaml</code> tool.
+ </p>
+
+
+ <p>When we load such YAML, we get this relational data:</p>
+ <m:pre src="examples/film-1.tabular" jazyk="text"/>
+ <p>Of course, there might be multiple relations, not only one.</p>
+
+
+ <p>
+ Once we convert the YAML to relational format, we can run any transformations and conversions as usual.
+ For example we can use SQL to sort the recordsy by year:
+ </p>
+
+ <m:pre jazyk="shell"><![CDATA[cat film-1.yaml \
+ | relpipe-in-yaml \
+ | relpipe-tr-sql --relation 'film' "SELECT * FROM film ORDER BY year" \
+ | relpipe-out-recfile > relpipe-data/examples/film-1.rec]]></m:pre>
+
+ <p>and Recfile output filter to get:</p>
+ <m:pre src="examples/film-1.rec" jazyk="text"/>
+
+ <p>Or we may use XPath filter:</p>
+ <m:pre jazyk="shell"><![CDATA[relpipe-tr-xpath --relation '.*' --where 'year >= 1980 and year < 1990']]></m:pre>
+ <p>to get the 80s films:</p>
+ <m:pre src="examples/film-1.filtered-2.tabular" jazyk="text"/>
+
+ <p>YAML comes into play when we are looking for a friendly format for user-supplied data.</p>
+
+ </text>
+
+</stránka>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/barcode-qr-IMG_5758.txt Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,17 @@
+symbol:
+ ╭──────────────┬───────────────┬─────────────────────────────────────────────────────┬─────────────┬─────────────┬─────────────────┬──────────────────╮
+ │ id (integer) │ type (string) │ value (string) │ x (integer) │ y (integer) │ width (integer) │ height (integer) │
+ ├──────────────┼───────────────┼─────────────────────────────────────────────────────┼─────────────┼─────────────┼─────────────────┼──────────────────┤
+ │ 0 │ QR-Code │ Every day, once a day, give yourself a present. │ 2100 │ 791 │ 411 │ 410 │
+ │ 1 │ QR-Code │ Does Barry Manilow know that you raid his wardrobe? │ 2844 │ 859 │ 375 │ 369 │
+ │ 2 │ QR-Code │ I'm in love. – That could be a problem. │ 2384 │ 2354 │ 320 │ 313 │
+ │ 3 │ QR-Code │ Grow up Heather, bulimia is so '87. │ 1625 │ 1003 │ 366 │ 365 │
+ │ 4 │ QR-Code │ Trust no 1. │ 4560 │ 917 │ 401 │ 384 │
+ │ 5 │ QR-Code │ Damn good coffee. │ 3738 │ 1775 │ 358 │ 348 │
+ │ 6 │ QR-Code │ Did you say a cherry or Coke slushie? │ 2946 │ 1784 │ 309 │ 310 │
+ │ 7 │ QR-Code │ The truth is out there. │ 4271 │ 285 │ 314 │ 301 │
+ │ 8 │ EAN-13 │ 9781593272203 │ 4254 │ 711 │ 138 │ 390 │
+ │ 9 │ EAN-13 │ 9788025712979 │ 2110 │ 1691 │ 481 │ 194 │
+ │ 10 │ EAN-13 │ 9788074329579 │ 2134 │ 1285 │ 314 │ 231 │
+ ╰──────────────┴───────────────┴─────────────────────────────────────────────────────┴─────────────┴─────────────┴─────────────────┴──────────────────╯
+Record count: 11
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-1-no-types.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,14 @@
+film:
+ ╭──────────────────────┬───────────────┬─────────────────────╮
+ │ title (string) │ year (string) │ director (string) │
+ ├──────────────────────┼───────────────┼─────────────────────┤
+ │ Beetlejuice │ 1988 │ Tim Burton │
+ │ Vacation │ 1983 │ Harold Ramis │
+ │ Christmas Vacation │ 1989 │ Jeremiah S. Chechik │
+ │ Celebrity │ 1998 │ Woody Allen │
+ │ Great Balls of Fire! │ 1989 │ Jim McBride │
+ │ Flatliners │ 1990 │ Joel Schumacher │
+ │ Heathers │ 1988 │ Michael Lehmann │
+ │ Blue Velvet │ 1986 │ David Lynch │
+ ╰──────────────────────┴───────────────┴─────────────────────╯
+Record count: 8
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-1-no-types.yaml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,25 @@
+film:
+ - title: Beetlejuice
+ year: 1988
+ director: Tim Burton
+ - title: Vacation
+ year: 1983
+ director: Harold Ramis
+ - title: Christmas Vacation
+ year: 1989
+ director: Jeremiah S. Chechik
+ - title: Celebrity
+ year: 1998
+ director: Woody Allen
+ - title: Great Balls of Fire!
+ year: 1989
+ director: Jim McBride
+ - title: Flatliners
+ year: 1990
+ director: Joel Schumacher
+ - title: Heathers
+ year: 1988
+ director: Michael Lehmann
+ - title: Blue Velvet
+ year: 1986
+ director: David Lynch
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-1.csv Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,9 @@
+"title::string","year::integer","director::string","screenwriter::string"
+Beetlejuice,1988,Tim Burton,Warren Skaaren + Michael McDowell
+Vacation,1983,Harold Ramis,John Hughes
+Christmas Vacation,1989,Jeremiah S. Chechik,John Hughes
+Celebrity,1998,Woody Allen,Woody Allen
+Great Balls of Fire!,1989,Jim McBride,Jack Baran + Jim McBride
+Flatliners,1990,Joel Schumacher,Peter Filardi
+Heathers,1988,Michael Lehmann,Daniel Waters
+Blue Velvet,1986,David Lynch,David Lynch
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-1.filtered-1.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,12 @@
+csv:
+ ╭──────────────────────┬────────────────┬─────────────────────┬───────────────────────────────────╮
+ │ title (string) │ year (integer) │ director (string) │ screenwriter (string) │
+ ├──────────────────────┼────────────────┼─────────────────────┼───────────────────────────────────┤
+ │ Beetlejuice │ 1988 │ Tim Burton │ Warren Skaaren + Michael McDowell │
+ │ Vacation │ 1983 │ Harold Ramis │ John Hughes │
+ │ Christmas Vacation │ 1989 │ Jeremiah S. Chechik │ John Hughes │
+ │ Great Balls of Fire! │ 1989 │ Jim McBride │ Jack Baran + Jim McBride │
+ │ Heathers │ 1988 │ Michael Lehmann │ Daniel Waters │
+ │ Blue Velvet │ 1986 │ David Lynch │ David Lynch │
+ ╰──────────────────────┴────────────────┴─────────────────────┴───────────────────────────────────╯
+Record count: 6
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-1.filtered-2.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,12 @@
+film:
+ ╭──────────────────────┬────────────────┬─────────────────────╮
+ │ title (string) │ year (integer) │ director (string) │
+ ├──────────────────────┼────────────────┼─────────────────────┤
+ │ Beetlejuice │ 1988 │ Tim Burton │
+ │ Vacation │ 1983 │ Harold Ramis │
+ │ Christmas Vacation │ 1989 │ Jeremiah S. Chechik │
+ │ Great Balls of Fire! │ 1989 │ Jim McBride │
+ │ Heathers │ 1988 │ Michael Lehmann │
+ │ Blue Velvet │ 1986 │ David Lynch │
+ ╰──────────────────────┴────────────────┴─────────────────────╯
+Record count: 6
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-1.rec Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,35 @@
+%rec: film
+%type: year int
+
+title: Vacation
+year: 1983
+director: Harold Ramis
+
+title: Blue Velvet
+year: 1986
+director: David Lynch
+
+title: Beetlejuice
+year: 1988
+director: Tim Burton
+
+title: Heathers
+year: 1988
+director: Michael Lehmann
+
+title: Christmas Vacation
+year: 1989
+director: Jeremiah S. Chechik
+
+title: Great Balls of Fire!
+year: 1989
+director: Jim McBride
+
+title: Flatliners
+year: 1990
+director: Joel Schumacher
+
+title: Celebrity
+year: 1998
+director: Woody Allen
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-1.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,14 @@
+film:
+ ╭──────────────────────┬────────────────┬─────────────────────╮
+ │ title (string) │ year (integer) │ director (string) │
+ ├──────────────────────┼────────────────┼─────────────────────┤
+ │ Beetlejuice │ 1988 │ Tim Burton │
+ │ Vacation │ 1983 │ Harold Ramis │
+ │ Christmas Vacation │ 1989 │ Jeremiah S. Chechik │
+ │ Celebrity │ 1998 │ Woody Allen │
+ │ Great Balls of Fire! │ 1989 │ Jim McBride │
+ │ Flatliners │ 1990 │ Joel Schumacher │
+ │ Heathers │ 1988 │ Michael Lehmann │
+ │ Blue Velvet │ 1986 │ David Lynch │
+ ╰──────────────────────┴────────────────┴─────────────────────╯
+Record count: 8
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-1.yaml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,33 @@
+"film":
+ "attribute-metadata":
+ - "name": "title"
+ "type": "string"
+ - "name": "year"
+ "type": "integer"
+ - "name": "director"
+ "type": "string"
+ "record":
+ - "title": "Beetlejuice"
+ "year": 1988
+ "director": "Tim Burton"
+ - "title": "Vacation"
+ "year": 1983
+ "director": "Harold Ramis"
+ - "title": "Christmas Vacation"
+ "year": 1989
+ "director": "Jeremiah S. Chechik"
+ - "title": "Celebrity"
+ "year": 1998
+ "director": "Woody Allen"
+ - "title": "Great Balls of Fire!"
+ "year": 1989
+ "director": "Jim McBride"
+ - "title": "Flatliners"
+ "year": 1990
+ "director": "Joel Schumacher"
+ - "title": "Heathers"
+ "year": 1988
+ "director": "Michael Lehmann"
+ - "title": "Blue Velvet"
+ "year": 1986
+ "director": "David Lynch"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-2.csv Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,9 @@
+"title::string","year::integer","metadata::string"
+Beetlejuice,1988,<film><director>Tim Burton</director><screenwriter>Warren Skaaren</screenwriter><screenwriter>Michael McDowell</screenwriter></film>
+Vacation,1983,<film><director>Harold Ramis</director><screenwriter>John Hughes</screenwriter></film>
+Christmas Vacation,1989,<film><director>Jeremiah S. Chechik</director><screenwriter>John Hughes</screenwriter></film>
+Celebrity,1998,<film><director>Woody Allen</director><screenwriter>Woody Allen</screenwriter></film>
+Great Balls of Fire!,1989,<film><director>Jim McBride</director><screenwriter>Jack Baran</screenwriter><screenwriter>Jim McBride</screenwriter></film>
+Flatliners,1990,<film><director>Joel Schumacher</director><screenwriter>Peter Filardi</screenwriter></film>
+Heathers,1988,<film><director>Michael Lehmann</director><screenwriter>Daniel Waters</screenwriter></film>
+Blue Velvet,1986,<film><director>David Lynch</director><screenwriter>David Lynch</screenwriter></film>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-2.filtered-1.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,9 @@
+film:
+ ╭────────────────────┬────────────────┬───────────────────────────────────────────────────────────────────────────────────────────────╮
+ │ title (string) │ year (integer) │ metadata (string) │
+ ├────────────────────┼────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────┤
+ │ Vacation │ 1983 │ <film><director>Harold Ramis</director><screenwriter>John Hughes</screenwriter></film> │
+ │ Christmas Vacation │ 1989 │ <film><director>Jeremiah S. Chechik</director><screenwriter>John Hughes</screenwriter></film> │
+ │ Blue Velvet │ 1986 │ <film><director>David Lynch</director><screenwriter>David Lynch</screenwriter></film> │
+ ╰────────────────────┴────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────╯
+Record count: 3
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-2.filtered-2.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,8 @@
+film:
+ ╭──────────────────────┬────────────────┬───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+ │ title (string) │ year (integer) │ metadata (string) │
+ ├──────────────────────┼────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
+ │ Beetlejuice │ 1988 │ <film><director>Tim Burton</director><screenwriter>Warren Skaaren</screenwriter><screenwriter>Michael McDowell</screenwriter></film> │
+ │ Great Balls of Fire! │ 1989 │ <film><director>Jim McBride</director><screenwriter>Jack Baran</screenwriter><screenwriter>Jim McBride</screenwriter></film> │
+ ╰──────────────────────┴────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+Record count: 2
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-2.filtered-3.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,14 @@
+film:
+ ╭──────────────────────┬─────────────────────┬──────────────────────────────╮
+ │ title (string) │ director (string) │ screenwriter_count (integer) │
+ ├──────────────────────┼─────────────────────┼──────────────────────────────┤
+ │ Beetlejuice │ Tim Burton │ 2 │
+ │ Vacation │ Harold Ramis │ 1 │
+ │ Christmas Vacation │ Jeremiah S. Chechik │ 1 │
+ │ Celebrity │ Woody Allen │ 1 │
+ │ Great Balls of Fire! │ Jim McBride │ 2 │
+ │ Flatliners │ Joel Schumacher │ 1 │
+ │ Heathers │ Michael Lehmann │ 1 │
+ │ Blue Velvet │ David Lynch │ 1 │
+ ╰──────────────────────┴─────────────────────┴──────────────────────────────╯
+Record count: 8
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-2.filtered-4.txt Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,8 @@
+The film Beetlejuice was directed by Tim Burton in year 1988.
+The film Vacation was directed by Harold Ramis in year 1983.
+The film Christmas Vacation was directed by Jeremiah S. Chechik in year 1989.
+The film Celebrity was directed by Woody Allen in year 1998.
+The film Great Balls of Fire! was directed by Jim McBride in year 1989.
+The film Flatliners was directed by Joel Schumacher in year 1990.
+The film Heathers was directed by Michael Lehmann in year 1988.
+The film Blue Velvet was directed by David Lynch in year 1986.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/film-2.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,14 @@
+film:
+ ╭──────────────────────┬────────────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+ │ title (string) │ year (integer) │ metadata (string) │
+ ├──────────────────────┼────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
+ │ Beetlejuice │ 1988 │ <film><director>Tim Burton</director><screenwriter>Warren Skaaren</screenwriter><screenwriter>Michael McDowell</screenwriter></film> │
+ │ Vacation │ 1983 │ <film><director>Harold Ramis</director><screenwriter>John Hughes</screenwriter></film> │
+ │ Christmas Vacation │ 1989 │ <film><director>Jeremiah S. Chechik</director><screenwriter>John Hughes</screenwriter></film> │
+ │ Celebrity │ 1998 │ <film><director>Woody Allen</director><screenwriter>Woody Allen</screenwriter></film> │
+ │ Great Balls of Fire! │ 1989 │ <film><director>Jim McBride</director><screenwriter>Jack Baran</screenwriter><screenwriter>Jim McBride</screenwriter></film> │
+ │ Flatliners │ 1990 │ <film><director>Joel Schumacher</director><screenwriter>Peter Filardi</screenwriter></film> │
+ │ Heathers │ 1988 │ <film><director>Michael Lehmann</director><screenwriter>Daniel Waters</screenwriter></film> │
+ │ Blue Velvet │ 1986 │ <film><director>David Lynch</director><screenwriter>David Lynch</screenwriter></film> │
+ ╰──────────────────────┴────────────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+Record count: 8
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/html-tagsoup-1.sh Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+HTML='
+
+<p>Our company is focused on:
+<ul>
+<li>video game arcades
+<li>laundry</li>
+<LI>cigarette machines and trucking
+<li>personal loans and politics
+</ul>
+
+<!-- TODO: add more GIFs -->
+
+<P>Visit our <a href="index.htm">front page</a> and check the <A href="news.php">news</a>!!!
+
+<!--
+Yes, this HTML tagsoup is total mess.
+But do you still remember the pure joy when you put your first website on the internet?
+-->
+
+<p>
+download <a href="mp3.cgi?token=61686f6a-e1ab-4b92-c357-474e552f4c69" class="mp3">free MP3</a> now
+</P>
+
+<!--
+Best viewed in Netscape Navigator and resolution 800×600
+(c) Fvkgrra Pnaqyrf ltd. 1984-2022 -->
+
+';
+
+fetch_html() {
+ # there might be a wget/curl call to download a fresh version of the web page
+ echo "$HTML";
+}
+
+
+extract_relations() {
+ relpipe-in-htmltable \
+ --relation 'field_of_business' \
+ --records '//li' \
+ --attribute 'priority' integer 'count(preceding::li)+1' \
+ --attribute 'name' string '.' \
+ --attribute 'normalized' string 'normalize-space(.)' \
+ --relation 'hyperlink' \
+ --records '//a' \
+ --attribute 'url' string '@href' \
+ --attribute 'name' string '.' \
+ --attribute 'xpath' string '.' --mode xpath \
+ --relation 'download_token' \
+ --records '//a[@class="mp3"]' \
+ --attribute 'value' string 'substring(@href, 15)' \
+ --relation 'hidden_footer' \
+ --records '//comment()[count(following::*)=0]' \
+ --attribute 'text' string 'normalize-space(.)'
+}
+
+format_result() { [[ -t 1 ]] && relpipe-out-tabular || cat; }
+
+
+# fetch_html | html2xml 1>&2
+
+fetch_html | extract_relations | format_result
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/html-tagsoup-1.txt Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,36 @@
+field_of_business:
+ ╭────────────────────┬──────────────────────────────────┬─────────────────────────────────╮
+ │ priority (integer) │ name (string) │ normalized (string) │
+ ├────────────────────┼──────────────────────────────────┼─────────────────────────────────┤
+ │ 1 │ video game arcades↲ │ video game arcades │
+ │ 2 │ laundry │ laundry │
+ │ 3 │ cigarette machines and trucking↲ │ cigarette machines and trucking │
+ │ 4 │ personal loans and politics↲ │ personal loans and politics │
+ ╰────────────────────┴──────────────────────────────────┴─────────────────────────────────╯
+Record count: 4
+
+hyperlink:
+ ╭────────────────────────────────────────────────────┬───────────────┬──────────────────────╮
+ │ url (string) │ name (string) │ xpath (string) │
+ ├────────────────────────────────────────────────────┼───────────────┼──────────────────────┤
+ │ index.htm │ front page │ /html/body/p[2]/a[1] │
+ │ news.php │ news │ /html/body/p[2]/a[2] │
+ │ mp3.cgi?token=61686f6a-e1ab-4b92-c357-474e552f4c69 │ free MP3 │ /html/body/p[3]/a │
+ ╰────────────────────────────────────────────────────┴───────────────┴──────────────────────╯
+Record count: 3
+
+download_token:
+ ╭──────────────────────────────────────╮
+ │ value (string) │
+ ├──────────────────────────────────────┤
+ │ 61686f6a-e1ab-4b92-c357-474e552f4c69 │
+ ╰──────────────────────────────────────╯
+Record count: 1
+
+hidden_footer:
+ ╭─────────────────────────────────────────────────────────────────────────────────────────────╮
+ │ text (string) │
+ ├─────────────────────────────────────────────────────────────────────────────────────────────┤
+ │ Best viewed in Netscape Navigator and resolution 800×600 (c) Fvkgrra Pnaqyrf ltd. 1984-2022 │
+ ╰─────────────────────────────────────────────────────────────────────────────────────────────╯
+Record count: 1
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/ini-complex.full.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,55 @@
+ini:
+ ╭────────────────┬─────────────────┬──────────────────┬─────────────────────┬──────────────────┬──────────────────────────────────────────────────────────┬─────────────────────────────────────────────────────────────────────────────────┬─────────────────────╮
+ │ line (integer) │ event (integer) │ section (string) │ key (string) │ sub_key (string) │ value (string) │ comment (string) │ whitespace (string) │
+ ├────────────────┼─────────────────┼──────────────────┼─────────────────────┼──────────────────┼──────────────────────────────────────────────────────────┼─────────────────────────────────────────────────────────────────────────────────┼─────────────────────┤
+ │ 1 │ 1 │ │ global │ │ value │ │ │
+ │ 2 │ 2 │ │ │ │ │ │ ↲ │
+ │ 4 │ 4 │ section │ │ │ │ section tags are currently supported by the parser (do not generate error) │ │
+ │ 5 │ 5 │ section │ │ │ │ but do not generate any output │ │
+ │ 6 │ 6 │ section │ section-tag-support │ │ partial │ │ │
+ │ 7 │ 7 │ section │ │ │ │ │ ↲ │
+ │ 9 │ 9 │ quoting │ │ │ │ │ ↲ │
+ │ 10 │ 10 │ quoting │ quoted │ │ some text in quotes │ │ │
+ │ 11 │ 11 │ quoting │ apostrophed │ │ some text in apostrophes │ │ │
+ │ 12 │ 12 │ quoting │ │ │ │ │ ↲ │
+ │ 13 │ 13 │ quoting │ unquoted │ │ values are trimmed │ │ │
+ │ 14 │ 14 │ quoting │ we-need │ │ quotes or apostrophes │ │ │
+ │ 15 │ 15 │ quoting │ to-keep │ │ the leading/trailing whitespace │ │ │
+ │ 16 │ 16 │ quoting │ │ │ │ │ ↲ │
+ │ 18 │ 18 │ multiline │ │ │ │ │ ↲ │
+ │ 19 │ 19 │ multiline │ quoted │ │ first line↲second line │ │ │
+ │ 21 │ 20 │ multiline │ │ │ │ │ ↲ │
+ │ 22 │ 21 │ multiline │ apostrophed │ │ first line↲second line │ │ │
+ │ 24 │ 22 │ multiline │ │ │ │ │ ↲ │
+ │ 25 │ 23 │ multiline │ │ │ │ unlike quoted/apostrophed strings, there is no line end inside the value: │ │
+ │ 26 │ 24 │ multiline │ backslashed │ │ first line continues here │ │ │
+ │ 28 │ 25 │ multiline │ │ │ │ │ ↲ │
+ │ 29 │ 26 │ comments │ │ │ │ we can comment also secions │ │
+ │ 30 │ 27 │ comments │ │ │ │ │ ↲ │
+ │ 31 │ 28 │ comments │ ini │ │ true │ and even values │ │
+ │ 32 │ 29 │ comments │ unix │ │ true │ │ │
+ │ 33 │ 30 │ comments │ │ │ │ │ ↲ │
+ │ 34 │ 31 │ comments │ │ │ │ classic INI comment │ │
+ │ 35 │ 32 │ comments │ │ │ │ unix-style comments are also supported │ │
+ │ 36 │ 33 │ comments │ │ │ │ │ ↲ │
+ │ 37 │ 34 │ comments │ but │ │ be aware that ; this is part of the value, not a comment │ │ │
+ │ 38 │ 35 │ comments │ │ │ │ we need '' or "" to put comment on the same line as the value │ │
+ │ 39 │ 36 │ comments │ │ │ │ │ ↲ │
+ │ 41 │ 38 │ subkeys │ │ │ │ │ ↲ │
+ │ 42 │ 39 │ subkeys │ │ │ │ we may specify subkeys in [] brackets: │ │
+ │ 43 │ 40 │ subkeys │ a │ x │ AX │ │ │
+ │ 44 │ 41 │ subkeys │ a │ y │ AY │ │ │
+ │ 45 │ 42 │ subkeys │ │ │ │ │ ↲ │
+ │ 46 │ 43 │ subkeys │ │ │ │ because this feature is quite uncommon, it is disabled by default │ │
+ │ 47 │ 44 │ subkeys │ │ │ │ and whole key+subkey becomes a key │ │
+ │ 48 │ 45 │ subkeys │ │ │ │ but we can turn on the subkey support by the CLI option: --enable-sub-keys true │ │
+ │ 49 │ 46 │ subkeys │ │ │ │ and get separete key and sub_key attributes on the output │ │
+ │ 50 │ 47 │ subkeys │ │ │ │ │ ↲ │
+ │ 52 │ 49 │ escaping │ │ │ │ │ ↲ │
+ │ 53 │ 50 │ escaping │ escaped-multiline │ │ first line↲second line │ │ │
+ │ 54 │ 51 │ escaping │ backslash │ │ back\slash │ │ │
+ │ 55 │ 52 │ escaping │ quotes │ │ quoted text with "quotes" inside │ │ │
+ │ 56 │ 53 │ escaping │ tab │ │ how↹very │ │ │
+ │ 57 │ 54 │ escaping │ │ │ │ │ ↲↲ │
+ ╰────────────────┴─────────────────┴──────────────────┴─────────────────────┴──────────────────┴──────────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────────────────────────┴─────────────────────╯
+Record count: 49
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/ini-complex.ini Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,58 @@
+global=value
+
+[section][section-tag]
+; section tags are currently supported by the parser (do not generate error)
+; but do not generate any output
+section-tag-support=partial
+
+[quoting]
+
+quoted = "some text in quotes"
+apostrophed = 'some text in apostrophes'
+
+unquoted = values are trimmed
+we-need = " quotes or apostrophes"
+to-keep = 'the leading/trailing whitespace '
+
+[multiline]
+
+quoted = "first line
+second line"
+
+apostrophed = 'first line
+second line'
+
+; unlike quoted/apostrophed strings, there is no line end inside the value:
+backslashed = first line \
+continues here
+
+[comments] ; we can comment also secions
+
+ini = "true" ; and even values
+unix = true
+
+; classic INI comment
+# unix-style comments are also supported
+
+but = be aware that ; this is part of the value, not a comment
+; we need '' or "" to put comment on the same line as the value
+
+[subkeys]
+
+; we may specify subkeys in [] brackets:
+a[x] = AX
+a[y] = AY
+
+; because this feature is quite uncommon, it is disabled by default
+; and whole key+subkey becomes a key
+; but we can turn on the subkey support by the CLI option: --enable-sub-keys true
+; and get separete key and sub_key attributes on the output
+
+[escaping]
+
+escaped-multiline = first line\nsecond line
+backslash = back\\slash
+quotes = "quoted text with \"quotes\" inside"
+tab = how\tvery
+
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/ini-complex.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,25 @@
+ini:
+ ╭──────────────────┬─────────────────────┬──────────────────┬──────────────────────────────────────────────────────────╮
+ │ section (string) │ key (string) │ sub_key (string) │ value (string) │
+ ├──────────────────┼─────────────────────┼──────────────────┼──────────────────────────────────────────────────────────┤
+ │ │ global │ │ value │
+ │ section │ section-tag-support │ │ partial │
+ │ quoting │ quoted │ │ some text in quotes │
+ │ quoting │ apostrophed │ │ some text in apostrophes │
+ │ quoting │ unquoted │ │ values are trimmed │
+ │ quoting │ we-need │ │ quotes or apostrophes │
+ │ quoting │ to-keep │ │ the leading/trailing whitespace │
+ │ multiline │ quoted │ │ first line↲second line │
+ │ multiline │ apostrophed │ │ first line↲second line │
+ │ multiline │ backslashed │ │ first line continues here │
+ │ comments │ ini │ │ true │
+ │ comments │ unix │ │ true │
+ │ comments │ but │ │ be aware that ; this is part of the value, not a comment │
+ │ subkeys │ a │ x │ AX │
+ │ subkeys │ a │ y │ AY │
+ │ escaping │ escaped-multiline │ │ first line↲second line │
+ │ escaping │ backslash │ │ back\slash │
+ │ escaping │ quotes │ │ quoted text with "quotes" inside │
+ │ escaping │ tab │ │ how↹very │
+ ╰──────────────────┴─────────────────────┴──────────────────┴──────────────────────────────────────────────────────────╯
+Record count: 19
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/ini-simple.ini Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,14 @@
+; this entry is a „global property“ – does not belong to any section:
+color=blue
+
+[first-section]
+x = 10
+y = 20
+z = 30
+some = value
+other = VALUE
+
+[second-section]
+x = 100
+y = 200
+z = 300
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/ini-simple.modified.ini Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,14 @@
+; this entry is a „global property“ – does not belong to any section:
+color = blue
+
+[first-section]
+x = 256
+y = 20
+z = 30
+some = value
+other = VALUE
+
+[second-section]
+x = 100
+y = 200
+z = 300
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/ini-simple.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,15 @@
+ini:
+ ╭──────────────────┬──────────────┬────────────────╮
+ │ section (string) │ key (string) │ value (string) │
+ ├──────────────────┼──────────────┼────────────────┤
+ │ │ color │ blue │
+ │ first-section │ x │ 10 │
+ │ first-section │ y │ 20 │
+ │ first-section │ z │ 30 │
+ │ first-section │ some │ value │
+ │ first-section │ other │ VALUE │
+ │ second-section │ x │ 100 │
+ │ second-section │ y │ 200 │
+ │ second-section │ z │ 300 │
+ ╰──────────────────┴──────────────┴────────────────╯
+Record count: 9
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/maven-filesystem-xpath.sh Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# replaces multiple: --option 'env:xmlns_m' 'http://maven.apache.org/POM/4.0.0'
+export xmlns_m='http://maven.apache.org/POM/4.0.0'
+
+find -name pom.xml -print0 \
+ | relpipe-in-filesystem \
+ --relation 'maven' \
+ --file path --as 'module' \
+ --script xpath --option 'env:xpath' '/m:project/m:groupId' --as 'groupId' \
+ --script xpath --option 'env:xpath' '/m:project/m:artifactId' --as 'artifactId' \
+ --script xpath --option 'env:xpath' '/m:project/m:version' --as 'version' \
+ --hash sha256 \
+ | relpipe-out-tabular
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/netplan-1.sh Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+cat netplan.yaml \
+ | relpipe-in-yamltable \
+ --relation 'nic' \
+ --records '/yaml/network/ethernets/*|/yaml/network/bridges/*' \
+ --attribute 'name' string 'name()' \
+ --attribute 'type' string 'name(..)' \
+ --attribute 'dhcp4' boolean 'dhcp4' \
+ --attribute 'dhcp6' boolean 'dhcp6' \
+ --attribute 'accept-ra' boolean 'accept-ra' \
+ --relation 'bridge' \
+ --records '/yaml/network/bridges/*' \
+ --attribute 'name' string 'name()' \
+ --attribute 'gateway4' string 'gateway4' \
+ --attribute 'mtu' string 'mtu' \
+ --relation 'bridge_interface' \
+ --records '/yaml/network/bridges/*/interfaces' \
+ --attribute 'bridge' string 'name(..)' \
+ --attribute 'interface' string '.' \
+ --relation 'ip' \
+ --records '/yaml/network/bridges/*/addresses' \
+ --attribute 'address' string 'substring-before(., "/")' \
+ --attribute 'mask' string 'substring-after(., "/")' \
+ --attribute 'interface' string 'name(..)' \
+ | relpipe-out-tabular
+
+# an early draft of the relational mapping
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/netplan-1.yaml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,26 @@
+network:
+ version: 2
+
+ ethernets:
+ enp84s0:
+ dhcp4: false
+ dhcp6: false
+ accept-ra: false
+
+ bridges:
+ br0:
+ interfaces: [enp84s0, eth0]
+ addresses:
+ - 192.168.1.101/24
+ gateway4: 192.168.1.1
+ nameservers:
+ addresses:
+ - 192.168.1.10
+ - 192.168.1.11
+ mtu: 1500
+ parameters:
+ stp: true
+ forward-delay: 4
+ dhcp4: false
+ dhcp6: false
+ accept-ra: false
--- a/relpipe-data/examples/release-v0.18.sh Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/examples/release-v0.18.sh Mon Feb 21 00:43:11 2022 +0100
@@ -13,14 +13,14 @@
RELPIPE_DEPENDENCIES+=(libzbar-dev libmagick++-dev) # needed only for relpipe-in-barcode and the barcode-reader streamlet
RELPIPE_DEPENDENCIES+=(libgumbo-dev) # needed only for relpipe-in-htmltable
RELPIPE_DEPENDENCIES+=(libyaml-dev) # needed only for relpipe-in-yaml, relpipe-in-yamltable, relpipe-in-json and relpipe-in-jsontable
+ RELPIPE_DEPENDENCIES+=(libcbor-dev) # needed only for relpipe-in-cbortable
RELPIPE_DEPENDENCIES+=(libvmime-dev) # needed only for relpipe-in-mimetable
apt install "${RELPIPE_DEPENDENCIES[@]}"
exit;
fi
# Run rest of installation as a non-root user:
-export RELPIPE_VERSION="v0.18"
-export RELPIPE_VERSION="tip" # FIXME: v0.18
+export RELPIPE_VERSION=${RELPIPE_VERSION:-"v0.18"}
export RELPIPE_SRC=~/src
export RELPIPE_BUILD=~/build
export RELPIPE_INSTALL=~/install
@@ -36,7 +36,7 @@
RELPIPE_LIBS=(lib-common lib-reader lib-writer lib-cli lib-xmlwriter)
RELPIPE_TOOLS=(in-cli in-fstab in-xml in-xmltable in-csv in-filesystem in-recfile out-nullbyte out-ods out-tabular out-xml out-csv out-asn1 out-recfile tr-cut tr-grep tr-sed tr-validator tr-scheme tr-awk tr-sql in-jack out-jack)
-RELPIPE_TOOLS+=(in-asn1table in-barcode in-htmltable in-ini in-initable in-mimetable in-x11 in-yaml in-yamltable out-ini out-x11 out-yaml tr-xpath) # since v0.18
+RELPIPE_TOOLS+=(in-asn1 in-asn1table in-barcode in-cbortable in-htmltable in-ini in-initable in-mimetable in-x11 in-yaml in-yamltable out-ini out-x11 out-yaml tr-infertypes tr-xpath) # since v0.18
# RELPIPE_TOOLS+=(out-gui.qt tr-python) # not enabled by default
@@ -65,5 +65,9 @@
unset -v RELPIPE_BUILD
unset -v RELPIPE_INSTALL
-# List available JACK ports (the JACK daemon should be running):
-relpipe-in-jack --list-ports true --list-midi-messages false | relpipe-out-tabular
+# List mice, keyboards and some fstab entries:
+(relpipe-in-fstab ; relpipe-in-x11 2>/dev/null) \
+ | relpipe-tr-xpath \
+ --relation 'fstab' --where 'pass > 1' \
+ --relation 'x11_input_device' --where 'type = "keyboard" or type = "mouse"' \
+ | relpipe-out-tabular
--- a/relpipe-data/examples/runnable-jars.sh Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/examples/runnable-jars.sh Mon Feb 21 00:43:11 2022 +0100
@@ -16,7 +16,7 @@
}
shortenPath() {
- relpipe-tr-sed '.*' 'path' "^$HOME/?" "~/";
+ relpipe-tr-sed --relation '.*' --attribute 'path' --value "^$HOME/?" --replacement "~/";
}
findFiles | fetchAttributes | filterRunable | shortenPath | relpipe-out-tabular
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/x509-gnu.org.txt Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,82 @@
+validity:
+ ╭───────────────────────────┬───────────────────────────╮
+ │ from (string) │ to (string) │
+ ├───────────────────────────┼───────────────────────────┤
+ │ 2021-12-10T13:52:28+00:00 │ 2022-03-10T13:52:27+00:00 │
+ ╰───────────────────────────┴───────────────────────────╯
+Record count: 1
+
+alternative_name:
+ ╭─────────────────────────────╮
+ │ name (string) │
+ ├─────────────────────────────┤
+ │ archive.gnewsense.org │
+ │ beta.gnewsense.org │
+ │ bloodnok.gnewsense.org │
+ │ bofh.gnewsense.org │
+ │ bugs.gnewsense.org │
+ │ bzr.gnewsense.org │
+ │ cdimage.gnewsense.org │
+ │ classpath.org │
+ │ config.gnewsense.org │
+ │ digitalspeech.org │
+ │ donate.digitalspeech.org │
+ │ dotgnu.org │
+ │ eccles.gnewsense.org │
+ │ emacs.org │
+ │ glibc.gnu.org │
+ │ gnewsense.org │
+ │ gnu.org │
+ │ gnukids.org │
+ │ gplfaq.org │
+ │ hurd.gnu.org │
+ │ ipv6.nongnu.org │
+ │ kindleswindle.org │
+ │ nongnu.org │
+ │ patch-tracker.gnewsense.org │
+ │ playfreedom.org │
+ │ playogg.com │
+ │ playogg.net │
+ │ playogg.org │
+ │ rsync.gnewsense.org │
+ │ seagoon.gnewsense.org │
+ │ security.gnewsense.org │
+ │ smalltalk.gnu.org │
+ │ torrent.gnewsense.org │
+ │ upgradefromwindows.com │
+ │ upgradefromwindows.org │
+ │ upgradefromwindows8.com │
+ │ upgradefromwindows8.org │
+ │ us.archive.gnewsense.org │
+ │ vcdimager.org │
+ │ wiki.gnewsense.org │
+ │ wildebeest.ipv6.gnu.org │
+ │ wildebeest1p.gnu.org │
+ │ www.classpath.org │
+ │ www.digitalspeech.org │
+ │ www.dotgnu.org │
+ │ www.emacs.org │
+ │ www.gnewsense.org │
+ │ www.gnu.org │
+ │ www.gnukids.org │
+ │ www.gplfaq.org │
+ │ www.hurd.gnu.org │
+ │ www.ipv6.gnu.org │
+ │ www.ipv6.nongnu.org │
+ │ www.kindleswindle.org │
+ │ www.nongnu.org │
+ │ www.playfreedom.org │
+ │ www.playogg.com │
+ │ www.playogg.net │
+ │ www.playogg.org │
+ │ www.upgradefromwindows.com │
+ │ www.upgradefromwindows.org │
+ │ www.upgradefromwindows8.com │
+ │ www.upgradefromwindows8.org │
+ │ www.vcdimager.org │
+ │ www6.gnu.org │
+ │ www6.nongnu.org │
+ │ x86-32.gnewsense.org │
+ │ x86-64.gnewsense.org │
+ ╰─────────────────────────────╯
+Record count: 68
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/x509-gnu.org.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,215 @@
+<?xml version="1.0"?>
+<asn1>
+ <sequence tag="16" class="universal">
+ <sequence tag="16" class="universal">
+ <constructed tag="0" class="context">
+ <integer hex="02" tag="2" class="universal">2</integer>
+ </constructed>
+ <integer hex="03abb913df3ec5e5c22f410b01bd46d4f6b7" tag="2" class="universal"/>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">1.2.840.113549.1.1.11</oid>
+ <null tag="5" class="universal"/>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <set tag="17" class="universal">
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.4.6</oid>
+ <text-string tag="19" class="universal">US</text-string>
+ </sequence>
+ </set>
+ <set tag="17" class="universal">
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.4.10</oid>
+ <text-string tag="19" class="universal">Let's Encrypt</text-string>
+ </sequence>
+ </set>
+ <set tag="17" class="universal">
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.4.3</oid>
+ <text-string tag="19" class="universal">R3</text-string>
+ </sequence>
+ </set>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <date-time tag="23" class="universal">2021-12-10T13:52:28+00:00</date-time>
+ <date-time tag="23" class="universal">2022-03-10T13:52:27+00:00</date-time>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <set tag="17" class="universal">
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.4.3</oid>
+ <text-string tag="19" class="universal">wildebeest1p.gnu.org</text-string>
+ </sequence>
+ </set>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">1.2.840.113549.1.1.1</oid>
+ <null tag="5" class="universal"/>
+ </sequence>
+ <bit-string length="2160" tag="3" class="universal">100000000000000010000000110000000100000010110010100110101010101011000010001101001001001101000111111011101100011100011110000011010000010000011010011110000101011001010011110101011111011110111011001001110001010101011111100111110101001001111001100001000101111100111000110111111010000111101010101110001111000001011000101100101111010010011111101101101100110000011110001101111100011011010010001001000100001100010011110101110011001110101010101111110011010011111110000010001101000100100011011111011010000010110001101101011000010111011101001110000011000001110001100100100001111100010101111000110011000100010000101011010010011101001101101100001000010011011011001010000011010110100001010111101011010110101001111110101011011011101110010110101111100111001101111010101111001101000100001001101100001101010111111110100111010011010001101101110111111111001110110010011110101011001110100111000100101101100101011100110001100001011000100000000001001100010001110100110001000000010011111101111111110000000010111100010101000010100111001110000011111001000110011001000111000010101001000000110101011100001110110001111000111010110101101011010110010110110010001100001010111001000111000001000001110011110010111110010101000011010101110110011001001000111101000111101111101100110010101000100111100110000010010111001110001010111101110000010101010011011001100011010011000111001010000000010110001001100011000010010010010110000001110011001011010100101101101111011100110100011111001010110111110101011111101101010111110110110000100001110101100101011011111100101101000100111111101110010100011011101010000100000100001010111111010110110010000100011011110101001101101110001001110000011101001000101001001101111101111100100001100011111101010001111001110111100100101100001110100010001001000101111100010001011010110010000111000000011011101101100010101100001110001110011100011100010110101101000101101100001100010101101110001010111000110000100110010011011101100000011000011101001111111111101100111011001011111001011010101010001101100000101111000001101110110110111101101011010000001101111001110011101110000010000110100001101100101101010101000000001000000010000000010000010100000001010000100000000100000100001100</bit-string>
+ </sequence>
+ <encapsulated tag="3" class="context">
+ <sequence tag="16" class="universal">
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.29.15</oid>
+ <boolean tag="1" class="universal">true</boolean>
+ <encapsulated tag="4" class="universal">
+ <bit-string length="3" tag="3" class="universal">101</bit-string>
+ </encapsulated>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.29.37</oid>
+ <encapsulated tag="4" class="universal">
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">1.3.6.1.5.5.7.3.1</oid>
+ <oid tag="6" class="universal">1.3.6.1.5.5.7.3.2</oid>
+ </sequence>
+ </encapsulated>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.29.19</oid>
+ <boolean tag="1" class="universal">true</boolean>
+ <encapsulated tag="4" class="universal">
+ <sequence tag="16" class="universal"/>
+ </encapsulated>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.29.14</oid>
+ <encapsulated tag="4" class="universal">
+ <octet-string length="20" tag="4" class="universal">9e05272100500e8f5139ae2e3ad87ea30ea08448</octet-string>
+ </encapsulated>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.29.35</oid>
+ <encapsulated tag="4" class="universal">
+ <sequence tag="16" class="universal">
+ <specific length="20" hex="142eb317b75856cbae500940e61faf9d8b14c2c6" tag="0" class="context">.....XV..P.@........</specific>
+ </sequence>
+ </encapsulated>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">1.3.6.1.5.5.7.1.1</oid>
+ <encapsulated tag="4" class="universal">
+ <sequence tag="16" class="universal">
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">1.3.6.1.5.5.7.48.1</oid>
+ <specific length="21" hex="687474703a2f2f72332e6f2e6c656e63722e6f7267" tag="6" class="context">http://r3.o.lencr.org</specific>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">1.3.6.1.5.5.7.48.2</oid>
+ <specific length="22" hex="687474703a2f2f72332e692e6c656e63722e6f72672f" tag="6" class="context">http://r3.i.lencr.org/</specific>
+ </sequence>
+ </sequence>
+ </encapsulated>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.29.17</oid>
+ <encapsulated tag="4" class="universal">
+ <sequence tag="16" class="universal">
+ <specific length="21" hex="617263686976652e676e657773656e73652e6f7267" tag="2" class="context">archive.gnewsense.org</specific>
+ <specific length="18" hex="626574612e676e657773656e73652e6f7267" tag="2" class="context">beta.gnewsense.org</specific>
+ <specific length="22" hex="626c6f6f646e6f6b2e676e657773656e73652e6f7267" tag="2" class="context">bloodnok.gnewsense.org</specific>
+ <specific length="18" hex="626f66682e676e657773656e73652e6f7267" tag="2" class="context">bofh.gnewsense.org</specific>
+ <specific length="18" hex="627567732e676e657773656e73652e6f7267" tag="2" class="context">bugs.gnewsense.org</specific>
+ <specific length="17" hex="627a722e676e657773656e73652e6f7267" tag="2" class="context">bzr.gnewsense.org</specific>
+ <specific length="21" hex="6364696d6167652e676e657773656e73652e6f7267" tag="2" class="context">cdimage.gnewsense.org</specific>
+ <specific length="13" hex="636c617373706174682e6f7267" tag="2" class="context">classpath.org</specific>
+ <specific length="20" hex="636f6e6669672e676e657773656e73652e6f7267" tag="2" class="context">config.gnewsense.org</specific>
+ <specific length="17" hex="6469676974616c7370656563682e6f7267" tag="2" class="context">digitalspeech.org</specific>
+ <specific length="24" hex="646f6e6174652e6469676974616c7370656563682e6f7267" tag="2" class="context">donate.digitalspeech.org</specific>
+ <specific length="10" hex="646f74676e752e6f7267" tag="2" class="context">dotgnu.org</specific>
+ <specific length="20" hex="6563636c65732e676e657773656e73652e6f7267" tag="2" class="context">eccles.gnewsense.org</specific>
+ <specific length="9" hex="656d6163732e6f7267" tag="2" class="context">emacs.org</specific>
+ <specific length="13" hex="676c6962632e676e752e6f7267" tag="2" class="context">glibc.gnu.org</specific>
+ <specific length="13" hex="676e657773656e73652e6f7267" tag="2" class="context">gnewsense.org</specific>
+ <specific length="7" hex="676e752e6f7267" tag="2" class="context">gnu.org</specific>
+ <specific length="11" hex="676e756b6964732e6f7267" tag="2" class="context">gnukids.org</specific>
+ <specific length="10" hex="67706c6661712e6f7267" tag="2" class="context">gplfaq.org</specific>
+ <specific length="12" hex="687572642e676e752e6f7267" tag="2" class="context">hurd.gnu.org</specific>
+ <specific length="15" hex="697076362e6e6f6e676e752e6f7267" tag="2" class="context">ipv6.nongnu.org</specific>
+ <specific length="17" hex="6b696e646c657377696e646c652e6f7267" tag="2" class="context">kindleswindle.org</specific>
+ <specific length="10" hex="6e6f6e676e752e6f7267" tag="2" class="context">nongnu.org</specific>
+ <specific length="27" hex="70617463682d747261636b65722e676e657773656e73652e6f7267" tag="2" class="context">patch-tracker.gnewsense.org</specific>
+ <specific length="15" hex="706c617966726565646f6d2e6f7267" tag="2" class="context">playfreedom.org</specific>
+ <specific length="11" hex="706c61796f67672e636f6d" tag="2" class="context">playogg.com</specific>
+ <specific length="11" hex="706c61796f67672e6e6574" tag="2" class="context">playogg.net</specific>
+ <specific length="11" hex="706c61796f67672e6f7267" tag="2" class="context">playogg.org</specific>
+ <specific length="19" hex="7273796e632e676e657773656e73652e6f7267" tag="2" class="context">rsync.gnewsense.org</specific>
+ <specific length="21" hex="736561676f6f6e2e676e657773656e73652e6f7267" tag="2" class="context">seagoon.gnewsense.org</specific>
+ <specific length="22" hex="73656375726974792e676e657773656e73652e6f7267" tag="2" class="context">security.gnewsense.org</specific>
+ <specific length="17" hex="736d616c6c74616c6b2e676e752e6f7267" tag="2" class="context">smalltalk.gnu.org</specific>
+ <specific length="21" hex="746f7272656e742e676e657773656e73652e6f7267" tag="2" class="context">torrent.gnewsense.org</specific>
+ <specific length="22" hex="7570677261646566726f6d77696e646f77732e636f6d" tag="2" class="context">upgradefromwindows.com</specific>
+ <specific length="22" hex="7570677261646566726f6d77696e646f77732e6f7267" tag="2" class="context">upgradefromwindows.org</specific>
+ <specific length="23" hex="7570677261646566726f6d77696e646f7773382e636f6d" tag="2" class="context">upgradefromwindows8.com</specific>
+ <specific length="23" hex="7570677261646566726f6d77696e646f7773382e6f7267" tag="2" class="context">upgradefromwindows8.org</specific>
+ <specific length="24" hex="75732e617263686976652e676e657773656e73652e6f7267" tag="2" class="context">us.archive.gnewsense.org</specific>
+ <specific length="13" hex="766364696d616765722e6f7267" tag="2" class="context">vcdimager.org</specific>
+ <specific length="18" hex="77696b692e676e657773656e73652e6f7267" tag="2" class="context">wiki.gnewsense.org</specific>
+ <specific length="23" hex="77696c646562656573742e697076362e676e752e6f7267" tag="2" class="context">wildebeest.ipv6.gnu.org</specific>
+ <specific length="20" hex="77696c6465626565737431702e676e752e6f7267" tag="2" class="context">wildebeest1p.gnu.org</specific>
+ <specific length="17" hex="7777772e636c617373706174682e6f7267" tag="2" class="context">www.classpath.org</specific>
+ <specific length="21" hex="7777772e6469676974616c7370656563682e6f7267" tag="2" class="context">www.digitalspeech.org</specific>
+ <specific length="14" hex="7777772e646f74676e752e6f7267" tag="2" class="context">www.dotgnu.org</specific>
+ <specific length="13" hex="7777772e656d6163732e6f7267" tag="2" class="context">www.emacs.org</specific>
+ <specific length="17" hex="7777772e676e657773656e73652e6f7267" tag="2" class="context">www.gnewsense.org</specific>
+ <specific length="11" hex="7777772e676e752e6f7267" tag="2" class="context">www.gnu.org</specific>
+ <specific length="15" hex="7777772e676e756b6964732e6f7267" tag="2" class="context">www.gnukids.org</specific>
+ <specific length="14" hex="7777772e67706c6661712e6f7267" tag="2" class="context">www.gplfaq.org</specific>
+ <specific length="16" hex="7777772e687572642e676e752e6f7267" tag="2" class="context">www.hurd.gnu.org</specific>
+ <specific length="16" hex="7777772e697076362e676e752e6f7267" tag="2" class="context">www.ipv6.gnu.org</specific>
+ <specific length="19" hex="7777772e697076362e6e6f6e676e752e6f7267" tag="2" class="context">www.ipv6.nongnu.org</specific>
+ <specific length="21" hex="7777772e6b696e646c657377696e646c652e6f7267" tag="2" class="context">www.kindleswindle.org</specific>
+ <specific length="14" hex="7777772e6e6f6e676e752e6f7267" tag="2" class="context">www.nongnu.org</specific>
+ <specific length="19" hex="7777772e706c617966726565646f6d2e6f7267" tag="2" class="context">www.playfreedom.org</specific>
+ <specific length="15" hex="7777772e706c61796f67672e636f6d" tag="2" class="context">www.playogg.com</specific>
+ <specific length="15" hex="7777772e706c61796f67672e6e6574" tag="2" class="context">www.playogg.net</specific>
+ <specific length="15" hex="7777772e706c61796f67672e6f7267" tag="2" class="context">www.playogg.org</specific>
+ <specific length="26" hex="7777772e7570677261646566726f6d77696e646f77732e636f6d" tag="2" class="context">www.upgradefromwindows.com</specific>
+ <specific length="26" hex="7777772e7570677261646566726f6d77696e646f77732e6f7267" tag="2" class="context">www.upgradefromwindows.org</specific>
+ <specific length="27" hex="7777772e7570677261646566726f6d77696e646f7773382e636f6d" tag="2" class="context">www.upgradefromwindows8.com</specific>
+ <specific length="27" hex="7777772e7570677261646566726f6d77696e646f7773382e6f7267" tag="2" class="context">www.upgradefromwindows8.org</specific>
+ <specific length="17" hex="7777772e766364696d616765722e6f7267" tag="2" class="context">www.vcdimager.org</specific>
+ <specific length="12" hex="777777362e676e752e6f7267" tag="2" class="context">www6.gnu.org</specific>
+ <specific length="15" hex="777777362e6e6f6e676e752e6f7267" tag="2" class="context">www6.nongnu.org</specific>
+ <specific length="20" hex="7838362d33322e676e657773656e73652e6f7267" tag="2" class="context">x86-32.gnewsense.org</specific>
+ <specific length="20" hex="7838362d36342e676e657773656e73652e6f7267" tag="2" class="context">x86-64.gnewsense.org</specific>
+ </sequence>
+ </encapsulated>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.5.29.32</oid>
+ <encapsulated tag="4" class="universal">
+ <sequence tag="16" class="universal">
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">2.23.140.1.2.1</oid>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">1.3.6.1.4.1.44947.1.1.1</oid>
+ <sequence tag="16" class="universal">
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">1.3.6.1.5.5.7.2.1</oid>
+ <specific length="26" hex="687474703a2f2f6370732e6c657473656e63727970742e6f7267" tag="22" class="universal">http://cps.letsencrypt.org</specific>
+ </sequence>
+ </sequence>
+ </sequence>
+ </sequence>
+ </encapsulated>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">1.3.6.1.4.1.11129.2.4.2</oid>
+ <encapsulated tag="4" class="universal">
+ <octet-string length="243" tag="4" class="universal">00f1007700dfa55eab68824f1f6cadeeb85f4e3e5aeacda212a46a5e8e3b12c020445c2a730000017da4d4c8e00000040300483046022100e459e6392e11771d2425d9ace43e194a9f1776ed5ec8f56db3db661b3a9ac77e022100d6fd9e62b74b86201d0aedc85e31a42fb3b6a3cbd46ef769de3bef49f0cf2b290076002979bef09e393921f056739f63a577e5be577d9c600af8f94d5d265c255dc7840000017da4d4c9540000040300473045022100de46ef3a2991ba373fef55fc6b03e0b813986e1393d5699f7f22ae84350d01d002202a205df263e45819c7d9afa7b0e2298b911414e5f940de94ec43c8cfba3e7659</octet-string>
+ </encapsulated>
+ </sequence>
+ </sequence>
+ </encapsulated>
+ </sequence>
+ <sequence tag="16" class="universal">
+ <oid tag="6" class="universal">1.2.840.113549.1.1.11</oid>
+ <null tag="5" class="universal"/>
+ </sequence>
+ <bit-string length="2048" tag="3" class="universal">11111111011111011010000011000111111100100110100101111000100011011011101111101100100011110111000100000000010010101011010111010001100001110000010010011010110101000101000000011111101110000111111000010110001000011101101101110110101001001001100001111011111100111100110110100010111000100001101111011100100011100101100101011110111010010110000101011111100101100100011011011011000111110000011111000001001110111111111111110111110110111011001000110000111001010100111010010000001100111001011000001001110100110000111010000011100111011100010110011011101111100101101111111001010000000100110011110010000101101011011011001111010100000110000110100001111011001001000000001011110110100111000111100010111111110111111111001010010001101000111100001111101111001010010011101100010111001100010001111101101001101100110000001010000100101001110001000010101010111000000001110011001100110100010101100111111001010001111110000101011000001100110110101100000000100101000000101011010000101011000111110000111011000011001001000001000110111001100011111100110011110101111101000010101100010000110011101111111100100011111010001111110110110110100011011101111111110001000010011101000000001100101010010000010011101101101110100010101000011010011111100111111011011111011001111100010101110000100011100110000111111111110011011011110001101001001001110100001110001000001001111010100000001010011111011111001110101100101001011011110100111111010001010011110000111001110010101110101011011101101101001101010000100001111111110100011011000100011101000010001011001010110110101110011010010110001101110110111100111110111101000111011111010000110010110100011110001000101011100010110100011110010001101001010010101111100011100110011110110010100101100010010110000101100110110010100110100000110011010011101100001111111110110011010011100001011001010010110000110101011001000011001010111010100111011110011010011000000010010111111001011001101000011110100100011000100001010111010010010101011111101000011110000001011001110110010110010011100000001000000111110100110100011100011010100010110111110101100000110001000110101010</bit-string>
+ </sequence>
+</asn1>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/xpath-maven-1.tabular Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,10 @@
+module:
+ ╭─────────────────────────────────────┬────────────────────────┬──────────────────────┬──────────────────╮
+ │ path (string) │ group_id (string) │ artifact_id (string) │ version (string) │
+ ├─────────────────────────────────────┼────────────────────────┼──────────────────────┼──────────────────┤
+ │ ./java/jdbc-dk-driver/pom.xml │ info.globalcode.sql.dk │ jdbc-dk-driver │ 0.11-SNAPSHOT │
+ │ ./java/jdbc-loopback-driver/pom.xml │ info.globalcode.sql.dk │ jdbc-loopback-driver │ 0.11-SNAPSHOT │
+ │ ./java/sql-dk/pom.xml │ info.globalcode.sql.dk │ sql-dk │ 0.11-SNAPSHOT │
+ │ ./java/pom.xml │ info.globalcode.sql.dk │ sql-dk-parent │ 0.11-SNAPSHOT │
+ ╰─────────────────────────────────────┴────────────────────────┴──────────────────────┴──────────────────╯
+Record count: 4
Binary file relpipe-data/img/barcode-qr-IMG_5758.jpeg has changed
Binary file relpipe-data/img/csv-sql-gui-ip-address-counts.png has changed
Binary file relpipe-data/img/wmaker-yaml-xml-tabular-1.png has changed
Binary file relpipe-data/img/xcalc-x11-embedding-1.png has changed
Binary file relpipe-data/img/xcalc-x11-embedding-2.png has changed
--- a/relpipe-data/implementation.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/implementation.xml Mon Feb 21 00:43:11 2022 +0100
@@ -14,14 +14,25 @@
<m:tabulka>
name type subtype language license
+ relpipe-in-asn1.cpp executable input c++ GNU GPLv3
+ relpipe-in-asn1table.cpp executable input c++ GNU GPLv3
+ relpipe-in-barcode.cpp executable input c++ GNU GPLv3
+ relpipe-in-cbortable.cpp executable input c++ GNU GPLv3
relpipe-in-cli.cpp executable input c++ GNU GPLv3
relpipe-in-csv.cpp executable input c++ GNU GPLv3
relpipe-in-filesystem.cpp executable input c++ GNU GPLv3
relpipe-in-fstab.cpp executable input c++ GNU GPLv3
+ relpipe-in-htmltable.cpp executable input c++ GNU GPLv3
+ relpipe-in-ini.cpp executable input c++ GNU GPLv3
+ relpipe-in-initable.cpp executable input c++ GNU GPLv3
relpipe-in-jack.cpp executable input c++ GNU GPLv3
+ relpipe-in-mimetable.cpp executable input c++ GNU GPLv3
relpipe-in-recfile.cpp executable input c++ GNU GPLv3
relpipe-in-xml.cpp executable input c++ GNU GPLv3
relpipe-in-xmltable.cpp executable input c++ GNU GPLv3
+ relpipe-in-x11.cpp executable input c++ GNU GPLv3
+ relpipe-in-yaml.cpp executable input c++ GNU GPLv3
+ relpipe-in-yamltable.cpp executable input c++ GNU GPLv3
relpipe-lib-cli.cpp library header-only c++ GNU GPLv3
relpipe-lib-common.cpp library shared c++ GNU LGPLv3 or GPLv2
relpipe-lib-reader.cpp library shared c++ GNU LGPLv3 or GPLv2
@@ -30,19 +41,24 @@
relpipe-out-asn1.cpp executable output c++ GNU GPLv3
relpipe-out-csv.cpp executable output c++ GNU GPLv3
relpipe-out-gui.qt.cpp executable output c++ GNU GPLv3
+ relpipe-out-ini.qt.cpp executable output c++ GNU GPLv3
relpipe-out-nullbyte.cpp executable output c++ GNU GPLv3
relpipe-out-ods.cpp executable output c++ GNU GPLv3
relpipe-out-recfile.cpp executable output c++ GNU GPLv3
relpipe-out-tabular.cpp executable output c++ GNU GPLv3
relpipe-out-xml.cpp executable output c++ GNU GPLv3
+ relpipe-out-x11.cpp executable output c++ GNU GPLv3
+ relpipe-out-yaml.cpp executable output c++ GNU GPLv3
relpipe-tr-awk.cpp executable transformation c++ GNU GPLv3
relpipe-tr-cut.cpp executable transformation c++ GNU GPLv3
relpipe-tr-grep.cpp executable transformation c++ GNU GPLv3
- relpipe-tr-scheme.cpp executable transformation c++ GNU GPLv3
+ relpipe-tr-infertypes.cpp executable transformation c++ GNU GPLv3
relpipe-tr-python.cpp executable transformation c++ GNU GPLv3
relpipe-tr-sed.cpp executable transformation c++ GNU GPLv3
+ relpipe-tr-scheme.cpp executable transformation c++ GNU GPLv3
relpipe-tr-sql.cpp executable transformation c++ GNU GPLv3
relpipe-tr-validator.cpp executable transformation c++ GNU GPLv3
+ relpipe-tr-xpath.cpp executable transformation c++ GNU GPLv3
</m:tabulka>
<!--
relpipe-web website - XWG XML GNU FDLv1.3+
--- a/relpipe-data/index.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/index.xml Mon Feb 21 00:43:11 2022 +0100
@@ -136,7 +136,7 @@
</p>
- <h2>What <m:name/> are not?</h2>
+ <h2 id="not">What <m:name/> are not?</h2>
<p>
<m:name/> respect the existing ecosystem and are rather an improvement or supplement than a replacement.
--- a/relpipe-data/principles.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/principles.xml Mon Feb 21 00:43:11 2022 +0100
@@ -103,7 +103,7 @@
-->
</p>
- <h2>Optional complexity</h2>
+ <h2 id="optional_complexity">Optional complexity</h2>
<p>
We are not scared by things like XML, SQL, RDF, Java or even C++ and we do not hate them.
--- a/relpipe-data/release-v0.11.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/release-v0.11.xml Mon Feb 21 00:43:11 2022 +0100
@@ -40,7 +40,7 @@
<p>
Please note that this is still a development release and thus the API (libraries, CLI arguments, formats) might and will change.
- Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mailing list</m:a>.
+ Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mail box</m:a>.
</p>
<h3>Data types</h3>
--- a/relpipe-data/release-v0.12.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/release-v0.12.xml Mon Feb 21 00:43:11 2022 +0100
@@ -38,7 +38,7 @@
<p>
Please note that this is still a development release and thus the API (libraries, CLI arguments, formats) might and will change.
- Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mailing list</m:a>.
+ Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mail box</m:a>.
</p>
<h3>Data types</h3>
--- a/relpipe-data/release-v0.13.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/release-v0.13.xml Mon Feb 21 00:43:11 2022 +0100
@@ -53,7 +53,7 @@
<p>
Please note that this is still a development release and thus the API (libraries, CLI arguments, formats) might and will change.
- Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mailing list</m:a>.
+ Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mail box</m:a>.
</p>
<h3>Data types</h3>
--- a/relpipe-data/release-v0.14.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/release-v0.14.xml Mon Feb 21 00:43:11 2022 +0100
@@ -55,7 +55,7 @@
<p>
Please note that this is still a development release and thus the API (libraries, CLI arguments, formats) might and will change.
- Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mailing list</m:a>.
+ Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mail box</m:a>.
</p>
<h3>Data types</h3>
--- a/relpipe-data/release-v0.15.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/release-v0.15.xml Mon Feb 21 00:43:11 2022 +0100
@@ -33,7 +33,7 @@
<p>
Please note that this is still a development release and thus the API (libraries, CLI arguments, formats) might and will change.
- Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mailing list</m:a>.
+ Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mail box</m:a>.
</p>
<h2>Streamlets</h2>
--- a/relpipe-data/release-v0.16.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/release-v0.16.xml Mon Feb 21 00:43:11 2022 +0100
@@ -29,7 +29,7 @@
<p>
Please note that this is still a development release and thus the API (libraries, CLI arguments, formats) might and will change.
- Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mailing list</m:a>.
+ Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mail box</m:a>.
</p>
<h2>ODBC in the SQL transformation module</h2>
--- a/relpipe-data/release-v0.17.1.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/release-v0.17.1.xml Mon Feb 21 00:43:11 2022 +0100
@@ -43,7 +43,7 @@
<p>
Please note that this is still a development release and thus the API (libraries, CLI arguments, formats) might and will change.
- Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mailing list</m:a>.
+ Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mail box</m:a>.
</p>
<h2>New CLI interfaces</h2>
--- a/relpipe-data/release-v0.17.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/release-v0.17.xml Mon Feb 21 00:43:11 2022 +0100
@@ -43,7 +43,7 @@
<p>
Please note that this is still a development release and thus the API (libraries, CLI arguments, formats) might and will change.
- Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mailing list</m:a>.
+ Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mail box</m:a>.
</p>
<h2>New CLI interfaces</h2>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/release-v0.18.xml Mon Feb 21 00:43:11 2022 +0100
@@ -0,0 +1,301 @@
+<stránka
+ xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+ xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+
+ <nadpis>Release v0.18</nadpis>
+ <perex>new public release of Relational pipes</perex>
+ <m:release>v0.18</m:release>
+
+ <text xmlns="http://www.w3.org/1999/xhtml">
+ <p>
+ We are pleased to introduce you the new development version of <m:name/>.
+ This release brings several new tools of each type (inputs, outputs, transformations and streamlet) and some smaller improvements.
+ </p>
+
+ <ul>
+ <li>
+ New <strong>input tools</strong> for reading various formats:
+ read <strong>ASN.1</strong>, <strong>YAML</strong> and <strong>JSON</strong>; see details below
+ </li>
+ <li>
+ New <strong>XMLTable-like input tools</strong> for converting arbitrary tree structures to relations:
+ read <strong>ASN.1</strong>, <strong>INI</strong>, <strong>MIME</strong>, <strong>YAML</strong>, <strong>JSON</strong>, <strong>CBOR</strong> and <strong>HTML</strong>; see details below
+ </li>
+ <li>
+ New <strong>input tool and streamlet</strong> for reading <strong>barcodes</strong> and <strong>QR codes</strong>: see details below
+ </li>
+ <li>
+ New <strong>input and output tool</strong> for the <strong>INI</strong> format: see details below
+ </li>
+ <li>
+ New <strong>input and output tool</strong> for interacting with <strong>X11</strong>: see details below
+ </li>
+ <li>
+ New <strong>XPath transformation</strong>: see details below
+ </li>
+ <li>
+ <strong>Data types</strong> support on <strong>CSV</strong> input and output: strings, integers and booleans; see details below
+ </li>
+ <li>
+ The <strong>grep</strong>, <strong>cut</strong> and <strong>sed</strong> transformations have new CLI interface: see details below
+ </li>
+ </ul>
+
+ <p>
+ See the (<a href="#newExamples">new</a>) <m:a href="examples">examples</m:a> and (<a href="#newScreenshots">new</a>) <m:a href="screenshots">screenshots</m:a> pages for details.
+ </p>
+
+ <p>
+ Please note that this is still a development release and thus the API (libraries, CLI arguments, formats) might and will change.
+ Any suggestions, ideas and bug reports are welcome in our <m:a href="contact">mail box</m:a>.
+ </p>
+
+ <h2 id="standardInputModules">New standard input modules</h2>
+
+ <p>
+ These tools allows serialization of arbitrary relational data
+ and then reading resulting file or stream back to the relational data.
+ An of course, we can read data created or generated somewhere else.
+ Since this release, we can newly serialize our relations to YAML and ASN.1, store, edit or transfer these files or streams and then deserialize them and pipe them through relational transformations or outputs.
+ Formats like XML or Recfiles were already supported in previous releases.
+ Full lossless round-trip is now possible with: XML, YAML, ASN.1 and Recfile formats.
+ Partial lossless round-trip is possible with CSV – these input and output tools are limited to a single relation.
+ Lossy round-trip is possible with <code>relpipe-out-nullbyte</code> and <code>relpipe-in-cli</code> –
+ the <i>nullbyte</i> stream may contain only a single relation and its data types need to be passed as CLI arguments to the input tool.
+ Other formats (e.g. JACK, INI or X11) may also have corresponding inputs and outputs, but they produce and consume some specific structures – they are not universal and usable with arbitrary relational data.
+ </p>
+
+
+ <h3 id="YAML-JSON">YAML and JSON</h3>
+
+ <p>
+ YAML is a text format for serializing tree structures into text (like XML).
+ It can be sometimes tricky, but basic scenarios are intuitive and straightforward.
+ We can use YAML to describe relational data and load them using the <code>relpipe-in-yaml</code> input filter
+ and we can produce YAML files using the <code>relpipe-out-yaml</code> output filter.
+ </p>
+
+ <p>More details in the example: <m:a href="examples-yaml">Reading and writing YAML</m:a></p>
+
+ <h3>ASN.1</h3>
+ <p>
+ There are actually more ASN.1 applications around us than most people realize.
+ ASN.1 is almost everywhere – in cryptography (X.509 / TLS / HTTP, S/MIME etc.), LDAP, payment cards, telecommunications (including GSM, GPRS, EDGE, UMTS, LTE, 5G), Kerberos authentication etc.
+ So it is very useful to be able to access these data.
+ Because ASN.1 is abstract, there are several ways how such data can be serialized into a stream of octets (bytes).
+ Most common way is BER (Basic Encoding Rules), so we support BER (and thus its variants DER and CER) in the first version of our tools.
+ Later other encoding might be supported and we could also parse the ASN.1 syntax (the schema language).
+ </p>
+ <p>
+ ASN.1 BER output was added several releases ago.
+ Now we have also ASN.1 BER input (<code>relpipe-in-asn1</code>) so we can read data in this format generated earlier.
+ Of course, BER data can be generated or read also by other ASN.1 capable software.
+ However, maybe more interesting than <code>relpipe-in-asn1</code> is <code>relpipe-in-asn1table</code> (see <a href="#XMLTable">below</a>) that reads arbitrary ASN.1 BER data (not only serialized relational data).
+ </p>
+
+ <h2 id="INI">Reading and writing INI</h2>
+ <p>
+ INI is very common simple text format used for configuration files.
+ It extends classic <i>key=value</i> config files and adds one more level – sections – for structuring.
+ In this release we got input and output filter for INI and similar formats (Java .properties, MANIFEST.MF, key=value configs).
+ Because comments and whitespace are also supported, we can do (almost) loss-less conversions and transformations (e.g. change value of an entry).
+ </p>
+
+ <p>More details in the example: <m:a href="examples-ini">Reading and writing INI and unix configs</m:a></p>
+
+ <h2>Reading barcodes and QR codes</h2>
+ <p>
+ Either ubiquitous 1D barcodes (procudct labels, ISBN in form of EAN-13 etc.) or 2D barcodes (QR containing hyperlinks, vCards etc.)
+ encode numbers, texts or other data into images that can be printed and scanned or photographed later.
+ Now we can interact with this technology – read barcodes using the
+ <code>relpipe-in-barcode</code> tool or <code>barcode-reader</code> streamlet.
+ </p>
+ <p>More details in the example: <m:a href="examples-barcode">Reading barcodes and QR</m:a></p>
+
+ <h2 id="XMLTable">New XMLTable-like input modules: ASN.1, INI, MIME, YAML, JSON, CBOR, HTML</h2>
+ <p>
+ More details in the examples:
+ </p>
+ <ul>
+ <li><m:a href="examples-reading-querying-uniform-way">Reading and querying JSON, YAML, CBOR, HTML, MIME, INI, ASN.1 and XML in a uniform way</m:a></li>
+ <li><m:a href="examples-asn1-x509">Exploring content of X.509 certificates</m:a></li>
+ </ul>
+
+ <h2 id="X11">X11 input and output modules</h2>
+ <p>
+ The X Window System or X11 comes from 1984 and is still widely used (despite we have some other options like Wayland).
+ This protocol and set of libraries and interfaces gives us GUI (graphical user interface), manages our displays, windows, keyboards and mice.
+ Since this release of <m:name/> we can interact with this wonderful technology through the <code>relpipe-in-x11</code> and <code>relpipe-out-x11</code> tools.
+ In the following example we will show how to acquire information about the input devices, screens and windows or capture and emit X11 events (key presses and mouse movements).
+ </p>
+ <p>More details in the example: <m:a href="examples-x11-basics">Exploring X11 windows and devices and emulating mouse movements and keystrokes</m:a></p>
+
+ <h2 id="XPath">XPath transformation</h2>
+ <p>
+ We got a new powerful language for filtering and transformations: XPath.
+ It is now part of the toolset consisting of SQL, AWK, Scheme and others.
+ However XPath is originally a language designed for XML, in <m:name/> we can use it for relational data coming from various sources, not only XML,
+ and also for data that violates the rules of normal forms.
+ We can process quite complex tree structures entangled in records but we can also write simple and intuitive expressions like <code>x = "a" or y = 123</code>.
+ </p>
+ <p>More details in the example: <m:a href="examples-xpath-filtering-transforming">Filtering and transforming relational data with XPath</m:a></p>
+
+ <h2 id="csvDataTypes">Data types in CSV + generic relpipe-tr-infertypes tool</h2>
+ <p>
+ CSV input and output filters now support data types i.e. CSV can now carry not only text strings, but also booleans and integers (or more types in future releases).
+ We also have a new tool <code>relpipe-tr-infertypes</code> that can automatically recognize the types in data that came from CSV or any other source.
+ </p>
+ <p>More details in the example: <m:a href="examples-csv-data-types">CSV and data types</m:a></p>
+
+ <h2>Feature overview</h2>
+
+ <h3>Data types</h3>
+ <ul>
+ <li m:since="v0.8">boolean</li>
+ <li m:since="v0.15">variable-length signed integer (SLEB128)</li>
+ <li m:since="v0.8">string in UTF-8</li>
+ </ul>
+ <h3>Inputs</h3>
+ <ul>
+ <li m:since="v0.11">Recfile</li>
+ <li m:since="v0.9">XML</li>
+ <li m:since="v0.13">XMLTable</li>
+ <li m:since="v0.9">CSV</li>
+ <li m:since="v0.9">file system</li>
+ <li m:since="v0.8">CLI</li>
+ <li m:since="v0.8">fstab</li>
+ <li m:since="v0.14">SQL script</li>
+ <li m:since="v0.16">JACK</li>
+ <li m:since="v0.18">barcode</li>
+ <li m:since="v0.18">X11</li>
+ <li m:since="v0.18">ASN.1 BER</li>
+ <li m:since="v0.18">ASN.1 BER Table</li>
+ <li m:since="v0.18">INI</li>
+ <li m:since="v0.18">INITable</li>
+ <li m:since="v0.18">MIMETable</li>
+ <li m:since="v0.18">YAML</li>
+ <li m:since="v0.18">YAMLTable</li>
+ <li m:since="v0.18">JSON</li>
+ <li m:since="v0.18">JSONTable</li>
+ <li m:since="v0.18">CBORTable</li>
+ <li m:since="v0.18">HTMLTable</li>
+ </ul>
+ <h3>Transformations</h3>
+ <ul>
+ <li m:since="v0.13">sql: filtering and transformations using the SQL language</li>
+ <li m:since="v0.18">xpath: filtering and transformations using the XPath language</li>
+ <li m:since="v0.12">awk: filtering and transformations using the classic AWK tool and language</li>
+ <li m:since="v0.10">scheme: filtering and transformations defined in the Scheme language using GNU Guile</li>
+ <li m:since="v0.8">grep: regular expression filter, removes unwanted records from the relation</li>
+ <li m:since="v0.8">cut: regular expression attribute cutter (removes or duplicates attributes and can also DROP whole relation)</li>
+ <li m:since="v0.8">sed: regular expression replacer</li>
+ <li m:since="v0.8">validator: just a pass-through filter that crashes on invalid data</li>
+ <li m:since="v0.18">infertypes: derive data types from attribute values</li>
+ <li m:since="v0.8">python: highly experimental</li>
+ </ul>
+ <h3>Streamlets</h3>
+ <ul>
+ <li m:since="v0.15">xpath (example, unstable)</li>
+ <li m:since="v0.15">hash (example, unstable)</li>
+ <li m:since="v0.15">jar_info (example, unstable)</li>
+ <li m:since="v0.15">mime_type (example, unstable)</li>
+ <li m:since="v0.15">exiftool (example, unstable)</li>
+ <li m:since="v0.15">pid (example, unstable)</li>
+ <li m:since="v0.15">cloc (example, unstable)</li>
+ <li m:since="v0.15">exiv2 (example, unstable)</li>
+ <li m:since="v0.15">inode (example, unstable)</li>
+ <li m:since="v0.15">lines_count (example, unstable)</li>
+ <li m:since="v0.15">pdftotext (example, unstable)</li>
+ <li m:since="v0.15">pdfinfo (example, unstable)</li>
+ <li m:since="v0.15">tesseract (example, unstable)</li>
+ <li m:since="v0.18">barcode (example, unstable)</li>
+ </ul>
+ <h3>Outputs</h3>
+ <ul>
+ <li m:since="v0.11">ASN.1 BER</li>
+ <li m:since="v0.11">Recfile</li>
+ <li m:since="v0.9">CSV</li>
+ <li m:since="v0.8">tabular</li>
+ <li m:since="v0.8">XML</li>
+ <li m:since="v0.8">nullbyte</li>
+ <li m:since="v0.8">GUI in Qt</li>
+ <li m:since="v0.8">ODS (LibreOffice)</li>
+ <li m:since="v0.17">JACK</li>
+ <li m:since="v0.18">X11</li>
+ <li m:since="v0.18">INI</li>
+ <li m:since="v0.18">YAML</li>
+ </ul>
+
+ <h2 id="newExamples">New examples</h2>
+ <ul>
+ <li><m:a href="examples-reading-querying-uniform-way">Reading and querying JSON, YAML, CBOR, HTML, MIME, INI, ASN.1 and XML in a uniform way</m:a></li>
+ <li><m:a href="examples-xpath-filtering-transforming">Filtering and transforming relational data with XPath</m:a></li>
+ <li><m:a href="examples-csv-data-types">CSV and data types</m:a></li>
+ <li><m:a href="examples-x11-basics">Exploring X11 windows and devices and emulating mouse movements and keystrokes</m:a></li>
+ <li><m:a href="examples-asn1-x509">Exploring content of X.509 certificates</m:a></li>
+ <li><m:a href="examples-csv-sql-join">Running SQL JOINs on multiple CSV files</m:a></li>
+ <li><m:a href="examples-yaml">Reading and writing YAML</m:a></li>
+ <li><m:a href="examples-ini">Reading and writing INI and unix configs</m:a></li>
+ <li><m:a href="examples-barcode">Reading barcodes and QR</m:a></li>
+ <!--
+ <li><m:a href="examples-x11-kvm">Software replacement of KVM switch</m:a></li>
+ -->
+ </ul>
+
+ <h2 id="newScreenshots">New screenshots</h2>
+
+ <m:img src="img/wmaker-yaml-xml-tabular-1.png"/>
+
+ <h2 id="incompatibleChanges">Backward incompatible changes</h2>
+
+ <p>
+ In <code>relpipe-tr-sql</code> and <code>relpipe-in-sql</code> the <code>--list-data-sources</code> option has now boolean parameter
+ i.e. it is not the presence of the option but the <code>true</code> or <code>false</code> value what matters.
+ We can also list the data sources while we simultaneously run some SQL.
+ </p>
+
+ <p>
+ The <code>relpipe-tr-grep</code>, <code>relpipe-tr-cut</code> and <code>relpipe-tr-sed</code>
+ were the last tools with the obsolete positional CLI interface.
+ In this release, they were updated and now share the same style of CLI arguments as other tools.
+ Besides consistent and predictable user interface, these tools are now more powerful because they can transforma more relations at once.
+ </p>
+
+ <p>Instead of e.g.</p>
+ <pre>relpipe-tr-grep 'a' 'b' 'c'</pre>
+ <p>we now write:</p>
+ <pre>relpipe-tr-grep --relation 'a' --attribute 'b' --value 'c'</pre>
+ <p>Bash completion helps us while writing such commands and result is much more readable than original cryptic version.</p>
+
+ <p>See updated examples:</p>
+
+ <ul>
+ <li><m:a href="examples-rename-vg-fstab">Renaming VG in /etc/fstab using relpipe-tr-sed</m:a></li>
+ <li><m:a href="examples-rename-groups-backreferences">Using relpipe-tr-sed with groups and backreferences</m:a></li>
+ <li><m:a href="examples-grep-fstab">Filtering /etc/fstab using relpipe-tr-grep</m:a></li>
+ <li><m:a href="examples-grep-cut-fstab">Doing projection and restriction using cut and grep</m:a></li>
+ <li><m:a href="examples-apt">Reading apt (Debian package system) results</m:a></li>
+ <li><m:a href="examples-in-xmltable-ssm-gui">Generating statistics and charts using XMLTable</m:a></li>
+ <li><m:a href="streamlets-preview">Streamlets preview</m:a></li>
+ <li><m:a href="examples-csv-sql-join">Running SQL JOINs on multiple CSV files</m:a></li>
+ <li><m:a href="examples-runnable-jars">Finding runnable JARs</m:a></li>
+ </ul>
+
+
+ <h2 id="installation">Installation</h2>
+
+ <p>
+ Instalation was tested on Debian GNU/Linux 10.2.
+ The process should be similar on other distributions.
+ </p>
+
+ <m:pre src="examples/release-v0.18.sh" jazyk="bash" odkaz="ano"/>
+
+ <p>
+ <m:name/> are modular thus you can download and install only parts you need (the libraries are needed always).
+ Tools <code>out-gui.qt</code> and <code>tr-python</code> require additional libraries and are not built by default.
+ </p>
+
+ </text>
+
+</stránka>
\ No newline at end of file
--- a/relpipe-data/roadmap.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/roadmap.xml Mon Feb 21 00:43:11 2022 +0100
@@ -16,12 +16,19 @@
Released versions are described on the <m:a href="download">download</m:a> page.
</p>
- <h2>v0.17, v0.18, v0.19 etc.</h2>
+ <h2>v0.19, v0.20, v0.21 etc.</h2>
<p>
Releases for discussion and verification of the format and API design.
</p>
+ <p>
+ This phase (before v1.0.0) might seem quite long.
+ But it is important to verify the ideas and design in various scenarios, on various use cases.
+ The general idea and the big picture are quite clear and stable.
+ However there are many technical details that need to be carefully tuned.
+ </p>
+
<h3>Data types</h3>
<ul>
<li>fractions</li>
@@ -59,6 +66,7 @@
<li>publish documentation of the stable API and file format</li>
<li>publish automated complex tests (specification vs. implementation compliance)</li>
<li>verify the format from the performance point of view</li>
+ <li>relpipe-lib-writer: several modes of output buffering (auto, relation, record, value) or manual control</li>
<li>improve parsing (corrupted input may currently lead to huge memory allocations), more fuzzing</li>
<li>code clean-up and refactoring, move some reusable parts to common libraries</li>
<li>test the build with another compiler and tune the code</li>
--- a/relpipe-data/screenshots.xml Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/screenshots.xml Mon Feb 21 00:43:11 2022 +0100
@@ -32,6 +32,9 @@
<m:img src="img/kde-fstab-ods-libreoffice-1.png"/>
<m:img src="img/kde-wireshark-asn1-1.png"/>
+ <h2>GNU/Linux and Window Maker</h2>
+ <m:img src="img/wmaker-yaml-xml-tabular-1.png"/>
+
<h2>Haiku OS</h2>
<m:img src="img/haiku-v0.8-1.png"/>
<!--
--- a/relpipe-data/zápatí.inc Fri Nov 26 22:14:18 2021 +0100
+++ b/relpipe-data/zápatí.inc Mon Feb 21 00:43:11 2022 +0100
@@ -1,7 +1,7 @@
<stránka xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana">
<text xmlns="http://www.w3.org/1999/xhtml">
<p>
- Relational pipes, open standard and <a href="https://www.gnu.org/philosophy/free-sw.html">free software</a> © 2018-2020 GlobalCode
+ Relational pipes, open standard and <a href="https://www.gnu.org/philosophy/free-sw.html">free software</a> © 2018-2022 GlobalCode
</p>
</text>
</stránka>
