# HG changeset patch
# User František Kučera <franta-hg@frantovo.cz>
# Date 1577546381 -3600
# Node ID 5cf3a702f47d11e875c2882e4b4aeaafd2ea8915
# Parent  780d782cdef5db5dec795b4b44075aa86d74f406
examples: AWKing through a XML file

diff -r 780d782cdef5 -r 5cf3a702f47d relpipe-data/examples-awk-through-xml.xml
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples-awk-through-xml.xml	Sat Dec 28 16:19:41 2019 +0100
@@ -0,0 +1,55 @@
+<stránka
+	xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
+	xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
+	
+	<nadpis>AWKing through a XML file</nadpis>
+	<perex>use AWK to filter data from a D-Bus policy file</perex>
+	<m:pořadí-příkladu>03700</m:pořadí-příkladu>
+
+	<text xmlns="http://www.w3.org/1999/xhtml">
+		
+		<p>
+			The <a href="https://en.wikipedia.org/wiki/AWK">AWK</a> programming language and tool is designed for processing text
+			but in <m:name/> we can use all the power of AWK for processing relational data that came from various sources.
+			And „all the power“ means that we internally call the regular AWK implementation installed in our system (which is usually GNU AWK).
+		</p>
+		
+		<p>
+			In this example, we will read a D-Bus policy configuration file (an XML)
+			and filter it using a simple AWK expression.
+			The XML file looks like this:
+		</p>
+		
+		<m:pre jazyk="xml" src="examples/awk-through-xml.xml"/>
+		
+		<p>
+			We separate particular steps of our pipeline to shell functions for better readability and maintenance:
+		</p>
+	
+		<m:pre jazyk="bash" src="examples/awk-through-xml.sh"/>
+		
+		<p>
+			This simple sequence of steps will generate following result:
+		</p>
+		
+		<m:pre jazyk="text" src="examples/awk-through-xml.txt"/>
+		
+		<p>
+			We have processed two relations.
+			One (<code>limit</code>) was just passed through
+			while another (<code>policy</code>) was transformed.
+			In this case, we did just restriction (filtered certain records), but we can do also projections or other relational operations in our pipelines.
+		</p>
+		
+		<p>
+			This example also shows how we can structure our scripts – compose the pipeline from reusable fragments (shell functions).
+			Once we write the <em>parse</em> function for an input format, we can use it many times, with various filters/transformations and output formatters.
+		</p>
+		<p>
+			Instead of converting to an output format, we can also <m:a href="examples-out-bash">execute arbitrary system commands</m:a> for each record 
+			(while having particular attribute values reliably separated in shell variables).
+		</p>
+		
+	</text>
+
+</stránka>
diff -r 780d782cdef5 -r 5cf3a702f47d relpipe-data/examples-in-xmltable-xhtml-outline.xml
--- a/relpipe-data/examples-in-xmltable-xhtml-outline.xml	Tue Dec 17 00:53:04 2019 +0100
+++ b/relpipe-data/examples-in-xmltable-xhtml-outline.xml	Sat Dec 28 16:19:41 2019 +0100
@@ -48,7 +48,7 @@
  │ h2             │ Oprava – výměna spínače            │
  │ h2             │ Myš Razer Orochi                   │
  │ h2             │ Trackball Logitech TrackMan Marble │
- │ h2             │ Myší spínače Omron a⎵Panasonic     │
+ │ h2             │ Myší spínače Omron a Panasonic     │
  │ h2             │ Závěr                              │
  ╰────────────────┴────────────────────────────────────╯
 Record count: 8]]></m:pre>
diff -r 780d782cdef5 -r 5cf3a702f47d relpipe-data/examples/awk-through-xml.sh
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/awk-through-xml.sh	Sat Dec 28 16:19:41 2019 +0100
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+getXML() {
+	# Just read a file:
+	cat "/etc/dbus-1/system.d/org.freedesktop.NetworkManager.conf"
+	# Or we can wget or curl it from the network
+	# or generate on-the-fly using some other command.
+}
+
+parseXML() {
+	# Convert the XML tree structure into multiple relations.
+	# Use XPath expressions to find record nodes
+	# and to find attributes inside them.
+	relpipe-in-xmltable \
+		--relation 'policy' \
+			--records '/busconfig/policy/allow|/busconfig/policy/deny' \
+			--attribute 'policy'           string 'name()' \
+			--attribute 'user'             string '../@user' \
+			--attribute 'context'          string '../@context' \
+			--attribute 'own'              string '@own' \
+			--attribute 'send_destination' string '@send_destination' \
+			--attribute 'send_interface'   string '@send_interface' \
+			--attribute 'send_member'      string '@send_member' \
+		--relation 'limit' \
+			--records '/busconfig/limit' \
+			--attribute 'name'             string '@name' \
+			--attribute 'value'            integer '.'
+}
+
+filterRecords() {
+	# Use native AWK command (called from relpipe-tr-awk)
+	# with all its power to filter the records.
+	# Work with named fields instead of numbered columns.
+	relpipe-tr-awk \
+		--relation "policy" \
+		--where 'policy == "allow" && user != "root"'
+
+	# Of course, we can do the same using SQL:
+	# relpipe-tr-sql \
+	#	--relation "policy" \
+	#		"SELECT * FROM policy WHERE policy = 'allow' AND user <> 'root'"
+	#	--copy 'limit'
+
+	# Or using Guile (Scheme):
+	# relpipe-tr-guile \
+	#	--relation policy \
+	#	--where '(and (string= $policy "allow") (not (string= $user "root")) )'
+}
+
+formatOutput() {
+	relpipe-out-tabular
+	# Or generate some other format:
+	# relpipe-out-ods       # spreadsheet for e.g. LibreOffice
+	# relpipe-out-xml       # XML e.g. for further XSLT or XQuery processing
+	# relpipe-out-recfile   # recfile for further GNU Recutils processing
+	# relpipe-out-asn1      # ASN.1 BER for the telco guys
+	# relpipe-out-gui       # display data in a GUI window
+	# etc.
+}
+
+# put the whole pipeline together:
+getXML | parseXML | filterRecords | formatOutput
diff -r 780d782cdef5 -r 5cf3a702f47d relpipe-data/examples/awk-through-xml.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/awk-through-xml.txt	Sat Dec 28 16:19:41 2019 +0100
@@ -0,0 +1,53 @@
+policy:
+ ╭─────────────────┬───────────────┬──────────────────┬──────────────┬────────────────────────────────┬────────────────────────────────────────────────────┬──────────────────────╮
+ │ policy (string) │ user (string) │ context (string) │ own (string) │ send_destination      (string) │ send_interface                            (string) │ send_member (string) │
+ ├─────────────────┼───────────────┼──────────────────┼──────────────┼────────────────────────────────┼────────────────────────────────────────────────────┼──────────────────────┤
+ │ allow           │ whoopsie      │                  │              │ org.freedesktop.NetworkManager │                                                    │                      │
+ │ allow           │ whoopsie      │                  │              │ org.freedesktop.NetworkManager │ org.freedesktop.DBus.Introspectable                │                      │
+ │ allow           │ whoopsie      │                  │              │ org.freedesktop.NetworkManager │ org.freedesktop.DBus.Properties                    │                      │
+ │ allow           │ whoopsie      │                  │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager                     │                      │
+ │ allow           │ whoopsie      │                  │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Connection.Active   │                      │
+ │ allow           │ whoopsie      │                  │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device              │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.DBus.Introspectable                │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.DBus.Properties                    │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.DBus.ObjectManager                 │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Adsl         │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Bond         │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Bridge       │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Bluetooth    │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Wired        │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Generic      │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Gre          │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Infiniband   │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Macvlan      │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Modem        │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.OlpcMesh     │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Team         │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Tun          │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Veth         │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Vlan         │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.WiMax.Nsp           │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.AccessPoint         │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.WiMax        │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device.Wireless     │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Device              │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Connection.Active   │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.DHCP4Config         │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.DHCP6Config         │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.IP4Config           │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.IP6Config           │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.VPN.Connection      │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager                     │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Settings            │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.Settings.Connection │                      │
+ │ allow           │               │ default          │              │ org.freedesktop.NetworkManager │ org.freedesktop.NetworkManager.AgentManager        │                      │
+ ╰─────────────────┴───────────────┴──────────────────┴──────────────┴────────────────────────────────┴────────────────────────────────────────────────────┴──────────────────────╯
+Record count: 39
+limit:
+ ╭────────────────────────────────┬─────────────────╮
+ │ name                  (string) │ value (integer) │
+ ├────────────────────────────────┼─────────────────┤
+ │ max_replies_per_connection     │            1024 │
+ │ max_match_rules_per_connection │            2048 │
+ ╰────────────────────────────────┴─────────────────╯
+Record count: 2
diff -r 780d782cdef5 -r 5cf3a702f47d relpipe-data/examples/awk-through-xml.xml
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/relpipe-data/examples/awk-through-xml.xml	Sat Dec 28 16:19:41 2019 +0100
@@ -0,0 +1,158 @@
+<!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN" "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
+<busconfig>
+	<policy user="root">
+		<allow own="org.freedesktop.NetworkManager"/>
+		<allow send_destination="org.freedesktop.NetworkManager"/>
+
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.PPP"/>
+
+		<allow send_interface="org.freedesktop.NetworkManager.SecretAgent"/>
+		<!-- These are there because some broken policies do
+		   <deny send_interface="..." /> (see dbus-daemon(8) for details).
+		   This seems to override that for the known VPN plugins.
+		-->
+		<allow send_destination="org.freedesktop.NetworkManager.openconnect"/>
+		<allow send_destination="org.freedesktop.NetworkManager.openswan"/>
+		<allow send_destination="org.freedesktop.NetworkManager.openvpn"/>
+		<allow send_destination="org.freedesktop.NetworkManager.pptp"/>
+		<allow send_destination="org.freedesktop.NetworkManager.vpnc"/>
+		<allow send_destination="org.freedesktop.NetworkManager.ssh"/>
+		<allow send_destination="org.freedesktop.NetworkManager.iodine"/>
+		<allow send_destination="org.freedesktop.NetworkManager.l2tp"/>
+		<allow send_destination="org.freedesktop.NetworkManager.libreswan"/>
+		<allow send_destination="org.freedesktop.NetworkManager.fortisslvpn"/>
+		<allow send_destination="org.freedesktop.NetworkManager.strongswan"/>
+		<allow send_interface="org.freedesktop.NetworkManager.VPN.Plugin"/>
+
+		<allow send_destination="org.fedoraproject.FirewallD1"/>
+
+		<!-- Allow the custom name for the dnsmasq instance spawned by NM
+			from the dns dnsmasq plugin to own it's dbus name, and for
+			messages to be sent to it.
+		-->
+		<allow own="org.freedesktop.NetworkManager.dnsmasq"/>
+		<allow send_destination="org.freedesktop.NetworkManager.dnsmasq"/>
+	</policy>
+	<policy user="whoopsie">
+		<allow send_destination="org.freedesktop.NetworkManager"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.DBus.Introspectable"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.DBus.Properties"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Connection.Active"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device"/>
+	</policy>
+	<policy context="default">
+		<deny own="org.freedesktop.NetworkManager"/>
+
+		<deny send_destination="org.freedesktop.NetworkManager"/>
+
+		<!-- Basic D-Bus API stuff -->
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.DBus.Introspectable"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.DBus.Properties"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.DBus.ObjectManager"/>
+
+		<!-- Devices (read-only properties, no methods) -->
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Adsl"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Bond"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Bridge"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Bluetooth"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Wired"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Generic"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Gre"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Infiniband"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Macvlan"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Modem"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.OlpcMesh"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Team"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Tun"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Veth"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Vlan"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.WiMax.Nsp"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.AccessPoint"/>
+
+		<!-- Devices (read-only, no security required) -->
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.WiMax"/>
+
+		<!-- Devices (read/write, secured with PolicyKit) -->
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device.Wireless"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Device"/>
+
+		<!-- Core stuff (read-only properties, no methods) -->
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Connection.Active"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.DHCP4Config"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.DHCP6Config"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.IP4Config"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.IP6Config"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.VPN.Connection"/>
+
+		<!-- Core stuff (read/write, secured with PolicyKit) -->
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Settings"/>
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.Settings.Connection"/>
+
+		<!-- Agents; secured with PolicyKit.  Any process can talk to
+		the AgentManager API, but only NetworkManager can talk
+		to the agents themselves. -->
+		<allow send_destination="org.freedesktop.NetworkManager"
+			   send_interface="org.freedesktop.NetworkManager.AgentManager"/>
+
+		<!-- Root-only functions -->
+		<deny send_destination="org.freedesktop.NetworkManager"
+			  send_interface="org.freedesktop.NetworkManager"
+			  send_member="SetLogging"/>
+		<deny send_destination="org.freedesktop.NetworkManager"
+			  send_interface="org.freedesktop.NetworkManager"
+			  send_member="Sleep"/>
+		<deny send_destination="org.freedesktop.NetworkManager"
+			  send_interface="org.freedesktop.NetworkManager.Settings"
+			  send_member="LoadConnections"/>
+		<deny send_destination="org.freedesktop.NetworkManager"
+			  send_interface="org.freedesktop.NetworkManager.Settings"
+			  send_member="ReloadConnections"/>
+
+		<deny own="org.freedesktop.NetworkManager.dnsmasq"/>
+		<deny send_destination="org.freedesktop.NetworkManager.dnsmasq"/>
+	</policy>
+
+	<limit name="max_replies_per_connection">1024</limit>
+	<limit name="max_match_rules_per_connection">2048</limit>
+</busconfig>
+