relpipe-data/roadmap.xml
author František Kučera <franta-hg@frantovo.cz>
Sat, 08 Dec 2018 18:08:37 +0100
branchv_0
changeset 196 8dd5b7ecb736
parent 184 9ad44348e6aa
child 218 d7a7f20aae51
permissions -rw-r--r--
examples: multiple relations in a single stream

<stránka
	xmlns="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/strana"
	xmlns:m="https://trac.frantovo.cz/xml-web-generator/wiki/xmlns/makro">
	
	<nadpis>Roadmap</nadpis>
	<perex>Vision of future versions</perex>
	<pořadí>14</pořadí>

	<text xmlns="http://www.w3.org/1999/xhtml">
		<p>
			Releases before v1.0.0 are development and are not intended for production use.
			Releases after v1.0.0 should follow the rules of <a href="http://semver.org/">Semantic versioning</a>.
		</p>
		
		<h2>v0.8</h2>
		
		<p>
			First public release for brave and courageous.
		</p>
		
		<h3>Data types</h3>
		<ul>
			<li>boolean</li>
			<li>variable unsigned integer (prototype)</li>
			<li>string in UTF-8</li>
		</ul>
		<h3>Inputs</h3>
		<ul>
			<li>CLI</li>
			<li>fstab</li>
		</ul>
		<h3>Transformations</h3>
		<ul>
			<li>grep: regular expression filter</li>
			<li>sed: regular expression replacer</li>
		</ul>
		<h3>Outputs</h3>
		<ul>
			<li>tabular</li>
			<li>XML</li>
			<li>nullbyte</li>
			<li>GUI in Qt</li>
			<li>ODS (LibreOffice)</li>
		</ul>
		
		
		<h2>v0.9, v0.10, v0.11 etc.</h2>
		
		<p>
			Releases for discussion and verification of the format and API design.
		</p>
		
		<h3>Data types</h3>
		<ul>
			<li>arrays/streams of bytes (octets)</li>
			<li>arrays of other types</li>
			<li>arrays of other types with support for NULL values</li>
			<li>nested relations</li>
			<li>date and time</li>
			<li>variable integer (probably LEB128 or similar)</li>
			<li>decimal numbers</li>
			<li>support NULL values (introduce bitmaps)</li>
			<li>support long strings (chunked, similar to octet streams)</li>
			
			<li>fixed size integers</li>
			<li>precise BigDecimal</li>
			<li>fractions</li>
			<li>string in UTF-16</li>
			<li>string in ASCII</li>
			<li>string in ISO 8859-1</li>
			<li>string in ISO 8859-2</li>
		</ul>
		
		<h3>Inputs</h3>
		<ul>
			<li>XML</li>
			<li>DSV</li>
			<li>regular expression</li>
		</ul>
		<h3>Transformations</h3>
		<ul>
			<li>SQL: sqlite</li>
		</ul>
		<h3>Outputs</h3>
		<ul>
			<li>CLI: null-byte separated values</li>
		</ul>
		
		<h3>Other tasks</h3>
		<ul>
			<li>review the build system (probably use cmake in all modules)</li>
			<li>pkg-config: version numbers, debug vs. release</li>
			<li>automated complex tests (specification vs. implementation compliance)</li>
		</ul>
		
		<h2>v1.0</h2>
		
		<p>
			First version for production use. 
			Brings no new features, just stabilized result of previous development.
			Stable must be:
		</p>
		
		<ul>
			<li>format specification</li>
			<li>relpipe-lib-writer</li>
			<li>relpipe-lib-reader</li>
			<li>relpipe-in-cli</li>
			<li>relpipe-out-tabular</li>
			<li>relpipe-out-xml</li>
		</ul>
		
		<p>
			Other parts might be released as stable later.
		</p>
		<p>
			After this point, all components (the format specification, particular libraries and particular tools) 
			will be versioned independently and the compatibility matrix will be maintained.
		</p>
		
		<h2>Further versions</h2>
		<p>
			Plans for next decades:
		</p>
		
		<h3>Data types</h3>
		<ul>
			<li>intervals of various types</li>
			<li>IPv4 address and subnet</li>
			<li>IPv6 address and subnet</li>
			<li>UUID</li>
			<li>e-mail</li>
			<li>URL / URI</li>
			<li>geographic locations</li>
			<li>OID: Object identifier</li>
		</ul>
		<h3>Inputs</h3>
		<p>Systems, commands:</p>
		<ul>
			<li>files: ls, find, grep</li>
			<li>network information: ip, iptables, netstat, ss, dhcp-lease-list</li>
			<li>network interaction: ping, host, wget, curl</li>
			<li>system information: ps, lsof</li>
			<li>versioning systems (Mercurial, Git, Subversion, Monotone, Bazaar)</li>
			<li>SELECTs from sqlite, <a href="https://sql-dk.globalcode.info/">SQL-DK</a></li>
		</ul>
			
		<p>Formats:</p>
		<ul>
			<li>XML: as xmlTable() in SQL</li>
			<li>ODS (LibreOffice)</li>
			<li>iCalendar</li>
			<li>vCard</li>
			<li>YAML, JSON, INI</li>
			<li>CSV, DSV</li>
			<li>Fsdb</li>
			<li>recfiles</li>
			<li>Java .properties</li>
			<li>Gettext / .po files</li>
			<li>sqlite (full database dump)</li>
			<li>STDIO log: captured STDIN, STDOUT and STDERR of other process with precise timing of all events</li>
			<li>pcap / tcpdump</li>
			<li>Inverse tabular</li>
			<li>Wikipedia / MediaWiki: harvest tables from given article</li>
		</ul>
		<h3>Transformations</h3>
		<ul>
			<li>iconv: character encoding converter</li>
			<li>rename: relations, attributes</li>
			<li>filter/skip: relations, attributes</li>
			<li>add constant attribute</li>
			<li>WHERE-like filter</li>
			<li>JOIN</li>
			<li>ORDER BY</li>
			<li>UNION, UNION ALL, explicit or implicit (relation interleaving)</li>
			<li>split single relation into multiple relations according to an attribute value</li>
			<li>union multiple relations and add an attribute with original relation name</li>
			<li>XPath, XSLT, XQuery</li>
			<li>statistics: compute/add aggregated values like min(), max(), avg(), sum() or percentiles</li>
			<li>function calls (probably should be part od SQL or XPath)</li>
			<li>pack and unpack: single record with arrays vs. multiple records with scalars</li>
			<li>Guile, Lua, Perl, Python</li>
			<li>system command executor</li>
		</ul>
		<h3>Outputs</h3>
		<ul>
			<li>GUI in GTK</li>
			<li>(La)TeX</li>
			<li>Record, Single (like in <a href="https://sql-dk.globalcode.info/">SQL-DK</a>)</li>
			<li>XML tree</li>
			<li>iCalendar</li>
			<li>vCard</li>
			<li>tar: directories and files <!--relations and records = directories; attribute values = files--></li>
			<li>YAML, JSON, INI</li>
			<li>CSV, DSV</li>
			<li>Fsdb</li>
			<li>recfiles</li>
			<li>Java .properties</li>
			<li>Gettext / .po files</li>
			<li>ASN.1</li>
			<li>XHTML: tables, graphs</li>
			<li>GraphViz: .dot files</li>
			<li>SQL: CREATE TABLE, INSERT</li>
			<li>SMTP, IMAP, files, HTTP, SOAP, ZeroMQ, TCP, UDP</li>
			<li>source code / literals in Java, C, C++, Bash</li>
			<li>return/exit code (single numeric value)</li>
		</ul>
		
		<h3>Libraries and tools</h3>
		<p>Readers (SAX-like parsers) and writers (generators) for:</p>
		<ul>
			<li>C++</li>
			<li>C</li>
			<li>Java</li>
			<li>Python</li>
			<li>Perl</li>
			<li>Rust</li>
			<li>Go</li>
			<li>PHP</li>
		</ul>
		<p>Other libraries and tools:</p>
		<ul>
			<li>ORM API: mapping between classes/objects and relations/records</li>
			<li>schemas (XSD-like)</li>
			<li>generators, compilers, validators, comparators</li>
			<li>transformer helpers</li>
			<li>Ragel input helpers</li>
			<li>PEG input helpers</li>
			<li>repair tools (for corrupted data)</li>
			<li>pull parsers</li>
			<li>SAX and DOM parsers (read relational data like it was XML)</li>
			<li>visual editor for pipeline design</li>
		</ul>
		
		
	</text>

</stránka>