--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/ASN1ContentHandler.h Sat Jul 24 17:35:27 2021 +0200
@@ -0,0 +1,381 @@
+/**
+ * Relational pipes
+ * Copyright © 2021 František Kučera (Frantovo.cz, GlobalCode.info)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <memory>
+#include <vector>
+#include <sstream>
+#include <iomanip>
+#include <cmath>
+
+#include "ProxyVector.h"
+
+namespace relpipe {
+namespace in {
+namespace asn1 {
+namespace lib {
+
+namespace UniversalType {
+static const uint64_t EndOfContent = 0x00;
+static const uint64_t Boolean = 0x01;
+static const uint64_t Integer = 0x02;
+static const uint64_t BitString = 0x03;
+static const uint64_t OctetString = 0x04;
+static const uint64_t Null = 0x05;
+static const uint64_t ObjectIdentifier = 0x06;
+static const uint64_t ObjectDescriptor = 0x07;
+static const uint64_t External = 0x08;
+static const uint64_t Real = 0x09;
+static const uint64_t Enumerated = 0xA;
+static const uint64_t Embedded = 0xB;
+static const uint64_t UTF8String = 0xC;
+static const uint64_t RelativeObjectIdentifier = 0xD;
+static const uint64_t Time = 0xE;
+static const uint64_t Reserved = 0xF;
+static const uint64_t Sequence = 0x10;
+static const uint64_t Set = 0x11;
+static const uint64_t NumericString = 0x12;
+static const uint64_t PrintableString = 0x13;
+static const uint64_t T61String = 0x14;
+static const uint64_t VideotexString = 0x15;
+static const uint64_t IA5String = 0x16;
+static const uint64_t UTCTime = 0x17;
+static const uint64_t GeneralizedTime = 0x18;
+static const uint64_t GraphicString = 0x19;
+static const uint64_t VisibleString = 0x1A;
+static const uint64_t GeneralString = 0x1B;
+static const uint64_t UniversalString = 0x1C;
+static const uint64_t CharacterString = 0x1D;
+static const uint64_t BMPString = 0x1E;
+static const uint64_t Date = 0x1F;
+static const uint64_t TimeOfDay = 0x20;
+static const uint64_t DateTime = 0x21;
+static const uint64_t Duration = 0x22;
+static const uint64_t ObjectIdentifierIRI = 0x23;
+static const uint64_t RelativeObjectIdentifierIRI = 0x24;
+}
+
+class ASN1ContentHandler {
+public:
+
+ enum class TagClass : uint8_t {
+ Universal = 0,
+ Application = 1,
+ ContextSpecific = 2,
+ Private = 3
+ };
+
+ enum class PC : uint8_t {
+ Primitive = 0,
+ Constructed = 1
+ };
+
+ class Header {
+ public:
+ TagClass tagClass;
+ PC pc;
+ uint64_t tag;
+ };
+
+ // TODO: separate implementation of particular types from the interface, separate SPI from API?
+
+ class Integer {
+ private:
+ // TODO: use std::string (of octets, not ASCII) instead of std::vector?
+ // TODO: use this class as BigInteger across Relational pipes?
+ std::vector<uint8_t> data;
+ public:
+
+ /**
+ * @param data integer octets as in BER encoding
+ */
+ Integer(std::vector<uint8_t> data) : data(data) {
+ }
+
+ virtual ~Integer() {
+ }
+
+ size_t size() const {
+ return data.size();
+ }
+
+ const uint8_t& operator[](std::size_t index) const {
+ return data[index];
+ }
+
+ const std::string toHex() const {
+ std::stringstream hex;
+ hex << std::hex << std::setfill('0');
+ for (uint8_t b : data) hex << std::setw(2) << (int) b;
+ return hex.str();
+ }
+
+ const std::string toString() const {
+ try {
+ return std::to_string(toInt64());
+ } catch (...) {
+ // integer has more than 64 bits → only HEX form value will be available
+ // TODO: support longer values than 64 bits
+ // TODO: do not ignore zero-length error?
+ return "";
+ }
+ }
+
+ const int64_t toInt64() const {
+ int64_t value = 0;
+
+ if (data.size() > sizeof (value)) throw std::invalid_argument("Integer is too long");
+ else if (data.size() == 0) throw std::invalid_argument("Integer has zero length");
+
+ value = data[0];
+ bool negative = data[0] & 0x80;
+
+ for (size_t i = 1, limit = data.size(); i < limit; i++) value = (value << 8) | data[i];
+
+ if (negative) value -= std::pow(256, data.size());
+
+ return value;
+ }
+
+ };
+
+ class ObjectIdentifier {
+ private:
+ // TODO: use std::string (of octets, not ASCII) instead of std::vector?
+ // TODO: use this class across Relational pipes as one of basic types?
+ std::vector<uint8_t> data;
+
+ public:
+
+ /**
+ * @param data integer octets as in BER encoding
+ */
+ ObjectIdentifier(std::vector<uint8_t> data) : data(data) {
+ // TODO: cache size and element values?
+ }
+
+ virtual ~ObjectIdentifier() {
+ }
+
+ /**
+ * @return number of elements, not octets
+ */
+ size_t size() const {
+ return 0; // FIXME: correct OID size
+ }
+
+ /**
+ * @param index 0 = root element
+ * @return value of the element at given position
+ */
+ const uint8_t& operator[](std::size_t index) const {
+ return data[index]; // FIXME: correct OID value
+ }
+
+ const std::string toString() const {
+ if (data.size() == 0) return "";
+
+ std::stringstream result;
+
+ result << (data[0] / 40) << "." << (data[0] % 40); // first two elements are encoded in the first octet
+
+ for (size_t i = 1, limit = data.size(), octet = 0, element = 0; i < limit; i++) {
+ octet = data[i];
+ element = element << 7 | (octet & 0xFF >> 1);
+ // TODO: throw exception if the element value overflows? (should not happen) or format even longer values
+ if ((octet & 1 << 7) == 0) {
+ result << "." << element;
+ element = 0;
+ }
+ }
+
+ return result.str();
+ }
+
+ };
+
+ class DateTime {
+ public:
+
+ enum class Precision {
+ Year,
+ Month,
+ Day,
+ Hour,
+ Minute,
+ Second,
+ Milisecond,
+ Nanosecond
+ };
+
+ // TODO: timezone (in minutes or 1/4 hours)
+
+ Precision precision = Precision::Second;
+ int32_t year = 1970;
+ int8_t month = 1;
+ int8_t day = 1;
+ int8_t hour = 0;
+ int8_t minute = 0;
+ int8_t second = 0;
+ int32_t nanosecond = 0;
+ int8_t timezoneHour = 0;
+ int8_t timezoneMinute = 0;
+
+ virtual ~DateTime() {
+ }
+
+ const std::string toString() const {
+ std::stringstream result;
+ result << std::setfill('0');
+ result << std::setw(4) << (int) year;
+ result << "-" << std::setw(2) << (int) month;
+ result << "-" << std::setw(2) << (int) day;
+ result << "T" << std::setw(2) << (int) hour;
+ result << ":" << std::setw(2) << (int) minute;
+ result << ":" << std::setw(2) << (int) second;
+ if (precision == Precision::Nanosecond) result << "," << (int) nanosecond;
+ result << (timezoneHour < 0 ? "-" : "+");
+ result << std::setw(2) << (int) std::abs(timezoneHour);
+ result << ":" << std::setw(2) << (int) timezoneMinute;
+ return result.str();
+ }
+ };
+
+ virtual ~ASN1ContentHandler() = default;
+
+ // TODO: more metadata, support OID decoding and ASN.1 modules (schema), probably through a plug-in
+ // TODO: support also extension extractor plug-ins? (could decode some opaque structures like octet strings and replace them with nested elements) e.g. subjectAltName in https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.6
+
+ /**
+ * @param uri identifier of the option
+ * @param value value of the option
+ * @return whether this option is supported and was applied here
+ */
+ virtual bool setOption(const std::string& uri, const std::string& value) {
+ return false;
+ }
+
+ virtual void writeStreamStart() = 0;
+ virtual void writeStreamEnd() = 0;
+
+ virtual void writeCollectionStart(const Header& header) = 0;
+ virtual void writeCollectionEnd() = 0;
+ virtual void writeBoolean(const Header& header, bool value) = 0;
+ virtual void writeNull(const Header& header) = 0;
+ virtual void writeInteger(const Header& header, Integer value) = 0;
+ /**
+ * @param type original type in ASN.1
+ * @param value original text converted to UTF-8
+ */
+ virtual void writeTextString(const Header& header, std::string value) = 0;
+ /**
+ * @param value arbitrary sequence of octets (bytes), usually not a human-readable text
+ */
+ virtual void writeOctetString(const Header& header, std::string value) = 0;
+ /**
+ * @param value arbitrary sequence of bits (booleans), usually not a human-readable text
+ */
+ virtual void writeBitString(const Header& header, std::vector<bool> value) = 0;
+ virtual void writeOID(const Header& header, ObjectIdentifier value) = 0;
+ virtual void writeDateTime(const Header& header, DateTime value) = 0;
+ // Object descriptor
+ // virtual void writeReal(float value) = 0;
+ // Enumerated
+ // Embedded PVD
+ // Relative OID
+ // OID-IRI
+ // Relative OID-IRI
+
+ /**
+ * Specific value that was not parsed.
+ * May be processed in a generic way (as binary data or ASCII or UTF-8 string, when possible)
+ * or according to given application, context or private specification.
+ *
+ * @param value original raw data
+ */
+ virtual void writeSpecific(const Header& header, std::string value) = 0;
+
+};
+
+class ASN1ContentHandlerProxy : public ASN1ContentHandler {
+private:
+ ProxyVector<ASN1ContentHandler> handlers;
+public:
+
+ void addHandler(std::shared_ptr<ASN1ContentHandler> handler) {
+ handlers.push_back(handler);
+ }
+
+ void writeStreamStart() override {
+ handlers.forward(&ASN1ContentHandler::writeStreamStart);
+ }
+
+ void writeStreamEnd() override {
+ handlers.forward(&ASN1ContentHandler::writeStreamEnd);
+ }
+
+ void writeCollectionStart(const Header& header) override {
+ handlers.forward(&ASN1ContentHandler::writeCollectionStart, header);
+ }
+
+ void writeCollectionEnd() override {
+ handlers.forward(&ASN1ContentHandler::writeCollectionEnd);
+ }
+
+ void writeBoolean(const Header& header, bool value) override {
+ handlers.forward(&ASN1ContentHandler::writeBoolean, header, value);
+ }
+
+ void writeNull(const Header& header) override {
+ handlers.forward(&ASN1ContentHandler::writeNull, header);
+ }
+
+ void writeInteger(const Header& header, Integer value) override {
+ handlers.forward(&ASN1ContentHandler::writeInteger, header, value);
+ }
+
+ void writeTextString(const Header& header, std::string value) override {
+ handlers.forward(&ASN1ContentHandler::writeTextString, header, value);
+ }
+
+ void writeOctetString(const Header& header, std::string value) override {
+ handlers.forward(&ASN1ContentHandler::writeOctetString, header, value);
+ }
+
+ void writeBitString(const Header& header, std::vector<bool> value) override {
+ handlers.forward(&ASN1ContentHandler::writeBitString, header, value);
+ }
+
+ void writeOID(const Header& header, ObjectIdentifier value) override {
+ handlers.forward(&ASN1ContentHandler::writeOID, header, value);
+ }
+
+ void writeDateTime(const Header& header, DateTime value) override {
+ handlers.forward(&ASN1ContentHandler::writeDateTime, header, value);
+ }
+
+ void writeSpecific(const Header& header, std::string value) override {
+ handlers.forward(&ASN1ContentHandler::writeSpecific, header, value);
+ }
+
+};
+
+
+}
+}
+}
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/ASN1Reader.h Sat Jul 24 17:35:27 2021 +0200
@@ -0,0 +1,53 @@
+/**
+ * Relational pipes
+ * Copyright © 2021 František Kučera (Frantovo.cz, GlobalCode.info)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <memory>
+
+#include "AbstractParser.h"
+#include "ASN1ContentHandler.h"
+
+namespace relpipe {
+namespace in {
+namespace asn1 {
+namespace lib {
+
+class ASN1Reader : public AbstractParser {
+public:
+ virtual ~ASN1Reader() = default;
+
+ virtual void addHandler(std::shared_ptr<ASN1ContentHandler> handler) {
+ handlers->addHandler(handler);
+ };
+
+ /**
+ * @param uri identifier of the option
+ * @param value value of the option
+ * @return whether this option is supported and was applied here
+ */
+ virtual bool setOption(const std::string& uri, const std::string& value) {
+ return false;
+ }
+
+protected:
+ std::shared_ptr<ASN1ContentHandlerProxy> handlers = std::make_shared<ASN1ContentHandlerProxy>();
+};
+
+}
+}
+}
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/AbstractParser.cpp Sat Jul 24 17:35:27 2021 +0200
@@ -0,0 +1,92 @@
+/**
+ * Relational pipes
+ * Copyright © 2021 František Kučera (Frantovo.cz, GlobalCode.info)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "AbstractParser.h"
+#include "TransactionalBuffer.h"
+
+namespace relpipe {
+namespace in {
+namespace asn1 {
+namespace lib {
+
+class AbstractParserImpl {
+private:
+ friend AbstractParser;
+ AbstractParser* interface;
+ TransactionalBuffer buffer;
+public:
+
+ AbstractParserImpl(AbstractParser* interface) : interface(interface) {
+ }
+
+ virtual ~AbstractParserImpl() {
+ }
+};
+
+AbstractParser::AbstractParser() {
+ implementation = new AbstractParserImpl(this);
+}
+
+AbstractParser::~AbstractParser() {
+ delete implementation;
+}
+
+void AbstractParser::write(const uint8_t* buffer, const size_t length) {
+ try {
+ // TODO: do not write to the buffer, just append in read()/peek() and write just the part that was not read during this cycle
+ implementation->buffer.write(buffer, length);
+
+ // TODO: call an overridable method to get preferred minimum block size and run cycle only if we have enough data or EOF
+ // and/or remember the length of last failed (rollbacked) read() call
+ update();
+ commit();
+ } catch (const TransactionalBuffer::ReadBufferUnderflowException& e) {
+ rollback();
+ } catch (const AbstractParser::ExplicitRollbackException& e) {
+ rollback();
+ }
+}
+
+void AbstractParser::close() {
+ // TODO: check remaining data + call update() or just let parsers override this method?
+}
+
+void AbstractParser::rollback() {
+ // TODO: notify rollback listeners? (they can monitor the performance / frequency of rollbacks)
+ implementation->buffer.rollbackRead();
+}
+
+void AbstractParser::commit() {
+ implementation->buffer.commitRead();
+}
+
+void AbstractParser::read(uint8_t* buffer, const size_t length) {
+ implementation->buffer.read(buffer, length);
+}
+
+void AbstractParser::peek(uint8_t* buffer, const size_t length) {
+ implementation->buffer.peek(buffer, length);
+}
+
+size_t AbstractParser::getBytesRead() {
+ return implementation->buffer.getBytesRead();
+}
+
+}
+}
+}
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/AbstractParser.h Sat Jul 24 17:35:27 2021 +0200
@@ -0,0 +1,126 @@
+/**
+ * Relational pipes
+ * Copyright © 2021 František Kučera (Frantovo.cz, GlobalCode.info)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <cstring>
+#include <cstdint>
+
+namespace relpipe {
+namespace in {
+namespace asn1 {
+namespace lib {
+
+class AbstractParserImpl;
+
+/**
+ * Event-driven parser that consumes sequence of byte buffers (chunked stream)
+ * and produces some output (implementation specific).
+ *
+ * The flow is controlled from outside: incomming data pieces are passed to the parser through the write() method as they come.
+ *
+ * From the child class perspective (implementation of particular parser)
+ * data pieces are obtained through the read() method from the internal buffer maintained by AbstractParser.
+ */
+class AbstractParser {
+private:
+ friend AbstractParserImpl;
+ AbstractParserImpl* implementation;
+ void rollback();
+public:
+ virtual ~AbstractParser();
+
+ /**
+ * Submit a part of incomming data to the parser.
+ * When possible, such data are usually processed synchronously during this method call.
+ * If not (incomplete TLV/AVP/PDU/token/etc.) this part is appended to the internal buffer maintained by AbstractParser
+ * and processed in some next cycle or – in worst case – during the close() call.
+ *
+ * @param buffer readable memory
+ * @param length size of the buffer
+ */
+ void write(const uint8_t* buffer, const size_t length);
+
+ /**
+ * Finalize the parsing process.
+ * After calling this method, all data from AbstractParser buffers should be consumed, parsed and results published.
+ * No write() call is expected after close() and would fail.
+ * However the parser object remains valid and may be used to get some auxiliary information (if supperted by given implementation).
+ */
+ virtual void close();
+protected:
+ AbstractParser();
+
+ /**
+ * May be thrown from the update() method in order to cancel currenty cycle and do explicit rollback.
+ * Same data will be processed in the next cycle.
+ */
+ class ExplicitRollbackException {
+ // TODO: common super-class for exceptions, hidden implementation
+ };
+
+ /**
+ * May be called from the update() method in order to explicitly confirm that read data was successfully processed.
+ * Such data will not be processed again during the next cycle (even if ReadBufferUnderflowException occurs later in this cycle).
+
+ * Explicit commit() call is useful when we did some demanding work (so we are not willing to do it again in case of ReadBufferUnderflowException);
+ * and is necessary when we already published some outputs or did other non-idempotent operation or caused some other significant side effects.
+
+ * If there is no commit() called and update() just finishes, commit() is called implicitly.
+ *
+ * Note: There is no accessible rollback() method – throw ExplicitRollbackException instead.
+ */
+ void commit();
+
+ /**
+ * Fill the buffer with incomming data of given length (exactly).
+ *
+ * If there are not enough data available, ReadBufferUnderflowException is thrown.
+ * This exception should not be caught in the child class – it should propagate back to the AbstractParser
+ * where it causes rollback(). In such case, the same data will be available during next update() cycle.
+ *
+ * @param buffer writable memory
+ * @param length size of the buffer
+ */
+ void read(uint8_t* buffer, const size_t length);
+
+ /**
+ * Like read(), but does not update the marks (buffer positions), so it can be called again and again with same result.
+ *
+ * @param buffer writable memory
+ * @param length size of the buffer
+ */
+ void peek(uint8_t* buffer, const size_t length);
+
+ /**
+ * @return total number of bytes that were successfully read
+ */
+ size_t getBytesRead();
+
+ /**
+ * Reads input from buffers using read(), parses data and usually emits the result (e.g. to a handler/listener).
+ * Is specific for particular format/parser.
+ *
+ * This method is called at least once at the end of the stream (with whole stream content in read buffers).
+ * Usually it is called more often, by default: after each write (with just already received data part in read buffers).
+ */
+ virtual void update() = 0;
+};
+
+}
+}
+}
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/BasicASN1Reader.h Sat Jul 24 17:35:27 2021 +0200
@@ -0,0 +1,375 @@
+/**
+ * Relational pipes
+ * Copyright © 2021 František Kučera (Frantovo.cz, GlobalCode.info)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <memory>
+#include <vector>
+#include <array>
+#include <sstream>
+#include <regex>
+
+#include "ASN1Reader.h"
+#include "ValidatingASN1ContentHandler.h"
+#include "uri.h"
+
+namespace relpipe {
+namespace in {
+namespace asn1 {
+namespace lib {
+
+/**
+ * Reads ASN.1 data encoded as BER (DER, CER).
+ */
+class BasicASN1Reader : public ASN1Reader {
+private:
+
+ bool started = false;
+
+ bool parseEncapsulated = true;
+
+ /**
+ * TODO: use a common method
+ */
+ bool parseBoolean(const std::string& value) {
+ if (value == "true") return true;
+ else if (value == "false") return false;
+ else throw std::invalid_argument(std::string("Unable to parse boolean value: ") + value + " (expecting true or false)");
+ }
+
+ class BasicHeader : public ASN1ContentHandler::Header {
+ public:
+ bool definiteLength;
+ size_t length;
+ };
+
+ class LevelMetadata {
+ public:
+ bool definiteLength;
+ size_t length;
+ size_t start;
+ };
+
+ std::vector<LevelMetadata> level;
+
+ void checkRemainingItems() {
+ if (level.size()) {
+ LevelMetadata& l = level.back();
+ if (l.definiteLength && l.length == getBytesRead() - l.start) {
+ level.pop_back();
+ handlers->writeCollectionEnd();
+ checkRemainingItems(); // multiple collections may end at the same point
+ }
+ }
+ }
+
+ BasicHeader readHeader() {
+ using TagClass = ASN1ContentHandler::TagClass;
+ using PC = ASN1ContentHandler::PC;
+
+ BasicHeader h;
+
+ memset(&h, 0, sizeof (h)); // TODO: remove, not needed
+
+ uint8_t tagByte;
+ read(&tagByte, 1);
+
+ h.tagClass = (TagClass) (tagByte >> 6);
+ h.pc = (PC) ((tagByte >> 5) & 1);
+ h.tag = tagByte & (0xFF >> 3);
+ if (h.tag == 31) { // all five tag bits are set → tag number (greater than 30) is encoded in following octets
+ h.tag = 0;
+ uint8_t moreTag = 0;
+ do {
+ read(&moreTag, 1);
+ h.tag = h.tag << 7 | (moreTag & (0xFF >> 1));
+ } while (moreTag & (1 << 7));
+ }
+
+ uint8_t lengthByte;
+ read(&lengthByte, 1);
+
+ if (lengthByte >> 7 == 0) {
+ // definite short
+ h.definiteLength = true;
+ h.length = lengthByte;
+ } else if (lengthByte == 0b10000000) {
+ // indefinite
+ h.definiteLength = false;
+ h.length = 0;
+ } else if (lengthByte == 0xFF) {
+ throw relpipe::writer::RelpipeWriterException(L"ASN.1 lengthByte == 0xFF (reserved value)"); // TODO: better exception
+ } else {
+ // definite long
+ h.definiteLength = true;
+ h.length = 0;
+ std::vector<uint8_t> lengthBytes(lengthByte & 0b01111111, 0);
+ read(lengthBytes.data(), lengthBytes.size());
+ for (uint8_t l : lengthBytes) h.length = (h.length << 8) + l;
+ }
+
+ return h;
+ }
+
+ const std::string readString(size_t length) {
+ std::string result;
+
+ for (size_t remaining = length; remaining;) {
+ size_t current = std::min(remaining, (size_t) 3);
+ result.resize(result.size() + current);
+ read((uint8_t*) result.data() + result.size() - current, current);
+ remaining -= current;
+ }
+
+ return result;
+ }
+
+ const std::vector<uint8_t> readVector(size_t length) {
+ std::vector<uint8_t> result;
+ std::string s = readString(length); // TODO: read directly to the vector
+ result.resize(length);
+ for (size_t i = 0; i < length; i++) result[i] = (uint8_t) s[i];
+ return result;
+ }
+
+ void processNext() {
+ using TagClass = ASN1ContentHandler::TagClass;
+ using PC = ASN1ContentHandler::PC;
+
+ checkRemainingItems();
+ BasicHeader typeHeader = readHeader();
+ // commit(); // TODO: commit here and recover later instead of rollback?
+
+ if (!started) {
+ handlers->writeStreamStart();
+ started = true;
+ }
+
+ // TODO: check tagClass and pc
+
+ // TODO: constants, more types
+ if (typeHeader.tag == UniversalType::EndOfContent && typeHeader.tagClass == TagClass::Universal && typeHeader.pc == PC::Primitive) {
+ handlers->writeCollectionEnd();
+ } else if (typeHeader.tag == UniversalType::Sequence) {
+ level.push_back({typeHeader.definiteLength, typeHeader.length, getBytesRead()}); // TODO: transaction
+ handlers->writeCollectionStart(typeHeader);
+ } else if (typeHeader.tag == UniversalType::Set) {
+ level.push_back({typeHeader.definiteLength, typeHeader.length, getBytesRead()}); // TODO: transaction
+ handlers->writeCollectionStart(typeHeader);
+ } else if (typeHeader.pc == PC::Constructed) {
+ level.push_back({typeHeader.definiteLength, typeHeader.length, getBytesRead()}); // TODO: transaction
+ handlers->writeCollectionStart(typeHeader);
+ } else if (typeHeader.tag == UniversalType::Null && typeHeader.length == 0) {
+ handlers->writeNull(typeHeader);
+ } else if (typeHeader.tag == UniversalType::Boolean && typeHeader.definiteLength && typeHeader.length == 1) {
+ bool value;
+ read((uint8_t*) & value, 1);
+ handlers->writeBoolean(typeHeader, value);
+ } else if (typeHeader.tag == UniversalType::Integer && typeHeader.tagClass == TagClass::Universal && typeHeader.definiteLength) {
+ std::vector<uint8_t> value = readVector(typeHeader.length);
+ handlers->writeInteger(typeHeader, ASN1ContentHandler::Integer(value));
+ } else if (typeHeader.tag == UniversalType::ObjectIdentifier && typeHeader.tagClass == TagClass::Universal && typeHeader.definiteLength) {
+ std::vector<uint8_t> value(typeHeader.length, 0x00);
+ read(value.data(), typeHeader.length);
+ handlers->writeOID(typeHeader,{value});
+ } else if (typeHeader.tag == UniversalType::UTF8String && typeHeader.tagClass == TagClass::Universal && typeHeader.definiteLength) {
+ std::string s = readString(typeHeader.length);
+ handlers->writeTextString(typeHeader, s);
+ } else if (typeHeader.tag == UniversalType::PrintableString && typeHeader.tagClass == TagClass::Universal && typeHeader.definiteLength) {
+ // TODO: check encoding
+ std::string s = readString(typeHeader.length);
+ handlers->writeTextString(typeHeader, s);
+ } else if (typeHeader.tag == UniversalType::OctetString && typeHeader.tagClass == TagClass::Universal && typeHeader.definiteLength) {
+ std::string s = readString(typeHeader.length);
+ if (processEncapsulatedContent(typeHeader, s) == false) handlers->writeOctetString(typeHeader, s);
+ } else if (typeHeader.tag == UniversalType::BitString && typeHeader.tagClass == TagClass::Universal && typeHeader.definiteLength) {
+ std::string s = readString(typeHeader.length);
+ if (processEncapsulatedContent(typeHeader, s) == false) {
+ std::vector<bool> bits;
+ // TODO: throw exception on wrong padding or insufficient length?
+ if (s.size() > 1) {
+ uint8_t padding = s[0];
+ for (uint8_t j = padding; j < 8; j++) bits.push_back(s.back() & 1 << j);
+ for (size_t i = s.size() - 2; i > 0; i--) for (uint8_t j = 0; j < 8; j++) bits.push_back(s[i] & 1 << j);
+ }
+ handlers->writeBitString(typeHeader, bits);
+ }
+ } else if (typeHeader.tag == UniversalType::UTCTime && typeHeader.tagClass == TagClass::Universal && typeHeader.definiteLength) {
+ // TODO: check encoding
+ std::string s = readString(typeHeader.length);
+
+ ASN1ContentHandler::DateTime dateTime;
+
+ std::smatch match;
+ if (std::regex_match(s, match, std::regex("([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})?(Z|([+-][0-9]{2})([0-9]{2}))"))) {
+ int i = 1;
+ uint32_t year = std::stoi(match[i++]);
+ dateTime.year = year < 50 ? 2000 + year : 1900 + year;
+ dateTime.month = std::stoi(match[i++]);
+ dateTime.day = std::stoi(match[i++]);
+ dateTime.hour = std::stoi(match[i++]);
+ dateTime.minute = std::stoi(match[i++]);
+ dateTime.precision = match[i].length() ? ASN1ContentHandler::DateTime::Precision::Second : ASN1ContentHandler::DateTime::Precision::Minute;
+ dateTime.second = match[i].length() ? std::stoi(match[i]) : 0;
+ i++;
+ if (match[i++] != "Z") {
+ dateTime.timezoneHour = std::stoi(match[i++]);
+ dateTime.timezoneMinute = std::stoi(match[i++]);
+ }
+ handlers->writeDateTime(typeHeader, dateTime);
+ } else {
+ throw std::invalid_argument("Unsupported UTCTime format: " + s); // TODO: better exception
+ }
+
+ } else if (typeHeader.tag == UniversalType::GeneralizedTime && typeHeader.tagClass == TagClass::Universal && typeHeader.definiteLength) {
+ std::string s = readString(typeHeader.length);
+
+ ASN1ContentHandler::DateTime dateTime;
+
+ std::smatch match;
+ if (std::regex_match(s, match, std::regex("([0-9]{4})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})(\\.([0-9]{1,3}))?(Z|([+-][0-9]{2})([0-9]{2}))"))) {
+ // TODO: support also fractions of minutes and hours in GeneralizedTime
+ int i = 1;
+ dateTime.year = std::stoi(match[i++]);
+ dateTime.month = std::stoi(match[i++]);
+ dateTime.day = std::stoi(match[i++]);
+ dateTime.hour = std::stoi(match[i++]);
+ dateTime.minute = std::stoi(match[i++]);
+ dateTime.second = match[i].length() ? std::stoi(match[i++]) : 0;
+ dateTime.precision = match[i++].length() ? ASN1ContentHandler::DateTime::Precision::Nanosecond : ASN1ContentHandler::DateTime::Precision::Second;
+ if (match[i].length() == 1) dateTime.nanosecond = std::stoi(match[i++]) * 100 * 1000000;
+ else if (match[i].length() == 2) dateTime.nanosecond = std::stoi(match[i++]) * 10 * 1000000;
+ else if (match[i].length() == 3) dateTime.nanosecond = std::stoi(match[i++]) * 1000000;
+ else i++;
+ if (match[i++] != "Z") {
+ dateTime.timezoneHour = std::stoi(match[i++]);
+ dateTime.timezoneMinute = std::stoi(match[i++]);
+ }
+ handlers->writeDateTime(typeHeader, dateTime);
+ } else {
+ throw std::invalid_argument("Unsupported GeneralizedTime format: " + s); // TODO: better exception
+ }
+
+ } else {
+ // TODO: do not skip, parse
+ std::string s = readString(typeHeader.length);
+ handlers->writeSpecific(typeHeader, s);
+ }
+
+ commit();
+ }
+
+ bool hasAvailableForReading() {
+ // TODO: API in AbstractParser for checking available bytes?
+ uint8_t tmp;
+ try {
+ peek(&tmp, 1);
+ return true;
+ } catch (...) {
+ return false;
+ }
+ }
+
+ bool isValidBER(const std::string& input) {
+ BasicASN1Reader encapsulatedReader;
+ std::shared_ptr<ValidatingASN1ContentHandler> validatingHandler = std::make_shared<ValidatingASN1ContentHandler>();
+ encapsulatedReader.addHandler(validatingHandler);
+ try {
+ encapsulatedReader.write((const uint8_t*) input.c_str(), input.size());
+ encapsulatedReader.close();
+ validatingHandler->finalCheck();
+ return true;
+ } catch (...) {
+ return false;
+ }
+ }
+
+ class EncapsulatedASN1ContentHandler : public ASN1ContentHandlerProxy {
+ public:
+
+ void writeStreamStart() override {
+ // skip this event
+ }
+
+ void writeStreamEnd() override {
+ // skip this event
+ }
+ };
+
+ /**
+ * @param typeHeader
+ * @param input OCTET STRING or BIT STRING raw bytes
+ * @return whether we found valid content and passed parsed results to handlers
+ */
+ bool processEncapsulatedContent(const BasicHeader& typeHeader, const std::string& input) {
+ // TODO: avoid double parsing + encapsulated content might be also processed at the XML/DOM level where we may even do conditional processing based on XPath (evaluate only certain octet- or bit- strings)
+ // We may also do the same as with SEQUENCE or SET (continue nested reading in this ASN1Rreader instance), but it would require valid encapsulated data and would avoid easy fallback to raw OCTET or BIT STRING. We would also have to check the boundaries of the nested part.
+ if (parseEncapsulated && isValidBER(input)) {
+ handlers->writeCollectionStart(typeHeader);
+
+ BasicASN1Reader encapsulatedReader;
+ std::shared_ptr<EncapsulatedASN1ContentHandler> encapsulatedHandler = std::make_shared<EncapsulatedASN1ContentHandler>();
+ encapsulatedHandler->addHandler(handlers);
+ encapsulatedReader.addHandler(encapsulatedHandler);
+
+ encapsulatedReader.write((const uint8_t*) input.c_str(), input.size());
+ encapsulatedReader.close();
+
+ handlers->writeCollectionEnd();
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+protected:
+
+ void update() override {
+ while (true) processNext();
+ }
+
+public:
+
+ bool setOption(const std::string& uri, const std::string& value) override {
+ if (uri == option::Encoding && value == encoding::ber); // currently, we support only BER (and thus also CER and DER) encoding, but options have no actual effect – we just validate them
+ else if (uri == option::Encoding && value == encoding::cer); // in future versions, this might switch the parser into more strict mode
+ else if (uri == option::Encoding && value == encoding::der); // in future versions, this might switch the parser into more strict mode
+ else if (uri == option::Encoding && value == encoding::per) throw std::invalid_argument("PER encoding is not yet supported");
+ else if (uri == option::Encoding && value == encoding::xer) throw std::invalid_argument("XER encoding is not yet supported");
+ else if (uri == option::Encoding && value == encoding::asn1) throw std::invalid_argument("ASN.1 encoding is not yet supported");
+ else if (uri == option::Encoding) throw std::invalid_argument("Unsupported ASN.1 encoding: " + value);
+ else if (uri == option::ParseEncapsulated) parseEncapsulated = parseBoolean(value);
+ else return false;
+
+ return true;
+ }
+
+ void close() override {
+ if (hasAvailableForReading()) throw std::logic_error("Unexpected content at the end of the stream"); // TODO: better exception
+
+ // TODO: check also open sequences etc.; maybe in the handler
+
+ checkRemainingItems();
+ // TODO: check the bytes remaining in the buffer
+ if (started) handlers->writeStreamEnd();
+ }
+
+};
+
+}
+}
+}
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/ProxyVector.h Sat Jul 24 17:35:27 2021 +0200
@@ -0,0 +1,47 @@
+/**
+ * Relational pipes
+ * Copyright © 2021 František Kučera (Frantovo.cz, GlobalCode.info)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <memory>
+
+namespace relpipe {
+namespace in {
+namespace asn1 {
+namespace lib {
+
+template <typename Class>
+class ProxyVector : public std::vector<std::shared_ptr<Class>> {
+public:
+
+ /**
+ * Call given method on all instances.
+ *
+ * @param method a method of Class
+ * @param arguments arguments of the method
+ */
+ template<typename Method, typename... Arguments> void forward(Method method, Arguments&&... arguments) {
+ for (auto& h : * this) {
+ // TODO: optionally handle/collect exceptions
+ (h.get()->*method)(arguments...);
+ }
+ }
+};
+
+}
+}
+}
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/TransactionalBuffer.h Sat Jul 24 17:35:27 2021 +0200
@@ -0,0 +1,139 @@
+/**
+ * Relational pipes
+ * Copyright © 2021 František Kučera (Frantovo.cz, GlobalCode.info)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <string>
+#include <exception>
+#include <cstring>
+
+#include "AbstractParser.h"
+
+namespace relpipe {
+namespace in {
+namespace asn1 {
+namespace lib {
+
+class TransactionalBuffer {
+private:
+ uint8_t* buffer;
+ size_t bufferSize;
+ bool hasData = false;
+ size_t writePosition = 0;
+ // size_t writePositionCommited = 0; // TODO: transactions also on the writing side?
+ size_t readPosition = 0;
+ size_t readPositionCommited = 0;
+
+ size_t bytesRead = 0;
+ size_t bytesReadCommited = 0;
+
+ size_t availableForReading() {
+ if (readPosition < writePosition) return writePosition - readPosition;
+ else if (readPosition > writePosition) return bufferSize - readPosition + writePosition;
+ else if (hasData) return bufferSize;
+ else return 0;
+ }
+
+ size_t availableForWriting() {
+ if (readPositionCommited < writePosition) return bufferSize - writePosition + readPositionCommited;
+ else if (readPositionCommited > writePosition) return readPositionCommited - writePosition;
+ else if (hasData) return 0;
+ else return bufferSize;
+ }
+
+public:
+
+ /**
+ * Is thrown from read() and peak() methods if there are not enough data.
+ * Interrupts current update() cycle and causes rollback.
+ */
+ class ReadBufferUnderflowException {
+ // TODO: common super-class for exceptions, hidden implementation
+ };
+
+ /**
+ * Is thrown from write() method of the buffer when there is not enough space.
+ * Interrupts whole process.
+ */
+ class WriteBufferOverflowException {
+ };
+
+ TransactionalBuffer(size_t initialSize = 4096) : bufferSize(initialSize) {
+ // TODO: initial size + resize + hard upper limit
+ buffer = (uint8_t*) malloc(bufferSize);
+ }
+
+ virtual ~TransactionalBuffer() {
+ free(buffer);
+ }
+
+ void write(const uint8_t* inputBuffer, const size_t length) {
+ if (length == 0) return;
+ if (length > availableForWriting()) throw WriteBufferOverflowException(); // TODO: optional resize
+
+ hasData = true;
+
+ const size_t a = std::min(length, bufferSize - writePosition);
+ const size_t b = length - a;
+
+ // TODO: map buffer twice in the memory in order to create a continuous area and avoid the second memcpy()?
+ memcpy(buffer + writePosition, inputBuffer, a);
+ memcpy(buffer, inputBuffer + a, b);
+
+ writePosition = (writePosition + length) % bufferSize;
+ }
+
+ void read(uint8_t* outputBuffer, const size_t length) {
+ if (length == 0) return;
+
+ peek(outputBuffer, length);
+ readPosition = (readPosition + length) % bufferSize;
+ hasData = readPosition != writePosition;
+ bytesRead += length;
+ }
+
+ void peek(uint8_t* outputBuffer, const size_t length) {
+ if (length == 0) return;
+ if (length > availableForReading()) throw ReadBufferUnderflowException();
+
+ const size_t a = std::min(length, bufferSize - readPosition);
+ const size_t b = length - a;
+
+ memcpy(outputBuffer, buffer + readPosition, a);
+ memcpy(outputBuffer + a, buffer, b);
+ }
+
+ void commitRead() {
+ readPositionCommited = readPosition;
+ bytesReadCommited = bytesRead;
+ }
+
+ void rollbackRead() {
+ readPosition = readPositionCommited;
+ bytesRead = bytesReadCommited;
+ }
+
+ size_t getBytesRead() {
+ return bytesRead;
+ }
+
+};
+
+
+}
+}
+}
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/ValidatingASN1ContentHandler.h Sat Jul 24 17:35:27 2021 +0200
@@ -0,0 +1,105 @@
+/**
+ * Relational pipes
+ * Copyright © 2021 František Kučera (Frantovo.cz, GlobalCode.info)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <string>
+#include <stdexcept>
+
+#include "ASN1ContentHandler.h"
+
+namespace relpipe {
+namespace in {
+namespace asn1 {
+namespace lib {
+
+/**
+ * Does not generate any output but enforces certain rules (related to stream and collection start/end).
+ * When invalid behavior occurs, an exception is thrown.
+ *
+ */
+class ValidatingASN1ContentHandler : public ASN1ContentHandler {
+private:
+ bool streamOpened = false;
+ bool streamClosed = false;
+ size_t collectionLevel = 0;
+public:
+
+ virtual void writeStreamStart() {
+ if (streamOpened) throw std::logic_error("ValidatingASN1ContentHandler: tried to open the stream twice"); // TODO: better exception
+ if (streamClosed) throw std::logic_error("ValidatingASN1ContentHandler: tried to open the stream that was already closed"); // TODO: better exception
+ streamOpened = true;
+ }
+
+ virtual void writeStreamEnd() {
+ if (streamClosed) throw std::logic_error("ValidatingASN1ContentHandler: tried to close the stream twice"); // TODO: better exception
+ if (!streamOpened) throw std::logic_error("ValidatingASN1ContentHandler: tried to close a stream that was already closed or never opened"); // TODO: better exception
+ if (collectionLevel != 0) throw std::logic_error("ValidatingASN1ContentHandler: not all opened collections was closed – remaining: " + std::to_string(collectionLevel)); // TODO: better exception
+ streamClosed = true;
+ }
+
+ virtual void writeCollectionStart(const Header& header) {
+ if (!streamOpened) throw std::logic_error("ValidatingASN1ContentHandler: tried to open a collection while the stream was not opened"); // TODO: better exception
+ if (streamClosed) throw std::logic_error("ValidatingASN1ContentHandler: tried to open a collection while the stream was already closed"); // TODO: better exception
+ collectionLevel++;
+ }
+
+ virtual void writeCollectionEnd() {
+ if (streamClosed) throw std::logic_error("ValidatingASN1ContentHandler: tried to close a collection while the stream was already closed"); // TODO: better exception
+ if (collectionLevel == 0) throw std::logic_error("ValidatingASN1ContentHandler: tried to close a collection while none was opened"); // TODO: better exception
+ collectionLevel--;
+ }
+
+ virtual void finalCheck() {
+ if (!streamOpened) throw std::logic_error("ValidatingASN1ContentHandler: the stream was not opened at all"); // TODO: better exception
+ if (!streamClosed) throw std::logic_error("ValidatingASN1ContentHandler: the stream was opened but not closed"); // TODO: better exception
+ }
+
+ // These events are intentionally ignored:
+
+ void writeBitString(const Header& header, std::vector<bool> value) override {
+ }
+
+ void writeBoolean(const Header& header, bool value) override {
+ }
+
+ void writeDateTime(const Header& header, DateTime value) override {
+ }
+
+ void writeInteger(const Header& header, Integer value) override {
+ }
+
+ void writeNull(const Header& header) override {
+ }
+
+ void writeOID(const Header& header, ObjectIdentifier value) override {
+ }
+
+ void writeOctetString(const Header& header, std::string value) override {
+ }
+
+ void writeTextString(const Header& header, std::string value) override {
+ }
+
+ void writeSpecific(const Header& header, std::string value) override {
+ }
+
+};
+
+}
+}
+}
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/uri.h Sat Jul 24 17:35:27 2021 +0200
@@ -0,0 +1,44 @@
+/**
+ * Relational pipes
+ * Copyright © 2021 František Kučera (Frantovo.cz, GlobalCode.info;
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+namespace relpipe {
+namespace in {
+namespace asn1 {
+namespace lib {
+
+// TODO: these strings will become globally unique URIs (or IRIs) after moving to alt2xml and relative/unprefixed names should also work
+
+/** general options of the ASN.1 parser */
+namespace option {
+static const char* Encoding = "encoding";
+static const char* ParseEncapsulated = "parse-encapsulated";
+}
+
+namespace encoding {
+static const char* ber = "BER";
+static const char* der = "DER";
+static const char* cer = "CER";
+static const char* xer = "XER";
+static const char* per = "PER";
+static const char* asn1 = "ASN.1"; // schema, model
+}
+
+}
+}
+}
+}