bytetokensource.h

Go to the documentation of this file.

/* DynGenPar: Dynamic Generalized Parser - Byte token source
   Copyright (C) 2011-2012 Kevin Kofler <kevin.kofler@chello.at>
   Copyright (C) 2015 DAGOPT Optimization Technologies GmbH
                      written by Kevin Kofler <kofler@dagopt.com>

   Support by the Austrian Science Fund FWF under contract numbers
   P20631 and P23554 is gratefully acknowledged.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#pragma once

#ifndef DYNGENPAR_INTEGER_CATEGORIES
#define DYNGENPAR_INTEGER_CATEGORIES
#endif

#include <cstdio>
#include <QIODevice>
#include <QFile>
#include <QBuffer>

#include "dyngenpar.h"

enum ByteTokens {
  ByteTokenEpsilon = 0,
  ByteTokenNul = 256, 
  ByteTokenError = 257
};
Q_DECLARE_TYPEINFO(ByteTokens, Q_PRIMITIVE_TYPE);

namespace DynGenPar {

class ByteTokenSource : public TokenSource {
  public:
    ByteTokenSource() : TokenSource(), stream(new QFile()) {
      static_cast<QFile *>(stream)->open(stdin, QIODevice::ReadOnly);
    }
    ByteTokenSource(const QString &fileName)
      : TokenSource(), stream(new QFile(fileName)) {
      stream->open(QIODevice::ReadOnly);
    }
    virtual ~ByteTokenSource() {delete stream;}
    virtual bool rewindTo(int pos, const LexerState & = LexerState()) {
      // Sequential streams cannot be rewound.
      if (stream->isSequential())
        return TokenSource::rewindTo(pos);
      else {
        if (pos != currPos && !stream->seek(pos)) {
          tree = Node();
          return false;
        }
        return simpleRewind(pos);
      }
    }
    void setInputStdin() {
      delete stream;
      stream = new QFile();
      static_cast<QFile *>(stream)->open(stdin, QIODevice::ReadOnly);
      reset();
    }
    void setInputFile(const QString &fileName) {
      delete stream;
      stream = new QFile(fileName);
      stream->open(QIODevice::ReadOnly);
      reset();
    }
    void setInputBytes(const QByteArray &bytes) {
      delete stream;
      stream = new QBuffer();
      static_cast<QBuffer *>(stream)->setData(bytes);
      stream->open(QIODevice::ReadOnly | QIODevice::Unbuffered);
      reset();
    }
    void setInputString(const QString &string) {
      setInputBytes(string.toLocal8Bit());
    }
    void setInputBuffer(QByteArray *buffer) {
      delete stream;
      stream = new QBuffer(buffer);
      stream->open(QIODevice::ReadOnly | QIODevice::Unbuffered);
      reset();
    }
  protected:
    virtual Cat readToken() {
      if (stream->atEnd()) return ByteTokenEpsilon;
      char c;
      if (!stream->getChar(&c)) return ByteTokenError;
      if (c) return (unsigned char)c; else return ByteTokenNul;
    }
    QIODevice *stream;
    virtual void reset() {
      tree = Node();
      currPos = 0;
    }
};
DYNGENPAR_DECLARE_TYPEINFO(ByteTokenSource, Q_MOVABLE_TYPE);


class TextByteLexerStateData : public AbstractLexerStateData {
  public:
    TextByteLexerStateData(qint64 streamPosition, TextPosition textPosition)
      : AbstractLexerStateData(), streamPos(streamPosition),
        textPos(textPosition) {}
    virtual AbstractLexerStateData *clone() {
      return new TextByteLexerStateData(*this);
    }
    qint64 streamPos;
    TextPosition textPos;
};
DYNGENPAR_DECLARE_TYPEINFO(TextByteLexerStateData, Q_MOVABLE_TYPE);

class TextByteTokenSource : public ByteTokenSource {
  public:
    TextByteTokenSource() : ByteTokenSource() {}
    TextByteTokenSource(const QString &fileName) : ByteTokenSource(fileName) {}
    virtual ~TextByteTokenSource() {}
    virtual bool rewindTo(int pos,
                          const LexerState &lexerState = LexerState()) {
      /* Sequential streams cannot be rewound at all.
         For other streams, we need a non-null lexer state to rewind to a
         non-zero position. */
      if (stream->isSequential() || (pos && lexerState.isNull()))
        return TokenSource::rewindTo(pos);
      else if (!pos && currPos) { // rewind to position zero = reset
        stream->reset();
        reset();
        return true;
      } else {
        if (pos != currPos) {
          const TextByteLexerStateData *data
            = static_cast<const TextByteLexerStateData *>(lexerState.data());
          if (!stream->seek(data->streamPos)) {
            tree = Node();
            return false;
          }
          textPos = data->textPos;
        }
        return simpleRewind(pos);
      }
    }
    virtual LexerState saveState() {
      TextByteLexerStateData *data
        = new TextByteLexerStateData(stream->pos(), textPos);
      return LexerState(data);
    }
    static TextPosition textPosition(const LexerState &lexerState) {
      return lexerState.isNull() ? TextPosition() :
               static_cast<const TextByteLexerStateData *>(lexerState.data())
                 ->textPos;
    }
  protected:
    virtual Cat readToken() {
      Cat token;
      do {
        token = ByteTokenSource::readToken();
      } while (token == '\r');
      if (token && token <= ByteTokenNul)
        textPos.countCharacter((unsigned char) token);
      return token;
    }
    virtual void reset() {
      ByteTokenSource::reset();
      textPos.reset();
    }
  private:
    TextPosition textPos;
};
DYNGENPAR_DECLARE_TYPEINFO(TextByteTokenSource, Q_MOVABLE_TYPE);

} // end namespace