qgeofiletilecache.cpp

Go to the documentation of this file.

// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
// Qt-Security score:critical reason:filesystem-manipulation
#include "qgeofiletilecache_p.h"
 
#include "qgeotilespec_p.h"
 
#include "qgeomappingmanager_p.h"
 
#include <QDir>
#include <QStandardPaths>
#include <QMetaType>
#include <QPixmap>
#include <QDebug>
 
#include <QtCore/qsavefile.h>
 
QT_BEGIN_NAMESPACE
 
using namespace Qt::StringLiterals;
 
class QGeoCachedTileMemory
{
public:
    ~QGeoCachedTileMemory()
    {
        if (cache)
            cache->evictFromMemoryCache(this);
    }
 
    QGeoTileSpec spec;
    QGeoFileTileCache *cache;
    QByteArray bytes;
    QString format;
};
 
void QCache3QTileEvictionPolicy::aboutToBeRemoved(const QGeoTileSpec &key, QSharedPointer<QGeoCachedTileDisk> obj)
{
    Q_UNUSED(key);
    // set the cache pointer to zero so we can't call evictFromDiskCache
    obj->cache = nullptr;
}
 
void QCache3QTileEvictionPolicy::aboutToBeEvicted(const QGeoTileSpec &key, QSharedPointer<QGeoCachedTileDisk> obj)
{
    Q_UNUSED(key);
    Q_UNUSED(obj);
    // leave the pointer set if it's a real eviction
}
 
QGeoCachedTileDisk::~QGeoCachedTileDisk()
{
    if (cache)
        cache->evictFromDiskCache(this);
}
 
QGeoFileTileCache::QGeoFileTileCache(const QString &directory, QObject *parent)
    : QAbstractGeoTileCache(parent), directory_(directory)
{
}
 
void QGeoFileTileCache::init()
{
    const QString basePath = baseCacheDirectory() + QLatin1String("QtLocation/");
 
    // delete old tiles from QtLocation 5.7 or prior
    // Newer version use plugin-specific subdirectories, versioned with qt version so those are not affected.
    // TODO Remove cache cleanup in Qt 6
    QDir baseDir(basePath);
    if (baseDir.exists()) {
        const QStringList oldCacheFiles = baseDir.entryList(QDir::Files);
        for (const QString& file : oldCacheFiles)
            baseDir.remove(file);
        const QStringList oldCacheDirs = { QStringLiteral("osm"), QStringLiteral("mapbox"), QStringLiteral("here") };
        for (const QString& d : oldCacheDirs) {
            QDir oldCacheDir(basePath + QLatin1Char('/') + d);
            if (oldCacheDir.exists())
                oldCacheDir.removeRecursively();
        }
    }
 
    if (directory_.isEmpty()) {
        directory_ = baseLocationCacheDirectory();
        qWarning() << "Plugin uses uninitialized QGeoFileTileCache directory which was deleted during startup";
    }
 
    const bool directoryCreated = QDir::root().mkpath(directory_);
    if (!directoryCreated)
        qWarning() << "Failed to create cache directory " << directory_;
 
    // default values
    if (!isDiskCostSet_) { // If setMaxDiskUsage has not been called yet
        if (costStrategyDisk_ == ByteSize)
            setMaxDiskUsage(50 * 1024 * 1024);
        else
            setMaxDiskUsage(1000);
    }
 
    if (!isMemoryCostSet_) { // If setMaxMemoryUsage has not been called yet
        if (costStrategyMemory_ == ByteSize)
            setMaxMemoryUsage(3 * 1024 * 1024);
        else
            setMaxMemoryUsage(100);
    }
 
    if (!isTextureCostSet_) { // If setExtraTextureUsage has not been called yet
        if (costStrategyTexture_ == ByteSize)
            setExtraTextureUsage(6 * 1024 * 1024);
        else
            setExtraTextureUsage(30); // byte size of texture is >> compressed image, hence unitary cost should be lower
    }
 
    loadTiles();
}
 
void QGeoFileTileCache::loadTiles()
{
    QStringList formats;
    formats << QLatin1String("*.*");
 
    QDir dir(directory_);
    const QStringList files = dir.entryList(formats, QDir::Files);
#if 0 // workaround for QTBUG-60581
    // Method:
    // 1. read each queue file then, if each file exists, deserialize the data into the appropriate
    // cache queue.
    for (int i = 1; i<=4; i++) {
        QString filename = dir.filePath(QString::fromLatin1("queue") + QString::number(i));
        QFile file(filename);
        if (!file.open(QIODevice::ReadOnly))
            continue;
        QList<QSharedPointer<QGeoCachedTileDisk> > queue;
        QList<QGeoTileSpec> specs;
        QList<int> costs;
        while (!file.atEnd()) {
            QByteArray line = file.readLine().trimmed();
            QString filename = QString::fromLatin1(line.constData(), line.length());
            if (dir.exists(filename)){
                files.removeOne(filename);
                QGeoTileSpec spec = filenameToTileSpec(filename);
                if (spec.zoom() == -1)
                    continue;
                QSharedPointer<QGeoCachedTileDisk> tileDisk(new QGeoCachedTileDisk);
                tileDisk->filename = dir.filePath(filename);
                tileDisk->cache = this;
                tileDisk->spec = spec;
                QFileInfo fi(tileDisk->filename);
                specs.append(spec);
                queue.append(tileDisk);
                if (costStrategyDisk_ == ByteSize)
                    costs.append(fi.size());
                else
                    costs.append(1);
 
            }
        }
 
        diskCache_.deserializeQueue(i, specs, queue, costs);
        file.close();
    }
#endif
    // 2. remaining tiles that aren't registered in a queue get pushed into cache here
    // this is a backup, in case the queue manifest files get deleted or out of sync due to
    // the application not closing down properly
    for (const auto &file : files) {
        QGeoTileSpec spec = filenameToTileSpec(file);
        if (spec.zoom() == -1)
            continue;
        QString filename = dir.filePath(file);
        addToDiskCache(spec, filename);
    }
}
 
QGeoFileTileCache::~QGeoFileTileCache()
{
#if 0 // workaround for QTBUG-60581
    // write disk cache queues to disk
    QDir dir(directory_);
    for (int i = 1; i<=4; i++) {
        QString filename = dir.filePath(QString::fromLatin1("queue") + QString::number(i));
        QFile file(filename);
        if (!file.open(QIODevice::WriteOnly)){
            qWarning() << "Unable to write tile cache file " << filename;
            continue;
        }
        QList<QSharedPointer<QGeoCachedTileDisk> > queue;
        diskCache_.serializeQueue(i, queue);
        for (const QSharedPointer<QGeoCachedTileDisk> &tile : queue) {
            if (tile.isNull())
                continue;
 
            // we just want the filename here, not the full path
            int index = tile->filename.lastIndexOf(QLatin1Char('/'));
            QByteArray filename = tile->filename.mid(index + 1).toLatin1() + '\n';
            file.write(filename);
        }
        file.close();
    }
#endif
}
 
void QGeoFileTileCache::printStats()
{
    textureCache_.printStats();
    memoryCache_.printStats();
    diskCache_.printStats();
}
 
void QGeoFileTileCache::setMaxDiskUsage(int diskUsage)
{
    diskCache_.setMaxCost(diskUsage);
    isDiskCostSet_ = true;
}
 
int QGeoFileTileCache::maxDiskUsage() const
{
    return diskCache_.maxCost();
}
 
int QGeoFileTileCache::diskUsage() const
{
    return diskCache_.totalCost();
}
 
void QGeoFileTileCache::setMaxMemoryUsage(int memoryUsage)
{
    memoryCache_.setMaxCost(memoryUsage);
    isMemoryCostSet_ = true;
}
 
int QGeoFileTileCache::maxMemoryUsage() const
{
    return memoryCache_.maxCost();
}
 
int QGeoFileTileCache::memoryUsage() const
{
    return memoryCache_.totalCost();
}
 
void QGeoFileTileCache::setExtraTextureUsage(int textureUsage)
{
    extraTextureUsage_ = textureUsage;
    textureCache_.setMaxCost(minTextureUsage_ + extraTextureUsage_);
    isTextureCostSet_ = true;
}
 
void QGeoFileTileCache::setMinTextureUsage(int textureUsage)
{
    minTextureUsage_ = textureUsage;
    textureCache_.setMaxCost(minTextureUsage_ + extraTextureUsage_);
}
 
int QGeoFileTileCache::maxTextureUsage() const
{
    return textureCache_.maxCost();
}
 
int QGeoFileTileCache::minTextureUsage() const
{
    return minTextureUsage_;
}
 
 
int QGeoFileTileCache::textureUsage() const
{
    return textureCache_.totalCost();
}
 
void QGeoFileTileCache::clearAll()
{
    textureCache_.clear();
    memoryCache_.clear();
    diskCache_.clear();
    QDir dir(directory_);
    dir.setNameFilters(QStringList() << QLatin1String("*-*-*-*.*"));
    dir.setFilter(QDir::Files);
    const auto entries = dir.entryList();
    for (const QString &dirFile : entries) {
        dir.remove(dirFile);
    }
}
 
void QGeoFileTileCache::clearMapId(const int mapId)
{
    {
        const auto keys = diskCache_.keys();
        for (const QGeoTileSpec &k : keys)
            if (k.mapId() == mapId)
                diskCache_.remove(k, true);
    }
    {
        const auto keys = memoryCache_.keys();
        for (const QGeoTileSpec &k : keys)
            if (k.mapId() == mapId)
                memoryCache_.remove(k);
    }
    {
        const auto keys = textureCache_.keys();
        for (const QGeoTileSpec &k : keys)
            if (k.mapId() == mapId)
                textureCache_.remove(k);
    }
 
    // TODO: It seems the cache leaves residues, like some tiles do not get picked up.
    // After the above calls, files that shouldnt be left behind are still on disk.
    // Do an additional pass and make sure what has to be deleted gets deleted.
    QDir dir(directory_);
    QStringList formats;
    formats << QLatin1String("*.*");
    const QStringList files = dir.entryList(formats, QDir::Files);
    qWarning() << "Old tile data detected. Cache eviction left out "<< files.size() << "tiles";
    for (const QString &tileFileName : files) {
        QGeoTileSpec spec = filenameToTileSpec(tileFileName);
        if (spec.mapId() != mapId)
            continue;
        QFile::remove(dir.filePath(tileFileName));
    }
}
 
void QGeoFileTileCache::setCostStrategyDisk(QAbstractGeoTileCache::CostStrategy costStrategy)
{
    costStrategyDisk_ = costStrategy;
}
 
QAbstractGeoTileCache::CostStrategy QGeoFileTileCache::costStrategyDisk() const
{
    return costStrategyDisk_;
}
 
void QGeoFileTileCache::setCostStrategyMemory(QAbstractGeoTileCache::CostStrategy costStrategy)
{
    costStrategyMemory_ = costStrategy;
}
 
QAbstractGeoTileCache::CostStrategy QGeoFileTileCache::costStrategyMemory() const
{
    return costStrategyMemory_;
}
 
void QGeoFileTileCache::setCostStrategyTexture(QAbstractGeoTileCache::CostStrategy costStrategy)
{
    costStrategyTexture_ = costStrategy;
}
 
QAbstractGeoTileCache::CostStrategy QGeoFileTileCache::costStrategyTexture() const
{
    return costStrategyTexture_;
}
 
QSharedPointer<QGeoTileTexture> QGeoFileTileCache::get(const QGeoTileSpec &spec)
{
    QSharedPointer<QGeoTileTexture> tt = getFromMemory(spec);
    if (tt)
        return tt;
    return getFromDisk(spec);
}
 
void QGeoFileTileCache::insert(const QGeoTileSpec &spec,
                           const QByteArray &bytes,
                           const QString &format,
                           QAbstractGeoTileCache::CacheAreas areas)
{
    if (bytes.isEmpty())
        return;
 
    if (areas & QAbstractGeoTileCache::DiskCache) {
        QString filename = tileSpecToFilename(spec, format, directory_);
        addToDiskCache(spec, filename, bytes);
    }
 
    if (areas & QAbstractGeoTileCache::MemoryCache) {
        addToMemoryCache(spec, bytes, format);
    }
 
    /* inserts do not hit the texture cache -- this actually reduces overall
     * cache hit rates because many tiles come too late to be useful
     * and act as a poison */
}
 
QString QGeoFileTileCache::tileSpecToFilenameDefault(const QGeoTileSpec &spec, const QString &format, const QString &directory)
{
    QString filename = spec.plugin();
    filename += QLatin1String("-");
    filename += QString::number(spec.mapId());
    filename += QLatin1String("-");
    filename += QString::number(spec.zoom());
    filename += QLatin1String("-");
    filename += QString::number(spec.x());
    filename += QLatin1String("-");
    filename += QString::number(spec.y());
 
    //Append version if real version number to ensure backwards compatibility and eviction of old tiles
    if (spec.version() != -1) {
        filename += QLatin1String("-");
        filename += QString::number(spec.version());
    }
 
    filename += QLatin1String(".");
    filename += format;
 
    QDir dir = QDir(directory);
 
    return dir.filePath(filename);
}
 
QGeoTileSpec QGeoFileTileCache::filenameToTileSpecDefault(const QString &filename)
{
    QGeoTileSpec emptySpec;
 
    const QStringList parts = filename.split(QLatin1Char('.'));
 
    if (parts.length() != 2)
        return emptySpec;
 
    const QString name = parts.at(0);
    const QStringList fields = name.split(QLatin1Char('-'));
 
    qsizetype length = fields.length();
    if (length != 5 && length != 6)
        return emptySpec;
 
    QList<int> numbers;
 
    bool ok = false;
    for (qsizetype i = 1; i < length; ++i) {
        ok = false;
        int value = fields.at(i).toInt(&ok);
        if (!ok)
            return emptySpec;
        numbers.append(value);
    }
 
    //File name without version, append default
    if (numbers.length() < 5)
        numbers.append(-1);
 
    return QGeoTileSpec(fields.at(0),
                    numbers.at(0),
                    numbers.at(1),
                    numbers.at(2),
                    numbers.at(3),
                    numbers.at(4));
}
 
void QGeoFileTileCache::evictFromDiskCache(QGeoCachedTileDisk *td)
{
    QFile::remove(td->filename);
}
 
void QGeoFileTileCache::evictFromMemoryCache(QGeoCachedTileMemory * /* tm  */)
{
}
 
QSharedPointer<QGeoCachedTileDisk> QGeoFileTileCache::addToDiskCache(const QGeoTileSpec &spec, const QString &filename)
{
    QSharedPointer<QGeoCachedTileDisk> td(new QGeoCachedTileDisk);
    td->spec = spec;
    td->filename = filename;
    td->cache = this;
 
    int cost = 1;
    if (costStrategyDisk_ == ByteSize) {
        QFileInfo fi(filename);
        cost = fi.size();
    }
    diskCache_.insert(spec, td, cost);
    return td;
}
 
bool QGeoFileTileCache::addToDiskCache(const QGeoTileSpec &spec, const QString &filename, const QByteArray &bytes)
{
    int cost = 1;
    if (costStrategyDisk_ == ByteSize)
        cost = bytes.size();
 
    if (cost > diskCache_.maxCost())
        return false; // insert() would fail anyway, so don't even try
 
    QSaveFile file(filename);
    if (!file.open(QIODevice::WriteOnly))
        return false;
    file.write(bytes);
    if (!file.commit())
        return false;
 
    QSharedPointer<QGeoCachedTileDisk> td(new QGeoCachedTileDisk);
    td->spec = spec;
    td->filename = filename;
    td->cache = this;
 
    [[maybe_unused]] const bool inserted = diskCache_.insert(spec, td, cost);
    Q_ASSERT(inserted);
 
    return true;
}
 
void QGeoFileTileCache::addToMemoryCache(const QGeoTileSpec &spec, const QByteArray &bytes, const QString &format)
{
    if (isTileBogus(bytes))
        return;
 
    QSharedPointer<QGeoCachedTileMemory> tm(new QGeoCachedTileMemory);
    tm->spec = spec;
    tm->cache = this;
    tm->bytes = bytes;
    tm->format = format;
 
    int cost = 1;
    if (costStrategyMemory_ == ByteSize)
        cost = bytes.size();
    memoryCache_.insert(spec, tm, cost);
}
 
QSharedPointer<QGeoTileTexture> QGeoFileTileCache::addToTextureCache(const QGeoTileSpec &spec, const QImage &image)
{
    QSharedPointer<QGeoTileTexture> tt(new QGeoTileTexture);
    tt->spec = spec;
    tt->image = image;
 
    int cost = 1;
    if (costStrategyTexture_ == ByteSize)
        cost = image.width() * image.height() * image.depth() / 8;
    textureCache_.insert(spec, tt, cost);
 
    return tt;
}
 
QSharedPointer<QGeoTileTexture> QGeoFileTileCache::getFromMemory(const QGeoTileSpec &spec)
{
    QSharedPointer<QGeoTileTexture> tt = textureCache_.object(spec);
    if (tt)
        return tt;
 
    QSharedPointer<QGeoCachedTileMemory> tm = memoryCache_.object(spec);
    if (tm) {
        QImage image;
        if (!image.loadFromData(tm->bytes)) {
            handleError(spec, QLatin1String("Problem with tile image"));
            return QSharedPointer<QGeoTileTexture>();
        }
        QSharedPointer<QGeoTileTexture> tt = addToTextureCache(spec, image);
        if (tt)
            return tt;
    }
    return QSharedPointer<QGeoTileTexture>();
}
 
QSharedPointer<QGeoTileTexture> QGeoFileTileCache::getFromDisk(const QGeoTileSpec &spec)
{
    QSharedPointer<QGeoCachedTileDisk> td = diskCache_.object(spec);
    if (td) {
        const QString format = QFileInfo(td->filename).suffix();
        QFile file(td->filename);
        if (!file.open(QIODevice::ReadOnly)) {
            handleError(spec, "Cannot open file %1: %2"_L1.arg(file.fileName(), file.errorString()));
            return nullptr;
        }
        QByteArray bytes = file.readAll();
        file.close();
 
        QImage image;
        // Some tiles from the servers could be valid images but the tile fetcher
        // might be able to recognize them as tiles that should not be shown.
        // If that's the case, the tile fetcher should write "NoRetry" inside the file.
        if (isTileBogus(bytes)) {
            QSharedPointer<QGeoTileTexture> tt(new QGeoTileTexture);
            tt->spec = spec;
            tt->image = image;
            return tt;
        }
 
        // This is a truly invalid image. The fetcher should try again.
        if (!image.loadFromData(bytes)) {
            handleError(spec, QLatin1String("Problem with tile image"));
            return QSharedPointer<QGeoTileTexture>();
        }
 
        // Converting it here, instead of in each QSGTexture::bind()
        if (image.format() != QImage::Format_RGB32 && image.format() != QImage::Format_ARGB32_Premultiplied)
            image = image.convertToFormat(QImage::Format_ARGB32_Premultiplied);
 
        addToMemoryCache(spec, bytes, format);
        QSharedPointer<QGeoTileTexture> tt = addToTextureCache(td->spec, image);
        if (tt)
            return tt;
    }
 
    return QSharedPointer<QGeoTileTexture>();
}
 
bool QGeoFileTileCache::isTileBogus(const QByteArray &bytes) const
{
    if (bytes.size() == 7 && bytes == QByteArrayLiteral("NoRetry"))
        return true;
    return false;
}
 
QString QGeoFileTileCache::tileSpecToFilename(const QGeoTileSpec &spec, const QString &format, const QString &directory) const
{
    return tileSpecToFilenameDefault(spec, format, directory);
}
 
QGeoTileSpec QGeoFileTileCache::filenameToTileSpec(const QString &filename) const
{
    return filenameToTileSpecDefault(filename);
}
 
QString QGeoFileTileCache::directory() const
{
    return directory_;
}
 
QT_END_NAMESPACE