scanner.cpp

Go to the documentation of this file.

// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
 
#include "scanner.h"
#include "logging.h"
 
#include <QtCore/qdir.h>
#include <QtCore/qhash.h>
#include <QtCore/qjsonarray.h>
#include <QtCore/qjsondocument.h>
#include <QtCore/qjsonobject.h>
#include <QtCore/qtextstream.h>
#include <QtCore/qvariant.h>
 
#include <iostream>
 
using namespace Qt::Literals::StringLiterals;
 
namespace Scanner {
 
static void missingPropertyWarning(const QString &filePath, const QString &property)
{
    std::cerr << qPrintable(tr("File %1: Missing mandatory property '%2'.").arg(
                                QDir::toNativeSeparators(filePath), property)) << std::endl;
}
 
bool validatePackage(Package &p, Checks checks, LogLevel logLevel)
{
    const auto &filePath = p.filePath;
    bool validPackage = true;
 
    if (filePath.isEmpty()) {
        std::cerr << qPrintable(tr("The origin file of package '%1' was not recorded.")
            .arg(p.id));
        return false;
    }
 
    if (p.qtParts.isEmpty())
        p.qtParts << u"libs"_s;
 
    if (p.name.isEmpty()) {
        if (p.id.startsWith("chromium-"_L1)) // Ignore invalid README.chromium files
            return false;
 
        if (logLevel != SilentLog)
            missingPropertyWarning(filePath, u"Name"_s);
        validPackage = false;
    }
 
    if (p.id.isEmpty()) {
        if (logLevel != SilentLog)
            missingPropertyWarning(filePath, u"Id"_s);
        validPackage = false;
    } else if (!p.id.isLower() || p.id.contains(' '_L1)) {
        if (logLevel != SilentLog)
            std::cerr << qPrintable(tr("File %1: Value of 'Id' must be in lowercase and without spaces.")
                                        .arg(QDir::toNativeSeparators(filePath))) << std::endl;
        validPackage = false;
    }
 
    if (p.license.isEmpty()) {
        if (logLevel != SilentLog)
            missingPropertyWarning(filePath, u"License"_s);
        validPackage = false;
    }
 
    if (!p.copyright.isEmpty() && !p.copyrightFile.isEmpty()) {
        if (logLevel != SilentLog) {
            std::cerr << qPrintable(tr("File %1: Properties 'Copyright' and 'CopyrightFile' are "
                                       "mutually exclusive.")
                                            .arg(QDir::toNativeSeparators(filePath)))
                      << std::endl;
        }
        validPackage = false;
    }
 
    if (p.securityCritical && p.downloadLocation.isEmpty()) {
        if (logLevel != SilentLog)
            missingPropertyWarning(filePath, u"DownloadLocation"_s);
        validPackage = false;
    }
 
    for (const QString &part : std::as_const(p.qtParts)) {
        if (part != "examples"_L1 && part != "tests"_L1
            && part != "tools"_L1 && part != "libs"_L1) {
 
            if (logLevel != SilentLog) {
                std::cerr << qPrintable(tr("File %1: Property 'QtPart' contains unknown element "
                                           "'%2'. Valid entries are 'examples', 'tests', 'tools' "
                                           "and 'libs'.").arg(
                                            QDir::toNativeSeparators(filePath), part))
                          << std::endl;
            }
            validPackage = false;
        }
    }
 
    if (!(checks & Check::Paths))
        return validPackage;
 
    const QDir dir = p.path;
    if (!dir.exists()) {
        std::cerr << qPrintable(
                tr("File %1: Directory '%2' does not exist.")
                        .arg(QDir::toNativeSeparators(filePath), QDir::toNativeSeparators(p.path)))
                  << std::endl;
        validPackage = false;
    } else {
        for (const QString &file : std::as_const(p.files)) {
            if (!dir.exists(file)) {
                if (logLevel != SilentLog) {
                    std::cerr << qPrintable(
                            tr("File %1: Path '%2' does not exist in directory '%3'.")
                                    .arg(QDir::toNativeSeparators(filePath),
                                         QDir::toNativeSeparators(file),
                                         QDir::toNativeSeparators(p.path)))
                              << std::endl;
                }
                validPackage = false;
            }
        }
    }
 
    return validPackage;
}
 
static std::optional<QStringList> toStringList(const QJsonValue &value)
{
    if (!value.isArray())
        return std::nullopt;
    QStringList result;
    for (const auto &iter : value.toArray()) {
        if (iter.type() != QJsonValue::String)
            return std::nullopt;
        result.push_back(iter.toString());
    }
    return result;
}
 
static std::optional<QString> arrayToMultiLineString(const QJsonValue &value)
{
    if (!value.isArray())
        return std::nullopt;
    QString result;
    for (const auto &iter : value.toArray()) {
        if (iter.type() != QJsonValue::String)
            return std::nullopt;
        result.append(iter.toString());
        result.append(QLatin1StringView("\n"));
    }
    return result;
}
 
// Extracts SPDX license ids from a SPDX license expression.
// For "(BSD-3-Clause AND BeerWare)" this function returns { "BSD-3-Clause", "BeerWare" }.
static QStringList extractLicenseIdsFromSPDXExpression(QString expression)
{
    const QStringList spdxOperators = {
        u"AND"_s,
        u"OR"_s,
        u"WITH"_s
    };
 
    // Replace parentheses with spaces. We're not interested in grouping.
    const QRegularExpression parensRegex(u"[()]"_s);
    expression.replace(parensRegex, u" "_s);
 
    // Split the string at space boundaries to extract tokens.
    QStringList result;
    for (const QString &token : expression.split(QLatin1Char(' '), Qt::SkipEmptyParts)) {
        if (spdxOperators.contains(token))
            continue;
 
        // Remove the unary + operator, if present.
        if (token.endsWith(QLatin1Char('+')))
            result.append(token.mid(0, token.size() - 1));
        else
            result.append(token);
    }
    return result;
}
 
// Starting at packageDir, look for a LICENSES subdirectory in the directory hierarchy upwards.
// Return a default-constructed QString if the directory was not found.
static QString locateLicensesDir(const QString &packageDir)
{
    static const QString licensesSubDir = u"LICENSES"_s;
    QDir dir(packageDir);
    while (true) {
        if (!dir.exists())
            break;
        if (dir.cd(licensesSubDir))
            return dir.path();
        if (dir.isRoot() || !dir.cdUp())
            break;
    }
    return {};
}
 
// Locates the license files that belong to the licenses mentioned in LicenseId and stores them in
// the specified package object.
static bool autoDetectLicenseFiles(Package &p)
{
    const QString licensesDirPath = locateLicensesDir(p.path);
    const QStringList licenseIds = extractLicenseIdsFromSPDXExpression(p.licenseId);
 
    bool success = true;
    QDir licensesDir(licensesDirPath);
    QDir licensesDirLocal = p.path;
    for (const QString &id : licenseIds) {
        QString fileName = id + u".txt";
        QString fileNameLocal = u"LICENSE." + id + u".txt";
 
        if (licensesDirLocal.exists(fileNameLocal)) {
            p.licenseFiles.append(licensesDirLocal.filePath(fileNameLocal));
        } else if (licensesDir.exists(fileName)) {
            p.licenseFiles.append(licensesDir.filePath(fileName));
        } else {
            std::cerr << qPrintable(tr("Missing expected license file:")) << std::endl;
            std::cerr << qPrintable(QDir::toNativeSeparators(licensesDirLocal.filePath(fileNameLocal)))
                      << std::endl;
            if (!licensesDirPath.isEmpty()) {
                std::cerr << qPrintable(tr("or\n %1").arg(
                                            QDir::toNativeSeparators(licensesDir.filePath(fileName))))
                          << std::endl;
            }
            success = false;
        }
    }
 
    return success;
}
 
// Tries to interpret a json value either as a string or an array of strings, and assigns the
// result to outList. Returns true on success, false on failure. On failure, it also conditionally
// prints an error.
static bool handleStringOrStringArrayJsonKey(QStringList &outList, const QString &key,
                                      QJsonValueConstRef jsonValue, const QString &filePath,
                                      LogLevel logLevel)
{
    if (jsonValue.isArray()) {
        auto maybeStringList = toStringList(jsonValue);
        if (maybeStringList)
            outList = maybeStringList.value();
    } else if (jsonValue.isString()) {
        outList.append(jsonValue.toString());
    } else {
        if (logLevel != SilentLog) {
            std::cerr << qPrintable(tr("File %1: Expected JSON array of strings or "
                                       "string as value of %2.").arg(
                                        QDir::toNativeSeparators(filePath), key))
                      << std::endl;
        }
        return false;
    }
    return true;
}
 
// Transforms a JSON object into a Package object
static std::optional<Package> readPackage(const QJsonObject &object, const QString &filePath,
                                          LogLevel logLevel)
{
    Package p;
    bool validPackage = true;
    const QString directory = QFileInfo(filePath).absolutePath();
    p.filePath = QFileInfo(filePath).absoluteFilePath();
    p.path = directory;
 
    for (auto iter = object.constBegin(); iter != object.constEnd(); ++iter) {
        const QString key = iter.key();
 
        if (!iter.value().isString() && key != "QtParts"_L1 && key != "SecurityCritical"_L1
            && key != "Files"_L1 && key != "LicenseFiles"_L1 && key != "Comment"_L1
            && key != "Copyright"_L1 && key != "CPE"_L1 && key != "PURL"_L1) {
            if (logLevel != SilentLog)
                std::cerr << qPrintable(tr("File %1: Expected JSON string as value of %2.").arg(
                                            QDir::toNativeSeparators(filePath), key)) << std::endl;
            validPackage = false;
            continue;
        }
        const QString value = iter.value().toString();
        if (key == "Name"_L1) {
            p.name = value;
        } else if (key == "Path"_L1) {
            p.path = QDir(directory).absoluteFilePath(value);
        } else if (key == "Files"_L1) {
            QJsonValueConstRef jsonValue = iter.value();
            if (jsonValue.isArray()) {
                auto maybeStringList = toStringList(jsonValue);
                if (maybeStringList)
                    p.files = maybeStringList.value();
            } else if (jsonValue.isString()) {
                // Legacy format: multiple values separated by space in one string.
                p.files = value.simplified().split(QLatin1Char(' '), Qt::SkipEmptyParts);
            } else {
                if (logLevel != SilentLog) {
                    std::cerr << qPrintable(tr("File %1: Expected JSON array of strings as value "
                                               "of Files.").arg(QDir::toNativeSeparators(filePath)))
                              << std::endl;
                    validPackage = false;
                    continue;
                }
            }
        } else if (key == "Comment"_L1) {
            // Accepted purely to record details of potential interest doing
            // updates in future. Value is an arbitrary object. Any number of
            // Comment entries may be present: JSON doesn't require names to be
            // unique, albeit some linters may kvetch.
        } else if (key == "Id"_L1) {
            p.id = value;
        } else if (key == "Homepage"_L1) {
            p.homepage = value;
        } else if (key == "Version"_L1) {
            p.version = value;
        } else if (key == "DownloadLocation"_L1) {
            p.downloadLocation = value;
        } else if (key == "License"_L1) {
            p.license = value;
        } else if (key == "LicenseId"_L1) {
            p.licenseId = value;
        } else if (key == "LicenseFile"_L1) {
            p.licenseFiles = QStringList(QDir(directory).absoluteFilePath(value));
        } else if (key == "LicenseFiles"_L1) {
            auto strings = toStringList(iter.value());
            if (!strings) {
                if (logLevel != SilentLog)
                    std::cerr << qPrintable(tr("File %1: Expected JSON array of strings in %2.")
                                                    .arg(QDir::toNativeSeparators(filePath), key))
                              << std::endl;
                validPackage = false;
                continue;
            }
            const QDir dir(directory);
            for (const auto &iter : std::as_const(strings.value()))
                p.licenseFiles.push_back(dir.absoluteFilePath(iter));
        } else if (key == "Copyright"_L1) {
            QJsonValueConstRef jsonValue = iter.value();
            if (jsonValue.isArray()) {
                // Array joined with new lines
                auto maybeString = arrayToMultiLineString(jsonValue);
                if (maybeString)
                    p.copyright = maybeString.value();
            } else if (jsonValue.isString()) {
                // Legacy format: multiple values separated by space in one string.
                p.copyright = value;
            } else {
                if (logLevel != SilentLog) {
                    std::cerr << qPrintable(tr("File %1: Expected JSON array of strings or "
                                               "string as value of %2.").arg(
                                                QDir::toNativeSeparators(filePath), key)) << std::endl;
                    validPackage = false;
                    continue;
                }
            }
        } else if (key == "CPE"_L1) {
            const QJsonValueConstRef jsonValue = iter.value();
            if (!handleStringOrStringArrayJsonKey(p.cpeList, key, jsonValue, filePath, logLevel)) {
                validPackage = false;
                continue;
            }
        } else if (key == "PURL"_L1) {
            const QJsonValueConstRef jsonValue = iter.value();
            if (!handleStringOrStringArrayJsonKey(p.purlList, key, jsonValue, filePath, logLevel)) {
                validPackage = false;
                continue;
            }
        } else if (key == "CopyrightFile"_L1) {
            p.copyrightFile = QDir(directory).absoluteFilePath(value);
        } else if (key == "PackageComment"_L1) {
            p.packageComment = value;
        } else if (key == "QDocModule"_L1) {
            p.qdocModule = value;
        } else if (key == "Description"_L1) {
            p.description = value;
        } else if (key == "QtUsage"_L1) {
            p.qtUsage = value;
        } else if (key == "SecurityCritical"_L1) {
            if (!iter.value().isBool()) {
                std::cerr << qPrintable(tr("File %1: Expected JSON boolean in %2.")
                                                .arg(QDir::toNativeSeparators(filePath), key))
                          << std::endl;
                validPackage = false;
                continue;
            }
            p.securityCritical = iter.value().toBool();
        } else if (key == "QtParts"_L1) {
            auto parts = toStringList(iter.value());
            if (!parts) {
                if (logLevel != SilentLog) {
                    std::cerr << qPrintable(tr("File %1: Expected JSON array of strings in %2.")
                                                    .arg(QDir::toNativeSeparators(filePath), key))
                              << std::endl;
                }
                validPackage = false;
                continue;
            }
            p.qtParts = parts.value();
        } else {
            if (logLevel != SilentLog) {
                std::cerr << qPrintable(tr("File %1: Unknown key %2.").arg(
                                            QDir::toNativeSeparators(filePath), key)) << std::endl;
            }
            validPackage = false;
        }
    }
 
    // Replace $<VERSION> and $<VERSION_DASHED> in string values
    {
        const QString versionVar = u"$<VERSION>"_s;
        const QString versionDashedVar = u"$<VERSION_DASHED>"_s;
        auto replaceInString = [&](QString &s) {
            if (s.contains(versionVar) || s.contains(versionDashedVar)) {
                if (p.version.isEmpty()) {
                    if (logLevel != SilentLog) {
                        std::cerr << qPrintable(
                                tr("File %1: $<VERSION> used but 'Version' is not set.")
                                        .arg(QDir::toNativeSeparators(filePath)))
                                  << std::endl;
                    }
                    validPackage = false;
                    return;
                }
                s.replace(versionVar, p.version);
                s.replace(versionDashedVar,
                          QString(p.version).replace(u'.', u'-'));
            }
        };
        auto replaceInList = [&](QStringList &list) {
            for (QString &s : list)
                replaceInString(s);
        };
        replaceInString(p.name);
        replaceInString(p.homepage);
        replaceInString(p.downloadLocation);
        replaceInString(p.description);
        replaceInString(p.qtUsage);
        replaceInString(p.packageComment);
        replaceInList(p.cpeList);
        replaceInList(p.purlList);
    }
 
    if (!p.copyrightFile.isEmpty()) {
        QFile file(p.copyrightFile);
        if (!file.open(QIODevice::ReadOnly)) {
            std::cerr << qPrintable(tr("File %1: Cannot open 'CopyrightFile' %2.\n")
                                            .arg(QDir::toNativeSeparators(filePath),
                                                 QDir::toNativeSeparators(p.copyrightFile)));
            validPackage = false;
        }
        p.copyrightFileContents = QString::fromUtf8(file.readAll());
    }
 
    if (p.licenseFiles.isEmpty() && !autoDetectLicenseFiles(p))
        return std::nullopt;
 
    for (const QString &licenseFile : std::as_const(p.licenseFiles)) {
        QFile file(licenseFile);
        if (!file.open(QIODevice::ReadOnly)) {
            if (logLevel != SilentLog) {
                std::cerr << qPrintable(tr("File %1: Cannot open 'LicenseFile' %2.\n")
                                                .arg(QDir::toNativeSeparators(filePath),
                                                     QDir::toNativeSeparators(licenseFile)));
            }
            validPackage = false;
        }
        p.licenseFilesContents << QString::fromUtf8(file.readAll()).trimmed();
    }
 
    if (!validPackage)
        return std::nullopt;
 
    return p;
}
 
// Parses a package's details from a README.chromium file
static Package parseChromiumFile(QFile &file, const QString &filePath, LogLevel logLevel)
{
    const QString directory = QFileInfo(filePath).absolutePath();
 
    // Parse the fields in the file
    QHash<QString, QString> fields;
 
    QTextStream in(&file);
    while (!in.atEnd()) {
        QString line = in.readLine().trimmed();
        QStringList parts = line.split(u":"_s);
 
        if (parts.size() < 2)
            continue;
 
        QString key = parts.at(0);
        parts.removeFirst();
        QString value = parts.join(QString()).trimmed();
 
        fields[key] = value;
 
        if (line == "Description:"_L1) { // special field : should handle multi-lines values
            while (!in.atEnd()) {
                QString line = in.readLine().trimmed();
 
                if (line.startsWith("Local Modifications:"_L1)) // Don't include this part
                    break;
 
                fields[key] += line + u"\n"_s;
            }
 
            break;
        }
    }
 
    // Construct the Package object
    Package p;
 
    QString shortName = fields.contains("Short Name"_L1)
            ? fields["Short Name"_L1]
            : fields["Name"_L1];
    QString version = fields[u"Version"_s];
 
    p.filePath = QFileInfo(filePath).absoluteFilePath();
    p.id =  u"chromium-"_s + shortName.toLower().replace(QChar::Space, u"-"_s);
    p.name = fields[u"Name"_s];
    if (version != QLatin1Char('0')) // "0" : not applicable
        p.version = version;
    p.license = fields[u"License"_s];
    p.homepage = fields[u"URL"_s];
    p.qdocModule = u"qtwebengine"_s;
    p.qtUsage = u"Used in Qt WebEngine"_s;
    p.description = fields[u"Description"_s].trimmed();
    p.path = directory;
 
    QString licenseFile = fields[u"License File"_s];
    if (licenseFile != QString() && licenseFile != "NOT_SHIPPED"_L1) {
        p.licenseFiles = QStringList(QDir(directory).absoluteFilePath(licenseFile));
    } else {
        // Look for a LICENSE or COPYING file as a fallback
        QDir dir = directory;
 
        dir.setNameFilters({ u"LICENSE"_s, u"COPYING"_s });
        dir.setFilter(QDir::Files | QDir::NoDotAndDotDot);
 
        const QFileInfoList entries = dir.entryInfoList();
        if (!entries.empty())
            p.licenseFiles = QStringList(entries.at(0).absoluteFilePath());
    }
 
    // let's ignore warnings regarding Chromium files for now
    Q_UNUSED(validatePackage(p, {}, logLevel));
 
    return p;
}
 
struct CursorPosition
{
    int line = -1;
    int column = -1;
};
 
static CursorPosition mapFromOffset(const QByteArray &content, int offset)
{
    CursorPosition pos{ 1, 1 };
    for (int i = 0; i < content.size(); ++i) {
        if (i == offset)
            return pos;
 
        if (content[i] == '\n') {
            pos.line++;
            pos.column = 1;
        } else {
            pos.column++;
        }
    }
    return CursorPosition();
}
 
std::optional<QList<Package>> readFile(const QString &filePath, LogLevel logLevel)
{
    QList<Package> packages;
    bool errorsFound = false;
 
    if (logLevel == VerboseLog) {
        std::cerr << qPrintable(tr("Reading file %1...").arg(
                                    QDir::toNativeSeparators(filePath))) << std::endl;
    }
    QFile file(filePath);
    if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
        if (logLevel != SilentLog)
            std::cerr << qPrintable(tr("Could not open file %1.").arg(
                                        QDir::toNativeSeparators(file.fileName()))) << std::endl;
        return std::nullopt;
    }
 
    if (filePath.endsWith(".json"_L1)) {
        QJsonParseError jsonParseError;
        const QByteArray content = file.readAll();
        const QJsonDocument document = QJsonDocument::fromJson(content, &jsonParseError);
        if (document.isNull()) {
            if (logLevel != SilentLog) {
                const CursorPosition pos = mapFromOffset(content, jsonParseError.offset);
                std::cerr << qPrintable(tr("Could not parse file %1: %2 at line %3, column %4")
                                                .arg(QDir::toNativeSeparators(file.fileName()),
                                                     jsonParseError.errorString(),
                                                     QString::number(pos.line),
                                                     QString::number(pos.column)))
                          << std::endl;
            }
            return std::nullopt;
        }
 
        if (document.isObject()) {
            std::optional<Package> p =
                    readPackage(document.object(), file.fileName(), logLevel);
            if (p) {
                packages << *p;
            } else {
                errorsFound = true;
            }
        } else if (document.isArray()) {
            QJsonArray array = document.array();
            for (int i = 0, size = array.size(); i < size; ++i) {
                QJsonValue value = array.at(i);
                if (value.isObject()) {
                    std::optional<Package> p =
                            readPackage(value.toObject(), file.fileName(), logLevel);
                    if (p) {
                        packages << *p;
                    } else {
                        errorsFound = true;
                    }
                } else {
                    if (logLevel != SilentLog) {
                        std::cerr << qPrintable(tr("File %1: Expecting JSON object in array.")
                                        .arg(QDir::toNativeSeparators(file.fileName())))
                                  << std::endl;
                    }
                    errorsFound = true;
                }
            }
        } else {
            if (logLevel != SilentLog) {
                std::cerr << qPrintable(tr("File %1: Expecting JSON object in array.").arg(
                                            QDir::toNativeSeparators(file.fileName()))) << std::endl;
            }
            errorsFound = true;
        }
    } else if (filePath.endsWith(".chromium"_L1)) {
        Package chromiumPackage = parseChromiumFile(file, filePath, logLevel);
        if (!chromiumPackage.name.isEmpty()) // Skip invalid README.chromium files
            packages << chromiumPackage;
    } else {
        if (logLevel != SilentLog) {
            std::cerr << qPrintable(tr("File %1: Unsupported file type.")
                            .arg(QDir::toNativeSeparators(file.fileName())))
                      << std::endl;
        }
        errorsFound = true;
    }
 
    if (errorsFound)
        return std::nullopt;
    return packages;
}
 
std::optional<QList<Package>> scanDirectory(const QString &directory, InputFormats inputFormats,
                                            LogLevel logLevel)
{
    QDir dir(directory);
    QList<Package> packages;
    bool errorsFound = false;
 
    QStringList nameFilters = QStringList();
    if (inputFormats & InputFormat::QtAttributions)
        nameFilters << u"qt_attribution.json"_s;
    if (inputFormats & InputFormat::ChromiumAttributions)
        nameFilters << u"README.chromium"_s;
    if (qEnvironmentVariableIsSet("QT_ATTRIBUTIONSSCANNER_TEST"))
        nameFilters << u"qt_attribution_test.json"_s << u"README_test.chromium"_s;
 
    dir.setNameFilters(nameFilters);
    dir.setFilter(QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Files);
 
    const QFileInfoList entries = dir.entryInfoList();
    for (const QFileInfo &info : entries) {
        if (info.isDir()) {
            std::optional<QList<Package>> ps =
                    scanDirectory(info.filePath(), inputFormats, logLevel);
            if (!ps)
                errorsFound = true;
            else
                packages += *ps;
        } else {
            std::optional p = readFile(info.filePath(), logLevel);
            if (!p)
                errorsFound = true;
            else
                packages += *p;
        }
    }
 
    if (errorsFound)
        return std::nullopt;
    return packages;
}
 
} // namespace Scanner