merge.cpp

Go to the documentation of this file.

// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
 
#include "trparser.h"
#include "simtexth.h"
#include "translator.h"
 
#include <QtCore/QCoreApplication>
#include <QtCore/QDebug>
#include <QtCore/QList>
#include <QtCore/QMap>
#include <QtCore/QStringList>
 
QT_BEGIN_NAMESPACE
 
using namespace Qt::Literals::StringLiterals;
 
/*
  Augments a Translator with trivially derived translations.
 
  For example, if "Enabled:" is consistendly translated as "Eingeschaltet:" no
  matter the context or the comment, "Eingeschaltet:" is added as the
  translation of any untranslated "Enabled:" text and is marked Unfinished.
 
  Returns the number of additional messages that this heuristic translated.
*/
 
int applySameTextHeuristic(Translator &tor)
{
    QMap<QString, QStringList> translated;
    QMap<QString, bool> avoid; // Want a QTreeSet, in fact
    QList<bool> untranslated(tor.messageCount());
    int inserted = 0;
 
    for (int i = 0; i < tor.messageCount(); ++i) {
        const TranslatorMessage &msg = tor.message(i);
        if (!msg.isTranslated()) {
            if (msg.type() == TranslatorMessage::Unfinished)
                untranslated[i] = true;
        } else {
            const QString &key = msg.sourceText();
            const auto t = translated.constFind(key);
            if (t != translated.constEnd()) {
                /*
                  The same source text is translated at least two
                  different ways. Do nothing then.
                */
                if (*t != msg.translations()) {
                    translated.remove(key);
                    avoid.insert(key, true);
                }
            } else if (!avoid.contains(key)) {
                translated.insert(key, msg.translations());
            }
        }
    }
 
    for (int i = 0; i < tor.messageCount(); ++i) {
        if (untranslated[i]) {
            TranslatorMessage &msg = tor.message(i);
            const auto t = translated.constFind(msg.sourceText());
            if (t != translated.constEnd()) {
                msg.setTranslations(*t);
                ++inserted;
            }
        }
    }
    return inserted;
}
 
 
 
/*
  Merges two Translator objects. The first one
  is a set of source texts and translations for a previous version of
  the internationalized program; the second one is a set of fresh
  source texts newly extracted from the source code, without any
  translation yet.
*/
 
Translator merge(
    const Translator &tor, const Translator &virginTor, const QList<Translator> &aliens,
    UpdateOptions options, QString &err)
{
    int known = 0;
    int neww = 0;
    int obsoleted = 0;
    int similarTextHeuristicCount = 0;
 
    Translator outTor;
    outTor.setLanguageCode(tor.languageCode());
    outTor.setSourceLanguageCode(tor.sourceLanguageCode());
    outTor.setLocationsType(tor.locationsType());
 
    /*
      The types of all the messages from the vernacular translator
      are updated according to the virgin translator.
    */
    for (TranslatorMessage m : tor.messages()) {
        TranslatorMessage::Type newType = TranslatorMessage::Finished;
 
 
        TranslatorMessage::ExtraData extras;
        const TranslatorMessage *mv;
        int mvi = virginTor.find(m);
        if (mvi < 0) {
            if (!(options & HeuristicSimilarText)) {
              makeObsolete:
                switch (m.type()) {
                case TranslatorMessage::Finished:
                    newType = TranslatorMessage::Vanished;
                    obsoleted++;
                    break;
                case TranslatorMessage::Unfinished:
                    newType = TranslatorMessage::Obsolete;
                    obsoleted++;
                    break;
                default:
                    newType = m.type();
                    break;
                }
                m.clearReferences();
            } else {
                mvi = virginTor.find(m.context(), m.comment(), m.allReferences());
                if (mvi < 0) {
                    // did not find it in the virgin, mark it as obsolete
                    goto makeObsolete;
                }
                mv = &virginTor.constMessage(mvi);
                // Do not just accept it if its on the same line number,
                // but different source text.
                // Also check if the texts are more or less similar before
                // we consider them to represent the same message...
                if (getSimilarityScore(m.sourceText(), mv->sourceText()) < textSimilarityThreshold) {
                    // The virgin and vernacular sourceTexts are so different that we could not find it
                    goto makeObsolete;
                }
                // It is just slightly modified, assume that it is the same string
 
                extras = mv->extras();
 
                // Mark it as unfinished. (Since the source text
                // was changed it might require re-translating...)
                newType = TranslatorMessage::Unfinished;
                ++similarTextHeuristicCount;
                neww++;
                goto outdateSource;
            }
        } else {
            mv = &virginTor.message(mvi);
            extras = mv->extras();
            if (!mv->id().isEmpty()
                && (mv->context() != m.context()
                    || mv->sourceText() != m.sourceText()
                    || mv->comment() != m.comment())) {
                known++;
                newType = TranslatorMessage::Unfinished;
                m.setContext(mv->context());
                m.setComment(mv->comment());
                if (mv->sourceText() != m.sourceText()) {
                  outdateSource:
                    m.setOldSourceText(m.sourceText());
                    m.setSourceText(mv->sourceText());
                    const QString &oldpluralsource = m.extra("po-msgid_plural"_L1);
                    if (!oldpluralsource.isEmpty())
                        extras.insert("po-old_msgid_plural"_L1, oldpluralsource);
                }
            } else {
                switch (m.type()) {
                case TranslatorMessage::Finished:
                default:
                    if (m.isPlural() == mv->isPlural()) {
                        newType = TranslatorMessage::Finished;
                    } else {
                        newType = TranslatorMessage::Unfinished;
                    }
                    known++;
                    break;
                case TranslatorMessage::Unfinished:
                    newType = TranslatorMessage::Unfinished;
                    known++;
                    break;
                case TranslatorMessage::Vanished:
                    newType = TranslatorMessage::Finished;
                    neww++;
                    break;
                case TranslatorMessage::Obsolete:
                    newType = TranslatorMessage::Unfinished;
                    neww++;
                    break;
                }
            }
 
            // Always get the filename and linenumber info from the
            // virgin Translator, in case it has changed location.
            // This should also enable us to read a file that does not
            // have the <location> element.
            // why not use operator=()? Because it overwrites e.g. userData.
            m.setReferences(mv->allReferences());
            m.setPlural(mv->isPlural());
            m.setExtras(extras);
            m.setExtraComment(mv->extraComment());
            m.setId(mv->id());
            m.setLabel(mv->label());
        }
 
 
        m.setType(newType);
        outTor.append(m);
    }
 
    /*
      Messages found only in the virgin translator are added to the
      vernacular translator.
    */
    for (const TranslatorMessage &mv : virginTor.messages()) {
 
        if (tor.find(mv) >= 0)
            continue;
        if (options & HeuristicSimilarText) {
            int mi = tor.find(mv.context(), mv.comment(), mv.allReferences());
            if (mi >= 0) {
                // The similar message found in tor (ts file) must NOT correspond exactly
                // to an other message is virginTor
                if (virginTor.find(tor.constMessage(mi)) < 0) {
                    if (getSimilarityScore(tor.constMessage(mi).sourceText(), mv.sourceText())
                            >= textSimilarityThreshold)
                        continue;
                }
            }
        }
 
        outTor.appendSorted(mv);
        ++neww;
    }
 
    /*
      "Alien" translators can be used to augment the vernacular translator.
    */
    for (const Translator &alf : aliens) {
        for (TranslatorMessage mv : alf.messages()) {
            if (mv.sourceText().isEmpty() || !mv.isTranslated())
                continue;
            int mvi = outTor.find(mv);
            if (mvi >= 0) {
                TranslatorMessage &tm = outTor.message(mvi);
                if (tm.type() != TranslatorMessage::Finished && !tm.isTranslated()) {
                    tm.setTranslations(mv.translations());
                    --neww;
                    ++known;
                }
            } else {
                /*
                 * Don't do simtex search, as the locations are likely to be
                 * completely off anyway, so we'd find nothing.
                 */
                /*
                 * Add the unmatched messages as obsoletes, so the Linguist GUI
                 * will offer them as possible translations.
                 */
                mv.clearReferences();
                mv.setType(mv.type() == TranslatorMessage::Finished
                           ? TranslatorMessage::Vanished : TranslatorMessage::Obsolete);
                outTor.appendSorted(mv);
                ++known;
                ++obsoleted;
            }
        }
    }
 
    /*
      The same-text heuristic handles cases where a message has an
      obsolete counterpart with a different context or comment.
    */
    int sameTextHeuristicCount = (options & HeuristicSameText) ? applySameTextHeuristic(outTor) : 0;
 
    if (options & Verbose) {
        int totalFound = neww + known;
        err += QStringLiteral("    Found %1 source text(s) (%2 new and %3 already existing)\n")
            .arg(totalFound).arg(neww).arg(known);
 
        if (obsoleted) {
            if (options & NoObsolete) {
                err += QStringLiteral("    Removed %1 obsolete entries\n").arg(obsoleted);
            } else {
                err += QStringLiteral("    Kept %1 obsolete entries\n").arg(obsoleted);
            }
        }
 
        if (sameTextHeuristicCount)
            err += QStringLiteral("    Same-text heuristic provided %1 translation(s)\n")
                .arg(sameTextHeuristicCount);
        if (similarTextHeuristicCount)
            err += QStringLiteral("    Similar-text heuristic provided %1 translation(s)\n")
                .arg(similarTextHeuristicCount);
    }
    return outTor;
}
 
QT_END_NAMESPACE