simtexth.cpp

Go to the documentation of this file.

// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
 
#include "simtexth.h"
#include "translator.h"
 
#include <QtCore/QByteArray>
#include <QtCore/QString>
#include <QtCore/QList>
 
 
QT_BEGIN_NAMESPACE
 
typedef QList<TranslatorMessage> TML;
 
/*
  How similar are two texts?  The approach used here relies on co-occurrence
  matrices and is very efficient.
 
  Let's see with an example: how similar are "here" and "hither"?  The
  co-occurrence matrix M for "here" is M[h,e] = 1, M[e,r] = 1, M[r,e] = 1, and 0
  elsewhere; the matrix N for "hither" is N[h,i] = 1, N[i,t] = 1, ...,
  N[h,e] = 1, N[e,r] = 1, and 0 elsewhere.  The union U of both matrices is the
  matrix U[i,j] = max { M[i,j], N[i,j] }, and the intersection V is
  V[i,j] = min { M[i,j], N[i,j] }.  The score for a pair of texts is
 
      score = (sum of V[i,j] over all i, j) / (sum of U[i,j] over all i, j),
 
  a formula suggested by Arnt Gulbrandsen.  Here we have
 
      score = 2 / 6,
 
  or one third.
 
  The implementation differs from this in a few details.  Most importantly,
  repetitions are ignored; for input "xxx", M[x,x] equals 1, not 2.
*/
 
/*
  Every character is assigned to one of 20 buckets so that the co-occurrence
  matrix requires only 20 * 20 = 400 bits, not 256 * 256 = 65536 bits or even
  more if we want the whole Unicode.  Which character falls in which bucket is
  arbitrary.
 
  The second half of the table is a replica of the first half, because of
  laziness.
*/
static const int indexOf[256] = {
    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
//      !   "   #   $   %   &   '   (   )   *   +   ,   -   .   /
    0,  2,  6,  7,  10, 12, 15, 19, 2,  6,  7,  10, 12, 15, 19, 0,
//  0   1   2   3   4   5   6   7   8   9   :   ;   <   =   >   ?
    1,  3,  4,  5,  8,  9,  11, 13, 14, 16, 2,  6,  7,  10, 12, 15,
//  @   A   B   C   D   E   F   G   H   I   J   K   L   M   N   O
    0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  6,  10, 11, 12, 13, 14,
//  P   Q   R   S   T   U   V   W   X   Y   Z   [   \   ]   ^   _
    15, 12, 16, 17, 18, 19, 2,  10, 15, 7,  19, 2,  6,  7,  10, 0,
//  `   a   b   c   d   e   f   g   h   i   j   k   l   m   n   o
    0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  6,  10, 11, 12, 13, 14,
//  p   q   r   s   t   u   v   w   x   y   z   {   |   }   ~
    15, 12, 16, 17, 18, 19, 2,  10, 15, 7,  19, 2,  6,  7,  10, 0,
 
    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
    0,  2,  6,  7,  10, 12, 15, 19, 2,  6,  7,  10, 12, 15, 19, 0,
    1,  3,  4,  5,  8,  9,  11, 13, 14, 16, 2,  6,  7,  10, 12, 15,
    0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  6,  10, 11, 12, 13, 14,
    15, 12, 16, 17, 18, 19, 2,  10, 15, 7,  19, 2,  6,  7,  10, 0,
    0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  6,  10, 11, 12, 13, 14,
    15, 12, 16, 17, 18, 19, 2,  10, 15, 7,  19, 2,  6,  7,  10, 0
};
 
/*
  The entry bitCount[i] (for i between 0 and 255) is the number of bits used to
  represent i in binary.
*/
static const int bitCount[256] = {
    0,  1,  1,  2,  1,  2,  2,  3,  1,  2,  2,  3,  2,  3,  3,  4,
    1,  2,  2,  3,  2,  3,  3,  4,  2,  3,  3,  4,  3,  4,  4,  5,
    1,  2,  2,  3,  2,  3,  3,  4,  2,  3,  3,  4,  3,  4,  4,  5,
    2,  3,  3,  4,  3,  4,  4,  5,  3,  4,  4,  5,  4,  5,  5,  6,
    1,  2,  2,  3,  2,  3,  3,  4,  2,  3,  3,  4,  3,  4,  4,  5,
    2,  3,  3,  4,  3,  4,  4,  5,  3,  4,  4,  5,  4,  5,  5,  6,
    2,  3,  3,  4,  3,  4,  4,  5,  3,  4,  4,  5,  4,  5,  5,  6,
    3,  4,  4,  5,  4,  5,  5,  6,  4,  5,  5,  6,  5,  6,  6,  7,
    1,  2,  2,  3,  2,  3,  3,  4,  2,  3,  3,  4,  3,  4,  4,  5,
    2,  3,  3,  4,  3,  4,  4,  5,  3,  4,  4,  5,  4,  5,  5,  6,
    2,  3,  3,  4,  3,  4,  4,  5,  3,  4,  4,  5,  4,  5,  5,  6,
    3,  4,  4,  5,  4,  5,  5,  6,  4,  5,  5,  6,  5,  6,  6,  7,
    2,  3,  3,  4,  3,  4,  4,  5,  3,  4,  4,  5,  4,  5,  5,  6,
    3,  4,  4,  5,  4,  5,  5,  6,  4,  5,  5,  6,  5,  6,  6,  7,
    3,  4,  4,  5,  4,  5,  5,  6,  4,  5,  5,  6,  5,  6,  6,  7,
    4,  5,  5,  6,  5,  6,  6,  7,  5,  6,  6,  7,  6,  7,  7,  8
};
 
static inline void setCoOccurence(CoMatrix &m, char c, char d)
{
    int k = indexOf[(uchar) c] + 20 * indexOf[(uchar) d];
    m.b[k >> 3] |= (1 << (k & 0x7));
}
 
CoMatrix::CoMatrix(const QString &str)
{
    QByteArray ba = str.toUtf8();
    const char *text = ba.constData();
    char c = '\0', d;
    memset( b, 0, 52 );
    /*
      The Knuth books are not in the office only for show; they help make
      loops 30% faster and 20% as readable.
    */
    while ( (d = *text) != '\0' ) {
        setCoOccurence(*this, c, d);
        if ( (c = *++text) != '\0' ) {
            setCoOccurence(*this, d, c);
            text++;
        }
    }
}
 
static inline int worth(const CoMatrix &m)
{
    int w = 0;
    for (int i = 0; i < 50; i++)
        w += bitCount[m.b[i]];
    return w;
}
 
static inline CoMatrix reunion(const CoMatrix &m, const CoMatrix &n)
{
    CoMatrix p;
    for (int i = 0; i < 13; ++i)
        p.w[i] = m.w[i] | n.w[i];
    return p;
}
 
static inline CoMatrix intersection(const CoMatrix &m, const CoMatrix &n)
{
    CoMatrix p;
    for (int i = 0; i < 13; ++i)
        p.w[i] = m.w[i] & n.w[i];
    return p;
}
 
StringSimilarityMatcher::StringSimilarityMatcher(const QString &stringToMatch)
    : m_cm(stringToMatch)
{
    m_length = stringToMatch.size();
}
 
int StringSimilarityMatcher::getSimilarityScore(const QString &strCandidate)
{
    CoMatrix cmTarget(strCandidate);
    int delta = qAbs(m_length - strCandidate.size());
    int score = ( (worth(intersection(m_cm, cmTarget)) + 1) << 10 ) /
        ( worth(reunion(m_cm, cmTarget)) + (delta << 1) + 1 );
    return score;
}
 
CandidateList similarTextHeuristicCandidates(const Translator *tor,
    const QString &text, int maxCandidates)
{
    QList<int> scores;
    CandidateList candidates;
    StringSimilarityMatcher matcher(text);
 
    for (const TranslatorMessage &mtm : tor->messages()) {
        if (mtm.type() == TranslatorMessage::Unfinished
            || mtm.translation().isEmpty())
            continue;
 
        QString s = mtm.sourceText();
        int score = matcher.getSimilarityScore(s);
 
        if (candidates.size() == maxCandidates && score > scores[maxCandidates - 1] )
            candidates.removeLast();
 
        if (candidates.size() < maxCandidates && score >= textSimilarityThreshold) {
            Candidate cand(mtm.context(), s, mtm.comment(), mtm.translation());
 
            int i;
            for (i = 0; i < candidates.size(); i++) {
                if (score >= scores.at(i)) {
                    if (score == scores.at(i)) {
                        if (candidates.at(i) == cand)
                            goto continue_outer_loop;
                    } else {
                        break;
                    }
                }
            }
            scores.insert(i, score);
            candidates.insert(i, cand);
        }
        continue_outer_loop:
        ;
    }
    return candidates;
}
 
QT_END_NAMESPACE