6#include <qhelp_global.h>
8#include <QtCore/QtMath>
10#include <QtCore/QFile>
11#include <QtCore/QFileInfo>
13#include <QtCore/QDebug>
14#include <QtCore/QRegularExpression>
17#include <QtCore/QVariant>
18#include <QtCore/QDateTime>
19#include <QtCore/QStringConverter>
20#include <QtCore/QDataStream>
21#include <QtSql/QSqlQuery>
27class HelpGeneratorPrivate :
public QObject
32 HelpGeneratorPrivate(QObject *parent =
nullptr) : QObject(parent) {}
34 bool generate(QHelpProjectData *helpData,
35 const QString &outputFileName);
36 bool checkLinks(
const QHelpProjectData &helpData);
37 QString error()
const;
40 void statusChanged(
const QString &msg);
41 void progressChanged(
double progress);
42 void warning(
const QString &msg);
45 struct FileNameTableData
52 void writeTree(QDataStream &s, QHelpDataContentItem *item,
int depth);
54 bool insertFileNotFoundFile();
55 bool registerCustomFilter(
const QString &filterName,
56 const QStringList &filterAttribs,
bool forceUpdate =
false);
57 bool registerVirtualFolder(
const QString &folderName,
const QString &ns);
58 bool insertFilterAttributes(
const QStringList &attributes);
59 bool insertKeywords(
const QList<QHelpDataIndexItem> &keywords,
60 const QStringList &filterAttributes);
61 bool insertFiles(
const QStringList &files,
const QString &rootPath,
62 const QStringList &filterAttributes);
63 bool insertContents(
const QByteArray &ba,
64 const QStringList &filterAttributes);
65 bool insertMetaData(
const QMap<QString, QVariant> &metaData);
67 void setupProgress(QHelpProjectData *helpData);
68 void addProgress(
double step);
71 QSqlQuery *m_query =
nullptr;
73 int m_namespaceId = -1;
74 int m_virtualFolderId = -1;
76 QMap<QString,
int> m_fileMap;
77 QMap<
int, QSet<
int> > m_fileFilterMap;
87
88
89
90
92 const QString &outputFileName)
94 emit progressChanged(0);
96 if (!helpData || helpData->namespaceName().isEmpty()) {
97 m_error = tr(
"Invalid help data.");
101 QString outFileName = outputFileName;
102 if (outFileName.isEmpty()) {
103 m_error = tr(
"No output file name specified.");
107 QFileInfo fi(outFileName);
109 if (!fi.dir().remove(fi.fileName())) {
110 m_error = tr(
"The file %1 cannot be overwritten.").arg(outFileName);
115 setupProgress(helpData);
117 emit statusChanged(tr(
"Building up file structure..."));
118 bool openingOk =
true;
120 QSqlDatabase db = QSqlDatabase::addDatabase(QLatin1String(
"QSQLITE"), QLatin1String(
"builder"));
121 db.setDatabaseName(outFileName);
122 openingOk = db.open();
124 m_query =
new QSqlQuery(db);
128 m_error = tr(
"Cannot open data base file %1.").arg(outFileName);
133 m_query->exec(QLatin1String(
"PRAGMA synchronous=OFF"));
134 m_query->exec(QLatin1String(
"PRAGMA cache_size=3000"));
138 insertFileNotFoundFile();
139 insertMetaData(helpData->metaData());
141 if (!registerVirtualFolder(helpData->virtualFolder(), helpData->namespaceName())) {
142 m_error = tr(
"Cannot register namespace \"%1\".").arg(helpData->namespaceName());
148 emit statusChanged(tr(
"Insert custom filters..."));
149 for (
const QHelpDataCustomFilter &f : helpData->customFilters()) {
150 if (!registerCustomFilter(f.name, f.filterAttributes,
true)) {
158 for (
const QHelpDataFilterSection &fs : helpData->filterSections()) {
159 emit statusChanged(tr(
"Insert help data for filter section (%1 of %2)...")
160 .arg(i++).arg(helpData->filterSections().size()));
161 insertFilterAttributes(fs.filterAttributes());
163 QDataStream s(&ba, QIODevice::WriteOnly);
164 for (QHelpDataContentItem *itm : fs.contents())
165 writeTree(s, itm, 0);
166 if (!insertFiles(fs.files(), helpData->rootPath(), fs.filterAttributes())
167 || !insertContents(ba, fs.filterAttributes())
168 || !insertKeywords(fs.indices(), fs.filterAttributes())) {
175 emit progressChanged(100);
176 emit statusChanged(tr(
"Documentation successfully generated."));
185 int numberOfFiles = 0;
186 int numberOfIndices = 0;
187 for (
const QHelpDataFilterSection &fs : helpData->filterSections()) {
188 numberOfFiles += fs.files().size();
189 numberOfIndices += fs.indices().size();
196 m_contentStep = 10.0 / qMax(helpData->customFilters().size(), 1);
197 m_fileStep = 60.0 / qMax(numberOfFiles, 1);
198 m_indexStep = 27.0 / qMax(numberOfIndices, 1);
201void HelpGeneratorPrivate::addProgress(
double step)
204 if ((m_progress - m_oldProgress) >= 1.0 && m_progress <= 100.0) {
205 m_oldProgress = m_progress;
206 emit progressChanged(qCeil(m_progress));
210void HelpGeneratorPrivate::cleanupDB()
217 QSqlDatabase::removeDatabase(QLatin1String(
"builder"));
220void HelpGeneratorPrivate::writeTree(QDataStream &s, QHelpDataContentItem *item,
int depth)
223 s << item->reference();
225 for (QHelpDataContentItem *i : item->children())
226 writeTree(s, i, depth + 1);
230
231
232QString HelpGeneratorPrivate::error()
const
237bool HelpGeneratorPrivate::createTables()
242 m_query->exec(QLatin1String(
"SELECT COUNT(*) FROM sqlite_master WHERE TYPE=\'table\'"
243 "AND Name=\'NamespaceTable\'"));
245 if (m_query->value(0).toInt() > 0) {
246 m_error = tr(
"Some tables already exist.");
250 const QStringList tables = QStringList()
251 << QLatin1String(
"CREATE TABLE NamespaceTable ("
252 "Id INTEGER PRIMARY KEY,"
254 << QLatin1String(
"CREATE TABLE FilterAttributeTable ("
255 "Id INTEGER PRIMARY KEY, "
257 << QLatin1String(
"CREATE TABLE FilterNameTable ("
258 "Id INTEGER PRIMARY KEY, "
260 << QLatin1String(
"CREATE TABLE FilterTable ("
262 "FilterAttributeId INTEGER )")
263 << QLatin1String(
"CREATE TABLE IndexTable ("
264 "Id INTEGER PRIMARY KEY, "
267 "NamespaceId INTEGER, "
270 << QLatin1String(
"CREATE TABLE IndexFilterTable ("
271 "FilterAttributeId INTEGER, "
273 << QLatin1String(
"CREATE TABLE ContentsTable ("
274 "Id INTEGER PRIMARY KEY, "
275 "NamespaceId INTEGER, "
277 << QLatin1String(
"CREATE TABLE ContentsFilterTable ("
278 "FilterAttributeId INTEGER, "
279 "ContentsId INTEGER )")
280 << QLatin1String(
"CREATE TABLE FileAttributeSetTable ("
282 "FilterAttributeId INTEGER )")
283 << QLatin1String(
"CREATE TABLE FileDataTable ("
284 "Id INTEGER PRIMARY KEY, "
286 << QLatin1String(
"CREATE TABLE FileFilterTable ("
287 "FilterAttributeId INTEGER, "
289 << QLatin1String(
"CREATE TABLE FileNameTable ("
294 << QLatin1String(
"CREATE TABLE FolderTable("
295 "Id INTEGER PRIMARY KEY, "
297 "NamespaceID INTEGER )")
298 << QLatin1String(
"CREATE TABLE MetaDataTable("
302 for (
const QString &q : tables) {
303 if (!m_query->exec(q)) {
304 m_error = tr(
"Cannot create tables.");
309 m_query->exec(QLatin1String(
"INSERT INTO MetaDataTable VALUES('qchVersion', '1.0')"));
314bool HelpGeneratorPrivate::insertFileNotFoundFile()
319 m_query->exec(QLatin1String(
"SELECT id FROM FileNameTable WHERE Name=\'\'"));
320 if (m_query->next() && m_query->isValid())
323 m_query->prepare(QLatin1String(
"INSERT INTO FileDataTable VALUES (Null, ?)"));
324 m_query->bindValue(0, QByteArray());
325 if (!m_query->exec())
328 const int fileId = m_query->lastInsertId().toInt();
329 m_query->prepare(QLatin1String(
"INSERT INTO FileNameTable (FolderId, Name, FileId, Title) "
330 " VALUES (0, '', ?, '')"));
331 m_query->bindValue(0, fileId);
332 if (fileId > -1 && m_query->exec()) {
333 m_fileMap.insert({}, fileId);
339bool HelpGeneratorPrivate::registerVirtualFolder(
const QString &folderName,
const QString &ns)
341 if (!m_query || folderName.isEmpty() || ns.isEmpty())
344 m_query->prepare(QLatin1String(
"SELECT Id FROM FolderTable WHERE Name=?"));
345 m_query->bindValue(0, folderName);
348 if (m_query->isValid() && m_query->value(0).toInt() > 0)
352 m_query->prepare(QLatin1String(
"SELECT Id FROM NamespaceTable WHERE Name=?"));
353 m_query->bindValue(0, ns);
355 while (m_query->next()) {
356 m_namespaceId = m_query->value(0).toInt();
360 if (m_namespaceId < 0) {
361 m_query->prepare(QLatin1String(
"INSERT INTO NamespaceTable VALUES(NULL, ?)"));
362 m_query->bindValue(0, ns);
364 m_namespaceId = m_query->lastInsertId().toInt();
367 if (m_namespaceId > 0) {
368 m_query->prepare(QLatin1String(
"SELECT Id FROM FolderTable WHERE Name=?"));
369 m_query->bindValue(0, folderName);
371 while (m_query->next())
372 m_virtualFolderId = m_query->value(0).toInt();
374 if (m_virtualFolderId > 0)
377 m_query->prepare(QLatin1String(
"INSERT INTO FolderTable (NamespaceId, Name) "
379 m_query->bindValue(0, m_namespaceId);
380 m_query->bindValue(1, folderName);
381 if (m_query->exec()) {
382 m_virtualFolderId = m_query->lastInsertId().toInt();
383 return m_virtualFolderId > 0;
386 m_error = tr(
"Cannot register virtual folder.");
390bool HelpGeneratorPrivate::insertFiles(
const QStringList &files,
const QString &rootPath,
391 const QStringList &filterAttributes)
396 emit statusChanged(tr(
"Insert files..."));
397 QSet<
int> filterAtts;
398 for (
const QString &filterAtt : filterAttributes) {
399 m_query->prepare(QLatin1String(
"SELECT Id FROM FilterAttributeTable "
401 m_query->bindValue(0, filterAtt);
404 filterAtts.insert(m_query->value(0).toInt());
407 int filterSetId = -1;
408 m_query->exec(QLatin1String(
"SELECT MAX(Id) FROM FileAttributeSetTable"));
410 filterSetId = m_query->value(0).toInt();
414 QList<
int> attValues = filterAtts.values();
415 std::sort(attValues.begin(), attValues.end());
416 for (
int attId : std::as_const(attValues)) {
417 m_query->prepare(QLatin1String(
"INSERT INTO FileAttributeSetTable "
419 m_query->bindValue(0, filterSetId);
420 m_query->bindValue(1, attId);
425 m_query->exec(QLatin1String(
"SELECT MAX(Id) FROM FileDataTable"));
427 tableFileId = m_query->value(0).toInt() + 1;
431 QList<QByteArray> fileDataList;
432 QMap<
int, QSet<
int> > tmpFileFilterMap;
433 QList<FileNameTableData> fileNameDataList;
436 for (
const QString &file : files) {
437 const QString fileName = QDir::cleanPath(file);
439 QFile fi(rootPath + QDir::separator() + fileName);
441 emit warning(tr(
"The file %1 does not exist, skipping it...")
442 .arg(QDir::cleanPath(rootPath + QDir::separator() + fileName)));
446 if (!fi.open(QIODevice::ReadOnly)) {
447 emit warning(tr(
"Cannot open file %1, skipping it...")
448 .arg(QDir::cleanPath(rootPath + QDir::separator() + fileName)));
452 QByteArray data = fi.readAll();
453 if (fileName.endsWith(QLatin1String(
".html"))
454 || fileName.endsWith(QLatin1String(
".htm"))) {
455 auto encoding = QStringDecoder::encodingForHtml(data);
457 encoding = QStringDecoder::Utf8;
458 title = QHelpGlobal::documentTitle(QStringDecoder(*encoding)(data));
460 title = fileName.mid(fileName.lastIndexOf(QLatin1Char(
'/')) + 1);
464 const auto &it = m_fileMap.constFind(fileName);
465 if (it == m_fileMap.cend()) {
466 fileDataList.append(qCompress(data));
468 FileNameTableData fileNameData;
469 fileNameData.name = fileName;
470 fileNameData.fileId = tableFileId;
471 fileNameData.title = title;
472 fileNameDataList.append(fileNameData);
474 m_fileMap.insert(fileName, tableFileId);
475 m_fileFilterMap.insert(tableFileId, filterAtts);
476 tmpFileFilterMap.insert(tableFileId, filterAtts);
481 QSet<
int> &fileFilterSet = m_fileFilterMap[fileId];
482 QSet<
int> &tmpFileFilterSet = tmpFileFilterMap[fileId];
483 for (
int filter : std::as_const(filterAtts)) {
484 if (!fileFilterSet.contains(filter)
485 && !tmpFileFilterSet.contains(filter)) {
486 fileFilterSet.insert(filter);
487 tmpFileFilterSet.insert(filter);
493 if (!tmpFileFilterMap.isEmpty()) {
494 m_query->exec(QLatin1String(
"BEGIN"));
495 for (
auto it = tmpFileFilterMap.cbegin(), end = tmpFileFilterMap.cend(); it != end; ++it) {
496 QList<
int> filterValues = it.value().values();
497 std::sort(filterValues.begin(), filterValues.end());
498 for (
int fv : std::as_const(filterValues)) {
499 m_query->prepare(QLatin1String(
"INSERT INTO FileFilterTable "
501 m_query->bindValue(0, fv);
502 m_query->bindValue(1, it.key());
507 for (
const QByteArray &fileData : std::as_const(fileDataList)) {
508 m_query->prepare(QLatin1String(
"INSERT INTO FileDataTable VALUES "
510 m_query->bindValue(0, fileData);
513 addProgress(m_fileStep * 20.0);
516 for (
const FileNameTableData &fnd : std::as_const(fileNameDataList)) {
517 m_query->prepare(QLatin1String(
"INSERT INTO FileNameTable "
518 "(FolderId, Name, FileId, Title) VALUES (?, ?, ?, ?)"));
519 m_query->bindValue(0, 1);
520 m_query->bindValue(1, fnd.name);
521 m_query->bindValue(2, fnd.fileId);
522 m_query->bindValue(3, fnd.title);
525 m_query->exec(QLatin1String(
"COMMIT"));
528 m_query->exec(QLatin1String(
"SELECT MAX(Id) FROM FileDataTable"));
530 && m_query->value(0).toInt() == tableFileId - 1) {
531 addProgress(m_fileStep*(i % 20));
537bool HelpGeneratorPrivate::registerCustomFilter(
const QString &filterName,
538 const QStringList &filterAttribs,
bool forceUpdate)
543 m_query->exec(QLatin1String(
"SELECT Id, Name FROM FilterAttributeTable"));
544 QStringList idsToInsert = filterAttribs;
545 QMap<QString,
int> attributeMap;
546 while (m_query->next()) {
547 attributeMap.insert(m_query->value(1).toString(),
548 m_query->value(0).toInt());
549 idsToInsert.removeAll(m_query->value(1).toString());
552 for (
const QString &id : std::as_const(idsToInsert)) {
553 m_query->prepare(QLatin1String(
"INSERT INTO FilterAttributeTable VALUES(NULL, ?)"));
554 m_query->bindValue(0, id);
556 attributeMap.insert(id, m_query->lastInsertId().toInt());
560 m_query->prepare(QLatin1String(
"SELECT Id FROM FilterNameTable WHERE Name=?"));
561 m_query->bindValue(0, filterName);
563 while (m_query->next()) {
564 nameId = m_query->value(0).toInt();
569 m_query->prepare(QLatin1String(
"INSERT INTO FilterNameTable VALUES(NULL, ?)"));
570 m_query->bindValue(0, filterName);
572 nameId = m_query->lastInsertId().toInt();
573 }
else if (!forceUpdate) {
574 m_error = tr(
"The filter %1 is already registered.").arg(filterName);
579 m_error = tr(
"Cannot register filter %1.").arg(filterName);
583 m_query->prepare(QLatin1String(
"DELETE FROM FilterTable WHERE NameId=?"));
584 m_query->bindValue(0, nameId);
587 for (
const QString &att : filterAttribs) {
588 m_query->prepare(QLatin1String(
"INSERT INTO FilterTable VALUES(?, ?)"));
589 m_query->bindValue(0, nameId);
590 m_query->bindValue(1, attributeMap[att]);
591 if (!m_query->exec())
597bool HelpGeneratorPrivate::insertKeywords(
const QList<QHelpDataIndexItem> &keywords,
598 const QStringList &filterAttributes)
603 emit statusChanged(tr(
"Insert indices..."));
605 m_query->exec(QLatin1String(
"SELECT MAX(Id) FROM IndexTable"));
607 indexId = m_query->value(0).toInt() + 1;
609 QList<
int> filterAtts;
610 for (
const QString &filterAtt : filterAttributes) {
611 m_query->prepare(QLatin1String(
"SELECT Id FROM FilterAttributeTable WHERE Name=?"));
612 m_query->bindValue(0, filterAtt);
615 filterAtts.append(m_query->value(0).toInt());
618 QList<
int> indexFilterTable;
621 m_query->exec(QLatin1String(
"BEGIN"));
622 QSet<QString> indices;
623 for (
const QHelpDataIndexItem &itm : keywords) {
626 if (indices.contains(itm.identifier))
631 if (!itm.identifier.isEmpty())
632 indices.insert(itm.identifier);
634 const int pos = itm.reference.indexOf(QLatin1Char(
'#'));
635 const QString &fileName = itm.reference.left(pos);
636 const QString anchor = pos < 0 ? QString() : itm.reference.mid(pos + 1);
638 const QString &fName = QDir::cleanPath(fileName);
640 const auto &it = m_fileMap.constFind(fName);
641 const int fileId = it == m_fileMap.cend() ? 1 : it.value();
643 m_query->prepare(QLatin1String(
"INSERT INTO IndexTable (Name, Identifier, NamespaceId, FileId, Anchor) "
644 "VALUES(?, ?, ?, ?, ?)"));
645 m_query->bindValue(0, itm.name);
646 m_query->bindValue(1, itm.identifier);
647 m_query->bindValue(2, m_namespaceId);
648 m_query->bindValue(3, fileId);
649 m_query->bindValue(4, anchor);
652 indexFilterTable.append(indexId++);
654 addProgress(m_indexStep * 100.0);
656 m_query->exec(QLatin1String(
"COMMIT"));
658 m_query->exec(QLatin1String(
"BEGIN"));
659 for (
int idx : std::as_const(indexFilterTable)) {
660 for (
int a : std::as_const(filterAtts)) {
661 m_query->prepare(QLatin1String(
"INSERT INTO IndexFilterTable (FilterAttributeId, IndexId) "
663 m_query->bindValue(0, a);
664 m_query->bindValue(1, idx);
668 m_query->exec(QLatin1String(
"COMMIT"));
670 m_query->exec(QLatin1String(
"SELECT COUNT(Id) FROM IndexTable"));
671 if (m_query->next() && m_query->value(0).toInt() >= indices.size())
676bool HelpGeneratorPrivate::insertContents(
const QByteArray &ba,
677 const QStringList &filterAttributes)
682 emit statusChanged(tr(
"Insert contents..."));
683 m_query->prepare(QLatin1String(
"INSERT INTO ContentsTable (NamespaceId, Data) "
685 m_query->bindValue(0, m_namespaceId);
686 m_query->bindValue(1, ba);
688 int contentId = m_query->lastInsertId().toInt();
690 m_error = tr(
"Cannot insert contents.");
695 for (
const QString &filterAtt : filterAttributes) {
696 m_query->prepare(QLatin1String(
"INSERT INTO ContentsFilterTable (FilterAttributeId, ContentsId) "
697 "SELECT Id, ? FROM FilterAttributeTable WHERE Name=?"));
698 m_query->bindValue(0, contentId);
699 m_query->bindValue(1, filterAtt);
701 if (!m_query->isActive()) {
702 m_error = tr(
"Cannot register contents.");
706 addProgress(m_contentStep);
710bool HelpGeneratorPrivate::insertFilterAttributes(
const QStringList &attributes)
715 m_query->exec(QLatin1String(
"SELECT Name FROM FilterAttributeTable"));
717 while (m_query->next())
718 atts.insert(m_query->value(0).toString());
720 for (
const QString &s : attributes) {
721 if (!atts.contains(s)) {
722 m_query->prepare(QLatin1String(
"INSERT INTO FilterAttributeTable VALUES(NULL, ?)"));
723 m_query->bindValue(0, s);
730bool HelpGeneratorPrivate::insertMetaData(
const QMap<QString, QVariant> &metaData)
735 for (
auto it = metaData.cbegin(), end = metaData.cend(); it != end; ++it) {
736 m_query->prepare(QLatin1String(
"INSERT INTO MetaDataTable VALUES(?, ?)"));
737 m_query->bindValue(0, it.key());
738 m_query->bindValue(1, it.value());
747
748
749
751 for (
const QHelpDataFilterSection &filterSection : helpData.filterSections()) {
752 for (
const QString &file : filterSection.files()) {
753 const QFileInfo fileInfo(helpData.rootPath() + QDir::separator() + file);
754 const QString &canonicalFileName = fileInfo.canonicalFilePath();
755 if (!fileInfo.exists())
756 emit warning(tr(
"File \"%1\" does not exist.").arg(file));
758 files.insert(canonicalFileName);
763
764
765
766
767
768 bool allLinksOk =
true;
769 for (
const QString &fileName : std::as_const(files)) {
770 if (!fileName.endsWith(QLatin1String(
"html"))
771 && !fileName.endsWith(QLatin1String(
"htm")))
773 QFile htmlFile(fileName);
774 if (!htmlFile.open(QIODevice::ReadOnly)) {
775 emit warning(tr(
"File \"%1\" cannot be opened.").arg(fileName));
778 const QRegularExpression linkPattern(QLatin1String(
"<(?:a href|img src)=\"?([^#\">]+)[#\">]"));
779 QByteArray data = htmlFile.readAll();
780 auto encoding = QStringDecoder::encodingForHtml(data);
782 encoding = QStringDecoder::Utf8;
783 const QString &content = QStringDecoder(*encoding)(data);
784 QStringList invalidLinks;
785 QRegularExpressionMatch match;
787 while ((match = linkPattern.match(content, pos)).hasMatch()) {
788 pos = match.capturedEnd();
789 const QString &linkedFileName = match.captured(1);
790 if (linkedFileName.contains(QLatin1String(
"://")))
792 const QString &curDir = QFileInfo(fileName).dir().path();
793 const QString &canonicalLinkedFileName =
794 QFileInfo(curDir + QDir::separator() + linkedFileName).canonicalFilePath();
795 if (!files.contains(canonicalLinkedFileName)
796 && !invalidLinks.contains(canonicalLinkedFileName)) {
797 emit warning(tr(
"File \"%1\" contains an invalid link to file \"%2\"").
798 arg(fileName).arg(linkedFileName));
800 invalidLinks.append(canonicalLinkedFileName);
806 m_error = tr(
"Invalid links in HTML files.");
814 m_private =
new HelpGeneratorPrivate(
this);
816 connect(m_private, &HelpGeneratorPrivate::statusChanged,
817 this, &HelpGenerator::printStatus);
819 connect(m_private, &HelpGeneratorPrivate::warning,
820 this, &HelpGenerator::printWarning);
824 const QString &outputFileName)
826 return m_private->generate(helpData, outputFileName);
831 return m_private->checkLinks(helpData);
836 return m_private->error();
841 puts(qPrintable(msg));
846 puts(qPrintable(tr(
"Warning: %1").arg(msg)));
851#include "helpgenerator.moc"
bool checkLinks(const QHelpProjectData &helpData)
The QHelpProjectData class stores all information found in a Qt help project file.