feat: fast hash checker

Core/UpdateController.cpp

@@ -1,6 +1,7 @@
 #include "UpdateController.h"
 
 
+#include <QElapsedTimer>
 #include <QThread>
 
 UpdateController::UpdateController(QObject *parent) :
@@ -15,23 +16,33 @@ void UpdateController::initialize(VersionContainer *versionContainer,DataParserO
     this->versionContainer = versionContainer;
     this->sendSystem = sendSystem;
     this->dataParserOut = dataParserOut;
+    hashCalculator = new FastHashCalculator;
 }
 
 void UpdateController::calculateCommonHash()
 {
     appDataList.clear();
-    appDataList = calculateHash(applicationFolderPath,"StreamingAssets");
+    QElapsedTimer timer;
+    timer.start();
+    qDebug() << "Start calculate... ";
+    //appDataList = calculateHash(applicationFolderPath,"StreamingAssets");
+    hashCalculator->calculateHashes(applicationFolderPath,"StreamingAssets");
+    appDataList = *hashCalculator->getHashList();
     calculateStreamingHash();
     appDataList.append(streamingDataList);
+    qDebug() << "Hash count: " << appDataList.count() ;
     dataParserOut->createFileDataList(appDataList,fullStaticDataFolderName + hashFilename);
     qDebug() << "UpdateController threadID " << QThread::currentThreadId();
+    qDebug() <<"Calculate time " << timer.elapsed();
 }
 
 void UpdateController::calculateStreamingHash()
 {
     streamingDataList.clear();
-    streamingDataList = calculateHash(QDir::currentPath() + streamingAssetsPath,"");
-    std::sort(streamingDataList.begin(),streamingDataList.end());
+    //streamingDataList = calculateHash(QDir::currentPath() + streamingAssetsPath,"");
+    hashCalculator->calculateHashes(QDir::currentPath() + streamingAssetsPath,"");
+    streamingDataList = *hashCalculator->getHashList();
+    //std::sort(streamingDataList.begin(),streamingDataList.end());
     dataParserOut->createFileDataList(streamingDataList,streamingHashFilename);
 }
 
@@ -84,7 +95,7 @@ QList<FileData> UpdateController::calculateHash(const QString& path,const QStrin
 
             emit sigSendHashInfo(fullSize,Tools::convertFileSize(currentSize,false));
             quint64 fileSize = file.size();     //буффер для хэширования крупных файлов
-            const quint64 bufferSize = 10240;
+            const quint64 bufferSize = 1024;
 
             if(fileInfo.isHidden()) continue;
             if(ignoreName != "" && fileInfo.path().contains(ignoreName)) continue;

Core/UpdateController.h

@@ -3,6 +3,7 @@
 
 #include "Core/sendsystem.h"
 #include "Core/versioncontainer.h"
+#include "fasthashcalculator.h"
 #include <QXmlStreamWriter>
 #include <QXmlStreamReader>
 #include <QXmlStreamAttribute>
@@ -40,6 +41,7 @@ signals:
 private:
     SendSystem *sendSystem;
     DataParserOutput * dataParserOut;
+    FastHashCalculator* hashCalculator;
     QString applicationFolderPath;
     VersionContainer *versionContainer;
     QList<FileData> appDataList;

Core/fasthashcalculator.cpp

@@ -0,0 +1,98 @@
+#include "fasthashcalculator.h"
+#include <QtConcurrent>
+
+
+FastHashCalculator::FastHashCalculator(QObject *parent) : QObject(parent)
+{
+    hashList = new QList<FileData>();
+}
+void FastHashCalculator::calculateHashes(const QString& path, const QString& ignoreName)
+{
+    hashList->clear();
+
+    if(!QDir(path).exists()){
+        QDir().mkdir(path);
+    }
+
+    QString hashString;
+    QStringList filter;
+    filter << "*";
+    QList<FileData> *folders = new QList<FileData>;
+    //QString fullSize = Tools::convertFileSize(getDirectorySize(path),false);
+
+    QDirIterator dirIterator(path,filter, QDir::AllEntries, QDirIterator::Subdirectories);
+
+    while (dirIterator.hasNext())
+    {
+        QFileInfo fileInfo(dirIterator.next());
+        FileData currentFolder;
+        if(fileInfo.isDir() && !fileInfo.fileName().startsWith(".") && fileInfo.fileName() != "RRJLoader")
+        {
+            currentFolder.path = Tools::createLocalPath(fileInfo.absoluteFilePath());
+            currentFolder.hash = "FOLDER";
+
+            if(!folders->contains(currentFolder))
+            {
+                folders->push_back(currentFolder);
+            }
+        }
+    }
+
+
+    QDirIterator fileIterator(path,filter,QDir::Files | QDir::NoDotAndDotDot,QDirIterator::Subdirectories);
+
+    QList<QString> files;
+    files.clear();
+
+    while(fileIterator.hasNext())
+    {
+        fileIterator.next();
+        QFileInfo fileInfo = fileIterator.fileInfo();
+        QString path = fileInfo.absoluteFilePath();
+
+        if (fileInfo.isHidden()) continue;
+        if (!fileInfo.isFile()) continue;
+        if (fileInfo.fileName().contains(".meta")) continue;
+        if (ignoreName != "" && fileInfo.path().contains(ignoreName)) continue;
+
+        files.append(path);
+    }
+
+    QtConcurrent::map(files, [this](const QString &filePath)
+    {
+        QByteArray hash = calculateFileHashOptimized(filePath);
+        QMutexLocker locker(&_mutex);
+        FileData currentFile;
+        QString hashName;
+
+        currentFile.path = Tools::createLocalPath(filePath);
+        currentFile.hash = hash.toHex();
+        hashList->append(currentFile);
+     }).waitForFinished();
+
+    hashList->append(*folders);
+    emit finished();
+}
+
+QByteArray FastHashCalculator::calculateFileHashOptimized(const QString &filePath)
+{
+    QFile file(filePath);
+            if (!file.open(QIODevice::ReadOnly)) return QByteArray();
+
+            QCryptographicHash hash(QCryptographicHash::Md5);
+            const qint64 bufferSize = 2 * 1024; // 2MB
+            QByteArray buffer;
+            buffer.resize(bufferSize);
+
+            while (!file.atEnd()) {
+                qint64 bytesRead = file.read(buffer.data(), bufferSize);
+                hash.addData(buffer.constData(), bytesRead);
+            }
+
+            return hash.result();
+}
+
+QList<FileData> *FastHashCalculator::getHashList() const
+{
+    return hashList;
+}

Core/fasthashcalculator.h

@@ -0,0 +1,34 @@
+#ifndef FASTHASHCALCULATOR_H
+#define FASTHASHCALCULATOR_H
+
+#include <QHash>
+#include <QMutex>
+#include <QObject>
+#include <QFile>
+#include <QDirIterator>
+#include <Data/FileData.h>
+#include "tools.h"
+
+
+class FastHashCalculator : public QObject
+{
+    Q_OBJECT
+public:
+    explicit FastHashCalculator(QObject *parent = nullptr);
+    void calculateHashes(const QString& path, const QString& ignoreName);
+    QList<FileData> *getHashList() const;
+
+signals:
+    void finished();
+
+private:
+    QByteArray calculateFileHashOptimized(const QString &filePath);
+    void calculateSingleHash(const QString &filePath);
+
+    QList<FileData>* hashList;
+    QMutex _mutex;
+    quint64 currentSize;
+
+};
+
+#endif // FASTHASHCALCULATOR_H

Core/tcpclient.cpp

@@ -47,7 +47,6 @@ void TCPClient::setConnect(ServerSettings *serverSettings)
     }
 }
 
-
 void TCPClient::setDisconnect()
 {
     socket->disconnect();

Core/tcpclient.h

@@ -23,7 +23,7 @@ public:
     void initialize(RecognizeSystem *recognize,SendSystem *sendSystem);
     void setConnect(ServerSettings *serverSettings);
     ~TCPClient(){};
-
+    void setDisconnect();
     bool getIsConnected() const;
 
 signals:
@@ -43,8 +43,6 @@ private:
     QTcpSocket *socket;
     RecognizeSystem *recognizeSystem;
     bool isConnected;
-
-    void setDisconnect();
 };
 
 #endif // TCPCLIENT_H