feat: fast hash checker

ServerLMS/CMakeLists.txt

@@ -41,6 +41,8 @@ add_library(ServerLMS SHARED
   Systems/tools.h
   Systems/chatsystem.cpp
   Systems/chatsystem.h
+  Systems/fasthashcalculator.cpp
+  Systems/fasthashcalculator.h
   Data/usertype.h
   providerdblms.cpp
   providerdblms.h

ServerLMS/Systems/fasthashcalculator.cpp

@@ -0,0 +1,96 @@
+#include "fasthashcalculator.h"
+#include <QtConcurrent>
+
+
+FastHashCalculator::FastHashCalculator(QObject *parent) : QObject(parent)
+{
+    hashList = new QList<FileData>();
+}
+void FastHashCalculator::calculateHashes(QString path)
+{
+    hashList->clear();
+
+    if(!QDir(path).exists()){
+        QDir().mkdir(path);
+    }
+
+    QString hashString;
+    QStringList filter;
+    filter << "*";
+    QList<FileData> *folders = new QList<FileData>;
+
+    QDirIterator dirIterator(path,filter, QDir::AllEntries, QDirIterator::Subdirectories);
+
+    while (dirIterator.hasNext())
+    {
+        QFileInfo fileInfo(dirIterator.next());
+        FileData currentFolder;
+        if(fileInfo.isDir() && !fileInfo.fileName().startsWith(".") && fileInfo.fileName() != projectFolderName)
+        {
+            currentFolder.path = Tools::createLocalPath(fileInfo.absoluteFilePath());
+            currentFolder.hash = "FOLDER";
+
+            if(!folders->contains(currentFolder))
+            {
+                folders->push_back(currentFolder);
+            }
+        }
+    }
+
+    hashList->append(*folders);
+
+    QDirIterator fileIterator(path,filter,QDir::Files | QDir::NoDotAndDotDot,QDirIterator::Subdirectories);
+
+    QList<QString> files;
+    files.clear();
+
+    while(fileIterator.hasNext())
+    {
+        fileIterator.next();
+        QFileInfo fileInfo = fileIterator.fileInfo();
+        QString path = fileInfo.absoluteFilePath();
+
+        if (fileInfo.isHidden()) continue;
+        if (!fileInfo.isFile()) continue;
+        if (fileInfo.fileName().contains(".meta")) continue;
+
+        files.append(path);
+    }
+
+    QtConcurrent::map(files, [this](const QString &filePath)
+    {
+        QByteArray hash = calculateFileHashOptimized(filePath);
+        QMutexLocker locker(&_mutex);
+        FileData currentFile;
+        QString hashName;
+
+        currentFile.path = Tools::createLocalPath(filePath);
+        currentFile.hash = hash.toHex();
+        hashList->append(currentFile);
+     }).waitForFinished();
+
+    emit finished();
+}
+
+QByteArray FastHashCalculator::calculateFileHashOptimized(const QString &filePath)
+{
+    QFile file(filePath);
+            if (!file.open(QIODevice::ReadOnly)) return QByteArray();
+
+            QCryptographicHash hash(QCryptographicHash::Md5);
+            const qint64 bufferSize = 2 * 1024; // 2MB
+            QByteArray buffer;
+            buffer.resize(bufferSize);
+
+            while (!file.atEnd()) {
+                qint64 bytesRead = file.read(buffer.data(), bufferSize);
+                hash.addData(buffer.constData(), bytesRead);
+            }
+
+            return hash.result();
+}
+
+QList<FileData> *FastHashCalculator::getHashList() const
+{
+    return hashList;
+}

ServerLMS/Systems/fasthashcalculator.h

@@ -0,0 +1,31 @@
+#ifndef FASTHASHCALCULATOR_H
+#define FASTHASHCALCULATOR_H
+
+#include <QHash>
+#include <QMutex>
+#include <QObject>
+#include <QFile>
+#include <QDirIterator>
+#include <Data/typesDataServerClient.h>
+#include "tools.h"
+
+class FastHashCalculator : public QObject
+{
+    Q_OBJECT
+public:
+    explicit FastHashCalculator(QObject *parent = nullptr);
+    void calculateHashes(QString path);
+    QList<FileData> *getHashList() const;
+
+signals:
+    void finished();
+
+private:
+    QByteArray calculateFileHashOptimized(const QString &filePath);
+    void calculateSingleHash(const QString &filePath);
+
+    QList<FileData>* hashList;
+    QMutex _mutex;
+};
+
+#endif // FASTHASHCALCULATOR_H

ServerLMS/Systems/updatecontroller.cpp

@@ -16,6 +16,7 @@ void UpdateController::initialize(CommonClientHandler *commonClientHandler,DataP
     this->commonClientHandler = commonClientHandler;
     this->dataParser = dataParser;
     this->assetManager = assetManager;
+    hashCalculator = new FastHashCalculator;
 
     if (!QDir(staticDataFolderName).exists())
     {
@@ -87,12 +88,21 @@ void UpdateController::showHash()
 void UpdateController::calculateFullHash()
 {
     Logger::instance().log("Calculate hash...");
+    QElapsedTimer timer;
+    timer.start();
+    qDebug() << "Start calculate... ";
     commonClientHandler->slot_sendPacketToAllClients(PacketType::BUSY);
-    auto *list = calculateHash(buildPath);
+    //auto *list = calculateHash(buildPath);
+
+    hashCalculator->calculateHashes(buildPath);
+    auto* list = hashCalculator->getHashList();
+    qDebug() << "Hash count: " << list->count();
     saveHash(buildHashName,list);
     calculateSharedHash();
     Logger::instance().log("Calculate hash complete");
+    qDebug() << "Calculate time " << timer.elapsed();
     commonClientHandler->slot_sendPacketToAllClients(PacketType::FREE);
+
 }
 
 void UpdateController::calculateFullHashWithSetup()
@@ -451,6 +461,7 @@ void UpdateController::calculateSharedHash()
 
     while (dirIterator.hasNext())
     {
+        fileList->clear();
         dirIterator.next();
         QFileInfo fileInfo = dirIterator.fileInfo();
         if (fileInfo.fileName() == "." || fileInfo.fileName() == "..")
@@ -458,7 +469,10 @@ void UpdateController::calculateSharedHash()
 
         QString fileName = Tools::createVersionHashFilepath(fileInfo.fileName());
 
-        fileList = calculateHash(fileInfo.absoluteFilePath());
+        hashCalculator->calculateHashes(fileInfo.filePath());
+        fileList = hashCalculator->getHashList();
+        //fileList = calculateHash(fileInfo.absoluteFilePath());
+        qDebug() << "Hash count: " << fileList->count();
         saveHash(fileName,fileList);
 
         StreamingVersionData *version = new StreamingVersionData(
@@ -475,7 +489,11 @@ void UpdateController::calculateCurrentSharedHash()
 {
     QList<FileData> *fileList = new QList<FileData>;
     QString fileName = Tools::createVersionHashFilepath(assetManager->getCurrentVersionData()->getViewName());
-    fileList = calculateHash(currentStreamingPath);
+
+    hashCalculator->calculateHashes(currentStreamingPath);
+    fileList = hashCalculator->getHashList();
+    //fileList = calculateHash(currentStreamingPath);
+    qDebug() << "Hash count: " << fileList->count();
     saveHash(fileName,fileList);
 }
 

ServerLMS/Systems/updatecontroller.h

@@ -1,7 +1,6 @@
 #ifndef UPDATECONTROLLER_H
 #define UPDATECONTROLLER_H
 
-
 #include <QString>
 #include <QObject>
 #include <QCryptographicHash>
@@ -11,10 +10,10 @@
 #include <QMutex>
 #include <QApplication>
 #include <QMessageBox>
-
 #include <Systems/Parsers/dataparser.h>
 #include <Data/typesDataServerClient.h>
 #include <Data/StreamingVersionData.h>
+#include "fasthashcalculator.h"
 
 class TCPServer;
 class SendSystem;
@@ -24,7 +23,6 @@ class AssetsManager;
 class ServerLMSWidget;
 class CommonClientHandler;
 class Logger;
-
 class UpdateController : public QObject
 {
     Q_OBJECT
@@ -83,6 +81,7 @@ private:
     QString currentStreamingPath;
     QString sharedDataPath;
     CommonClientHandler *commonClientHandler;
+    FastHashCalculator *hashCalculator;
     DataParser *dataParser;
     AssetsManager *assetManager;
     quint64 sizeToSend;