diff --git a/ServerLMS/CMakeLists.txt b/ServerLMS/CMakeLists.txt index f725f7c..9a22b20 100644 --- a/ServerLMS/CMakeLists.txt +++ b/ServerLMS/CMakeLists.txt @@ -41,6 +41,8 @@ add_library(ServerLMS SHARED Systems/tools.h Systems/chatsystem.cpp Systems/chatsystem.h + Systems/fasthashcalculator.cpp + Systems/fasthashcalculator.h Data/usertype.h providerdblms.cpp providerdblms.h diff --git a/ServerLMS/Systems/fasthashcalculator.cpp b/ServerLMS/Systems/fasthashcalculator.cpp new file mode 100644 index 0000000..9d648d9 --- /dev/null +++ b/ServerLMS/Systems/fasthashcalculator.cpp @@ -0,0 +1,96 @@ +#include "fasthashcalculator.h" +#include + + +FastHashCalculator::FastHashCalculator(QObject *parent) : QObject(parent) +{ + hashList = new QList(); +} +void FastHashCalculator::calculateHashes(QString path) +{ + hashList->clear(); + + if(!QDir(path).exists()){ + QDir().mkdir(path); + } + + QString hashString; + QStringList filter; + filter << "*"; + QList *folders = new QList; + + QDirIterator dirIterator(path,filter, QDir::AllEntries, QDirIterator::Subdirectories); + + while (dirIterator.hasNext()) + { + QFileInfo fileInfo(dirIterator.next()); + FileData currentFolder; + if(fileInfo.isDir() && !fileInfo.fileName().startsWith(".") && fileInfo.fileName() != projectFolderName) + { + currentFolder.path = Tools::createLocalPath(fileInfo.absoluteFilePath()); + currentFolder.hash = "FOLDER"; + + if(!folders->contains(currentFolder)) + { + folders->push_back(currentFolder); + } + } + } + + hashList->append(*folders); + + QDirIterator fileIterator(path,filter,QDir::Files | QDir::NoDotAndDotDot,QDirIterator::Subdirectories); + + QList files; + files.clear(); + + while(fileIterator.hasNext()) + { + fileIterator.next(); + QFileInfo fileInfo = fileIterator.fileInfo(); + QString path = fileInfo.absoluteFilePath(); + + if (fileInfo.isHidden()) continue; + if (!fileInfo.isFile()) continue; + if (fileInfo.fileName().contains(".meta")) continue; + + files.append(path); + } + + QtConcurrent::map(files, [this](const QString &filePath) + { + QByteArray hash = calculateFileHashOptimized(filePath); + QMutexLocker locker(&_mutex); + FileData currentFile; + QString hashName; + + currentFile.path = Tools::createLocalPath(filePath); + currentFile.hash = hash.toHex(); + hashList->append(currentFile); + }).waitForFinished(); + + emit finished(); +} + +QByteArray FastHashCalculator::calculateFileHashOptimized(const QString &filePath) +{ + QFile file(filePath); + if (!file.open(QIODevice::ReadOnly)) return QByteArray(); + + QCryptographicHash hash(QCryptographicHash::Md5); + const qint64 bufferSize = 2 * 1024; // 2MB + QByteArray buffer; + buffer.resize(bufferSize); + + while (!file.atEnd()) { + qint64 bytesRead = file.read(buffer.data(), bufferSize); + hash.addData(buffer.constData(), bytesRead); + } + + return hash.result(); +} + +QList *FastHashCalculator::getHashList() const +{ + return hashList; +} diff --git a/ServerLMS/Systems/fasthashcalculator.h b/ServerLMS/Systems/fasthashcalculator.h new file mode 100644 index 0000000..0990925 --- /dev/null +++ b/ServerLMS/Systems/fasthashcalculator.h @@ -0,0 +1,31 @@ +#ifndef FASTHASHCALCULATOR_H +#define FASTHASHCALCULATOR_H + +#include +#include +#include +#include +#include +#include +#include "tools.h" + +class FastHashCalculator : public QObject +{ + Q_OBJECT +public: + explicit FastHashCalculator(QObject *parent = nullptr); + void calculateHashes(QString path); + QList *getHashList() const; + +signals: + void finished(); + +private: + QByteArray calculateFileHashOptimized(const QString &filePath); + void calculateSingleHash(const QString &filePath); + + QList* hashList; + QMutex _mutex; +}; + +#endif // FASTHASHCALCULATOR_H diff --git a/ServerLMS/Systems/updatecontroller.cpp b/ServerLMS/Systems/updatecontroller.cpp index 7506836..461c4ef 100644 --- a/ServerLMS/Systems/updatecontroller.cpp +++ b/ServerLMS/Systems/updatecontroller.cpp @@ -16,6 +16,7 @@ void UpdateController::initialize(CommonClientHandler *commonClientHandler,DataP this->commonClientHandler = commonClientHandler; this->dataParser = dataParser; this->assetManager = assetManager; + hashCalculator = new FastHashCalculator; if (!QDir(staticDataFolderName).exists()) { @@ -87,12 +88,21 @@ void UpdateController::showHash() void UpdateController::calculateFullHash() { Logger::instance().log("Calculate hash..."); + QElapsedTimer timer; + timer.start(); + qDebug() << "Start calculate... "; commonClientHandler->slot_sendPacketToAllClients(PacketType::BUSY); - auto *list = calculateHash(buildPath); + //auto *list = calculateHash(buildPath); + + hashCalculator->calculateHashes(buildPath); + auto* list = hashCalculator->getHashList(); + qDebug() << "Hash count: " << list->count(); saveHash(buildHashName,list); calculateSharedHash(); Logger::instance().log("Calculate hash complete"); + qDebug() << "Calculate time " << timer.elapsed(); commonClientHandler->slot_sendPacketToAllClients(PacketType::FREE); + } void UpdateController::calculateFullHashWithSetup() @@ -451,6 +461,7 @@ void UpdateController::calculateSharedHash() while (dirIterator.hasNext()) { + fileList->clear(); dirIterator.next(); QFileInfo fileInfo = dirIterator.fileInfo(); if (fileInfo.fileName() == "." || fileInfo.fileName() == "..") @@ -458,7 +469,10 @@ void UpdateController::calculateSharedHash() QString fileName = Tools::createVersionHashFilepath(fileInfo.fileName()); - fileList = calculateHash(fileInfo.absoluteFilePath()); + hashCalculator->calculateHashes(fileInfo.filePath()); + fileList = hashCalculator->getHashList(); + //fileList = calculateHash(fileInfo.absoluteFilePath()); + qDebug() << "Hash count: " << fileList->count(); saveHash(fileName,fileList); StreamingVersionData *version = new StreamingVersionData( @@ -475,7 +489,11 @@ void UpdateController::calculateCurrentSharedHash() { QList *fileList = new QList; QString fileName = Tools::createVersionHashFilepath(assetManager->getCurrentVersionData()->getViewName()); - fileList = calculateHash(currentStreamingPath); + + hashCalculator->calculateHashes(currentStreamingPath); + fileList = hashCalculator->getHashList(); + //fileList = calculateHash(currentStreamingPath); + qDebug() << "Hash count: " << fileList->count(); saveHash(fileName,fileList); } diff --git a/ServerLMS/Systems/updatecontroller.h b/ServerLMS/Systems/updatecontroller.h index ffb6312..de6f087 100644 --- a/ServerLMS/Systems/updatecontroller.h +++ b/ServerLMS/Systems/updatecontroller.h @@ -1,7 +1,6 @@ #ifndef UPDATECONTROLLER_H #define UPDATECONTROLLER_H - #include #include #include @@ -11,10 +10,10 @@ #include #include #include - #include #include #include +#include "fasthashcalculator.h" class TCPServer; class SendSystem; @@ -24,7 +23,6 @@ class AssetsManager; class ServerLMSWidget; class CommonClientHandler; class Logger; - class UpdateController : public QObject { Q_OBJECT @@ -83,6 +81,7 @@ private: QString currentStreamingPath; QString sharedDataPath; CommonClientHandler *commonClientHandler; + FastHashCalculator *hashCalculator; DataParser *dataParser; AssetsManager *assetManager; quint64 sizeToSend;