diff --git a/.yarnrc.yml b/.yarnrc.yml deleted file mode 100644 index 9eb0583..0000000 --- a/.yarnrc.yml +++ /dev/null @@ -1,2 +0,0 @@ -nmHoistingLimits: dependencies -nodeLinker: node-modules diff --git a/package.json b/package.json index 25fe615..2855a95 100644 --- a/package.json +++ b/package.json @@ -31,8 +31,7 @@ } }, "scripts": { - "build": "cross-env rimraf ./dist & tsup", - "ci": "yarn run coverage", + "build": "tsup", "test": "vitest" }, "devDependencies": { @@ -49,6 +48,7 @@ "crc-32": "^1.2.2", "csv-parse": "^5.5.5", "csv-stringify": "^6.4.6", + "debounce": "^2.0.0", "multiformats": "^13.1.0", "p-queue": "^8.0.1", "piscina": "^4.4.0" diff --git a/src/lib/folder-watcher/FileProcessorInterface.ts b/src/lib/folder-watcher/FileProcessorInterface.ts index 4d5e654..8de2982 100644 --- a/src/lib/folder-watcher/FileProcessorInterface.ts +++ b/src/lib/folder-watcher/FileProcessorInterface.ts @@ -3,5 +3,7 @@ export interface FileProcessorInterface { canProcessFile(filePath: string): Promise; - deleteFile(filePath: string): Promise + deleteFile?(filePath: string): Promise; + + finalize?(): Promise; } \ No newline at end of file diff --git a/src/lib/folder-watcher/FolderWatcher.ts b/src/lib/folder-watcher/FolderWatcher.ts index 8857504..ac99c6e 100644 --- a/src/lib/folder-watcher/FolderWatcher.ts +++ b/src/lib/folder-watcher/FolderWatcher.ts @@ -2,6 +2,7 @@ import * as path from 'path'; import chokidar from 'chokidar'; import {readdir} from "fs/promises"; import PQueue from "p-queue"; +import debounce from 'debounce'; import {FileProcessorInterface} from "./FileProcessorInterface.js"; export class FolderWatcher { @@ -11,13 +12,24 @@ export class FolderWatcher { queueSize = -1; current = 0; processing = new Set(); + private readonly updateDebounced: () => Promise; constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{ interval?: number, stabilityThreshold?: number, pollInterval?: number, + outputFolderUpdateIntervalMs?: number }) { this.fileProcessor = fileProcessor; + if(this.fileProcessor.finalize) { + const debounced = debounce(() => this.fileProcessor.finalize(), this.config.outputFolderUpdateIntervalMs || 10000, {immediate: true}); + this.updateDebounced = async () => { + if (this.initialized) { + return debounced(); + } + } + } + } /** @@ -108,6 +120,7 @@ export class FolderWatcher { .on('add', async (filePath) => { try { await this.processFileExtended(filePath); + await this.updateDebounced(); } catch (error) { console.error(`Error processing file ${filePath}:`, error); } @@ -115,14 +128,18 @@ export class FolderWatcher { .on('change', async (filePath) => { try { await this.processFileExtended(filePath); + await this.updateDebounced(); } catch (e) { console.error(`Error processing file ${filePath}:`, e); } }) .on('unlink', async (filePath) => { try { - await this.fileProcessor.deleteFile(filePath); + if(this.fileProcessor.deleteFile) { + await this.fileProcessor.deleteFile(filePath); + } await this.processFileExtended(filePath); + await this.updateDebounced(); } catch (e) { console.error(`Error processing file deletion:`, e); } @@ -158,6 +175,7 @@ export class FolderWatcher { } await Promise.all(promises); this.initialized = true; + await this.updateDebounced(); console.log(`Watcher ready`); } diff --git a/src/lib/hash-compute/HashComputerFile.ts b/src/lib/hash-compute/HashComputerFile.ts new file mode 100644 index 0000000..8684b2e --- /dev/null +++ b/src/lib/hash-compute/HashComputerFile.ts @@ -0,0 +1,31 @@ +import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData"; +import {Readable} from "stream"; + +import {computeCIDs} from "@root/file-id/ComputeHash"; +import fs from "fs"; + +export class HashComputerFile implements ComputeInterface{ + + constructor(private targetHash: CID_ALGORITHM_NAMES[]) { + } + + public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { + // Dynamically determine which hashes are needed + const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]); + + // If all hashes are already computed, skip the processing + if (neededHashes.length === 0) { + return; + } + + // Compute only the needed CIDs + const stream = fs.createReadStream(filePath); + const cids = await computeCIDs({stream, algorithms:neededHashes}); + + // Map the computed CIDs back to their respective metadata properties + for (const [index, cid] of cids.entries()) { + const hashType = neededHashes[index]; + metadata[hashType] = cid; + } + } +} diff --git a/src/lib/hash-compute/HashIndexManager.ts b/src/lib/hash-compute/HashIndexManager.ts index 149a78b..8a02c19 100644 --- a/src/lib/hash-compute/HashIndexManager.ts +++ b/src/lib/hash-compute/HashIndexManager.ts @@ -42,7 +42,8 @@ export class HashIndexManager { } /** - * After init consseutively calls to this method will not reload the index + * After init consecutively calls to this method will not reload the index + * readThe index for the first time and enable autosave * @param autosave */ public async init(autosave = true) { @@ -56,7 +57,7 @@ export class HashIndexManager { await this.loadIndex(hash); } if (autosave) { - this.start(); + this.startAutoSave() } resolve(); } catch (e) { @@ -91,17 +92,13 @@ export class HashIndexManager { return requiredHeaders.every(header => headers.includes(header)); }; - public start() { - this.startAutoSave(this.intervalTime); - } - public stopAutoSave() { if (this.intervalId) { clearInterval(this.intervalId); } } - private startAutoSave(time: number) { + public startAutoSave(time: number = this.intervalTime) { this.stopAutoSave(); this.intervalTime = time; this.intervalId = setInterval(() => this.saveCacheToFile(), time); @@ -216,6 +213,10 @@ export class HashIndexManager { this.indexOpsInProgress = false; } + /** + * this.getCidForFile(fileName, stats.size, stats.mtime.toISOString()) + * @param filePath + */ public async getCidForFileAsync(filePath: string): Promise { const fileName = path.basename(filePath); const stats = await stat(filePath); diff --git a/src/lib/index.ts b/src/lib/index.ts index 50d2a60..5b916f2 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -6,6 +6,8 @@ export * from './file-id/ShaComputeWorker.js';//worker export * from './hash-compute/HashComputerIndexCache.js'; export * from './hash-compute/HashComputerWorker.js'; export * from './hash-compute/HashIndexManager.js'; +export * from './hash-compute/HashComputer.js'; +export * from './hash-compute/HashComputerFile.js'; //folder-watcher export * from './folder-watcher/FolderWatcher.js';