From c2b86a44e68c8fb49b492ce348bcb6c71a931796 Mon Sep 17 00:00:00 2001 From: Pierre-Henri Date: Sat, 20 Apr 2024 17:13:55 +0200 Subject: [PATCH] use folder instead of file for configuration --- .gitignore | 1 + src/lib/file-id/FileIDComputerWorker.ts | 1 - src/lib/folder-watcher/FolderWatcher.ts | 20 ++++++++---- .../hash-compute/HashComputerIndexCache.ts | 4 +-- src/lib/hash-compute/HashIndexManager.ts | 31 +++++++++++-------- src/lib/index.ts | 4 ++- 6 files changed, 38 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index a35a66d..45b6ce0 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ yarn-error.log* /coverage/ /dist/ .idea +build diff --git a/src/lib/file-id/FileIDComputerWorker.ts b/src/lib/file-id/FileIDComputerWorker.ts index c52f793..1de4192 100644 --- a/src/lib/file-id/FileIDComputerWorker.ts +++ b/src/lib/file-id/FileIDComputerWorker.ts @@ -7,7 +7,6 @@ let distFolder = import.meta.dirname; distFolder = distFolder.replace('src', 'dist'); distFolder = distFolder + "/worker.js"; const workerUrl = new URL(distFolder).href; -console.log("Worker URL: ", workerUrl); export class FileIDComputerWorker{ private piscina: Piscina; diff --git a/src/lib/folder-watcher/FolderWatcher.ts b/src/lib/folder-watcher/FolderWatcher.ts index ac99c6e..20662a6 100644 --- a/src/lib/folder-watcher/FolderWatcher.ts +++ b/src/lib/folder-watcher/FolderWatcher.ts @@ -12,7 +12,7 @@ export class FolderWatcher { queueSize = -1; current = 0; processing = new Set(); - private readonly updateDebounced: () => Promise; + private readonly finalizeDebounced: () => Promise; constructor(fileProcessor:FileProcessorInterface,private WATCH_FOLDER_LIST: string, private config:{ interval?: number, @@ -23,7 +23,7 @@ export class FolderWatcher { this.fileProcessor = fileProcessor; if(this.fileProcessor.finalize) { const debounced = debounce(() => this.fileProcessor.finalize(), this.config.outputFolderUpdateIntervalMs || 10000, {immediate: true}); - this.updateDebounced = async () => { + this.finalizeDebounced = async () => { if (this.initialized) { return debounced(); } @@ -120,7 +120,9 @@ export class FolderWatcher { .on('add', async (filePath) => { try { await this.processFileExtended(filePath); - await this.updateDebounced(); + if(this.finalizeDebounced) { + await this.finalizeDebounced(); + } } catch (error) { console.error(`Error processing file ${filePath}:`, error); } @@ -128,7 +130,9 @@ export class FolderWatcher { .on('change', async (filePath) => { try { await this.processFileExtended(filePath); - await this.updateDebounced(); + if(this.finalizeDebounced) { + await this.finalizeDebounced(); + } } catch (e) { console.error(`Error processing file ${filePath}:`, e); } @@ -139,7 +143,9 @@ export class FolderWatcher { await this.fileProcessor.deleteFile(filePath); } await this.processFileExtended(filePath); - await this.updateDebounced(); + if(this.finalizeDebounced) { + await this.finalizeDebounced(); + } } catch (e) { console.error(`Error processing file deletion:`, e); } @@ -175,7 +181,9 @@ export class FolderWatcher { } await Promise.all(promises); this.initialized = true; - await this.updateDebounced(); + if(this.finalizeDebounced) { + await this.finalizeDebounced(); + } console.log(`Watcher ready`); } diff --git a/src/lib/hash-compute/HashComputerIndexCache.ts b/src/lib/hash-compute/HashComputerIndexCache.ts index c81a7b6..0f3fb42 100644 --- a/src/lib/hash-compute/HashComputerIndexCache.ts +++ b/src/lib/hash-compute/HashComputerIndexCache.ts @@ -8,9 +8,9 @@ export class HashComputerIndexCache implements ComputeInterface { private hashIndexManager: HashIndexManager; private hashComputer: HashComputerWorker; - constructor(indexFilePath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) { + constructor(indexFolderPath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) { this.hashComputer = new HashComputerWorker(targetHash,workerPath); - this.hashIndexManager = new HashIndexManager(indexFilePath,targetHash); + this.hashIndexManager = new HashIndexManager(indexFolderPath,targetHash); } public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise { diff --git a/src/lib/hash-compute/HashIndexManager.ts b/src/lib/hash-compute/HashIndexManager.ts index 8a02c19..638fa4c 100644 --- a/src/lib/hash-compute/HashIndexManager.ts +++ b/src/lib/hash-compute/HashIndexManager.ts @@ -27,14 +27,8 @@ export class HashIndexManager { private initialLoad: Promise; private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {}; - constructor(filePath: string, private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256]) { - filePath = filePath.replace(".csv", ""); - for (const hash of this.targetHash) { - this.filePaths[hash] = `${filePath}-${hash}.csv`; - if (!this.filePaths[hash]) { - throw new Error(`Invalid index file path for ${hash}`); - } - } + constructor(private indexFolderPath: string, + private targetHash: CID_ALGORITHM_NAMES[]) { } public getCache(): Map { @@ -49,6 +43,17 @@ export class HashIndexManager { public async init(autosave = true) { if (!this.initialLoad) { this.initialLoad = new Promise(async (resolve, reject) => { + const stat = await fs.stat(this.indexFolderPath); + if (!stat.isDirectory()) { + throw new Error(`Invalid index folder path ${this.indexFolderPath}`); + } + for (const hash of this.targetHash) { + this.filePaths[hash] = path.join(this.indexFolderPath,`index-${hash}.csv`); + console.log(`Index file path for ${hash} is ${this.filePaths[hash]}`); + if (!this.filePaths[hash]) { + throw new Error(`Invalid index file path for ${hash}`); + } + } try { for (const hash of this.targetHash) { if (!this.checkCSVHeaders(this.filePaths[hash], hash)) { @@ -226,12 +231,12 @@ export class HashIndexManager { public getCidForFile(filePath: string, fileSize: number, mtime: string): IndexLine { const fileName = path.basename(filePath); let fileNameIndex = this.cache.get(fileName); - for (const hash of this.targetHash) { - if (!fileNameIndex[hash]) { - delete fileNameIndex[hash]; - } - } if (fileNameIndex) { + for (const hash of this.targetHash) { + if (!fileNameIndex[hash]) { + delete fileNameIndex[hash]; + } + } if (fileNameIndex.mtime) { //if we have a mtime, we need to check it if (fileNameIndex.size === (fileSize + "") && fileNameIndex.mtime === mtime) { diff --git a/src/lib/index.ts b/src/lib/index.ts index 5b916f2..6e6880d 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -11,4 +11,6 @@ export * from './hash-compute/HashComputerFile.js'; //folder-watcher export * from './folder-watcher/FolderWatcher.js'; -export * from './index-interface.js'; \ No newline at end of file +export * from './index-interface.js'; + +export * from './utils/ExistsAsync.js'; \ No newline at end of file