metastar/meta-hash
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix tests

Browse Source
This commit is contained in:
Metazla
2024-04-02 19:57:03 +02:00
parent 60898adca5
commit 38b78f4f92
32 changed files with 923 additions and 2396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dist/index.js
Vendored
+5 -6
View File
@@ -1,16 +1,15 @@
export { c as CID_ALGORITHM, a as CID_ALGORITHM_CODES, b as CID_ALGORITHM_NAMES } from './chunk-VGBIRFOR.js';
import { Piscina } from 'piscina';
import { promises, createReadStream } from 'fs';
import { parse as parse$1 } from 'csv-parse';
import { parse } from 'csv-parse/sync';
import { stringify } from 'csv-stringify/sync';
import { access, constants, stat } from 'fs/promises';
import y from 'path';
import { clearInterval } from 'timers';
import 'multiformats/cid';
import 'multiformats/hashes/digest';
import I from 'path';
import { clearInterval } from 'node:timers';
var C=Object.defineProperty;var g=Object.getOwnPropertySymbols;var H=Object.prototype.hasOwnProperty,b=Object.prototype.propertyIsEnumerable;var x=(a,e,t)=>e in a?C(a,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):a[e]=t,I=(a,e)=>{for(var t in e||(e={}))H.call(e,t)&&x(a,t,e[t]);if(g)for(var t of g(e))b.call(e,t)&&x(a,t,e[t]);return a};var n=(a,e,t)=>new Promise((i,s)=>{var r=c=>{try{h(t.next(c));}catch(f){s(f);}},o=c=>{try{h(t.throw(c));}catch(f){s(f);}},h=c=>c.done?i(c.value):Promise.resolve(c.value).then(r,o);h((t=t.apply(a,e)).next());});var S={};var m=class{constructor(){this.piscina=new Piscina({maxThreads:4,filename:new URL("./ShaComputeWorker.js",S.url).href});}computeCIDs(e,t){return n(this,null,function*(){return this.piscina.run({filePath:e,algorithms:t})})}};var d=class{constructor(e){this.targetHash=e;this.fileIDComputer=new m;}computeMissingHash(e,t){return n(this,null,function*(){let i=this.targetHash.filter(r=>!t[r]);if(i.length===0)return;(yield this.fileIDComputer.computeCIDs(e,i)).forEach((r,o)=>{let h=i[o];t[h]=r;});})}};var v=(s=>(s[s.sha256=18]="sha256",s[s.sha1=17]="sha1",s[s.md5=213]="md5",s[s.sha3_256=22]="sha3_256",s))(v||{}),p=(s=>(s.sha256="cid_sha2-256",s.sha1="cid_sha1",s.md5="cid_md5",s.sha3_256="cid_sha3-256",s))(p||{}),F=(s=>(s.sha256="sha2-256",s.sha1="sha1",s.md5="md5",s.sha3_256="sha3-256",s))(F||{});function u(a){return n(this,null,function*(){try{return yield access(a,constants.F_OK),!0}catch(e){return !1}})}var ie=["path","size","mtime"],l=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.filePath=e;this.targetHash=t;this.cache=new Map;this.intervalTime=3e4;this.lastIndexFileSize=0;this.indexOpsInProgress=!1;this.hasChanged=!1;if(!e)throw new Error("Invalid index file path")}getCache(){return new Map(this.cache)}init(e=!0){return n(this,null,function*(){return this.initialLoad||(this.initialLoad=new Promise((t,i)=>n(this,null,function*(){try{if(!this.checkCSVHeaders(this.filePath))throw new Error("Invalid index file headers");yield this.loadIndex(),e&&this.start(),t();}catch(s){i(s);}}))),this.initialLoad})}checkCSVHeaders(e){let t=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(t.length===0)return !0;let i=Object.keys(t[0]);return ["path","size","mtime",...this.targetHash].every(r=>i.includes(r))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}loadIndex(){return n(this,null,function*(){if(yield u(this.filePath)){let e=yield promises.stat(this.filePath);if(this.lastIndexFileSize!==e.size){let t=yield this.readCsv();for(let i of t)this.cache.set(i.path,i);return this.lastIndexFileSize=e.size,this.lastCacheFile=t,t}else return this.lastCacheFile}return []})}loadIndexFromCache(){return Array.from(this.cache.values())}readCsv(){return n(this,null,function*(){if(!(yield u(this.filePath)))return [];let e=performance.now(),t=parse$1({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,r)=>{createReadStream(this.filePath).pipe(t).on("data",o=>{i.push(o);}).on("end",()=>{s(i),console.log(`Index read time ${performance.now()-e}ms`);}).on("error",o=>{r(o);});})})}saveCacheToFile(){return n(this,null,function*(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t=yield this.loadIndex();if(this.cache.size!==0){let r=this.loadIndexFromCache().filter(o=>!t.find(h=>h.path===o.path));if(r.length!==0){let o=stringify(r,{header:t.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},...this.targetHash.map(h=>({key:h,header:h}))]});yield promises.appendFile(this.filePath,o);}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;})}getCidForFile(e,t,i){let s=y.basename(e),r=this.cache.get(s);if(r){if(r.mtime){if(r.size===t+""&&r.mtime===i)return r}else if(r.size===t+"")return r}return null}addFileCid(e,t,i,s){if(!e||!t||!i||!s)throw new Error("Invalid parameters");let r=t+"",o=y.basename(e),h=I({path:o,size:r,mtime:i},s);this.cache.set(o,h),this.hasChanged=!0;}};var M=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.targetHash=t;this.hashComputer=new d(this.targetHash),this.hashIndexManager=new l(e);}computeMissingHash(e,t){return n(this,null,function*(){yield this.hashIndexManager.init();let i=yield stat(e);if(this.hashIndexManager.getCache().has(y.basename(e))){let s=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(s)for(let r of this.targetHash)!t[r]&&s[r]&&(t[r]=s[r]);}yield this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);})}};
var h=import.meta.dirname;h=h.replace("src","dist");h=h+"/worker.js";var f=new URL(h).href;console.log("Worker URL: ",f);var c=class{piscina;constructor(e){this.piscina=new Piscina({maxThreads:4,filename:e||process.env.WORKER_URL||f});}async computeCIDs(e,t){return this.piscina.run({filePath:e,algorithms:t})}};var m=class{constructor(e,t){this.targetHash=e;this.fileIDComputer=new c(t);}fileIDComputer;async computeMissingHash(e,t){let i=this.targetHash.filter(r=>!t[r]);if(i.length===0)return;(await this.fileIDComputer.computeCIDs(e,i)).forEach((r,a)=>{let n=i[a];t[n]=r;});}};async function l(o){try{return await access(o,constants.F_OK),!0}catch{return !1}}var q=["path","size","mtime"],p=class{constructor(e,t=["cid_sha1","cid_sha2-256"]){this.filePath=e;this.targetHash=t;if(!e)throw new Error("Invalid index file path")}cache=new Map;intervalId;intervalTime=3e4;lastIndexFileSize=0;lastCacheFile;indexOpsInProgress=!1;hasChanged=!1;initialLoad;getCache(){return new Map(this.cache)}async init(e=!0){return this.initialLoad||(this.initialLoad=new Promise(async(t,i)=>{try{if(!this.checkCSVHeaders(this.filePath))throw new Error("Invalid index file headers");await this.loadIndex(),e&&this.start(),t();}catch(s){i(s);}})),this.initialLoad}checkCSVHeaders(e){let t=parse(e,{bom:!0,columns:!0,skip_empty_lines:!0});if(t.length===0)return !0;let i=Object.keys(t[0]);return ["path","size","mtime",...this.targetHash].every(r=>i.includes(r))}start(){this.startAutoSave(this.intervalTime);}stopAutoSave(){this.intervalId&&clearInterval(this.intervalId);}startAutoSave(e){this.stopAutoSave(),this.intervalTime=e,this.intervalId=setInterval(()=>this.saveCacheToFile(),e);}async loadIndex(){if(await l(this.filePath)){let e=await promises.stat(this.filePath);if(this.lastIndexFileSize!==e.size){let t=await this.readCsv();for(let i of t)this.cache.set(i.path,i);return this.lastIndexFileSize=e.size,this.lastCacheFile=t,t}else return this.lastCacheFile}return []}loadIndexFromCache(){return Array.from(this.cache.values())}async readCsv(){if(!await l(this.filePath))return [];let e=performance.now(),t=parse$1({columns:!0,skip_empty_lines:!0}),i=[];return new Promise((s,r)=>{createReadStream(this.filePath).pipe(t).on("data",a=>{i.push(a);}).on("end",()=>{s(i),console.log(`Index read time ${performance.now()-e}ms`);}).on("error",a=>{r(a);});})}async saveCacheToFile(){if(this.indexOpsInProgress||!this.hasChanged)return;this.hasChanged=!1,this.indexOpsInProgress=!0;let e=performance.now(),t=await this.loadIndex();if(this.cache.size!==0){let r=this.loadIndexFromCache().filter(a=>!t.find(n=>n.path===a.path));if(r.length!==0){let a=stringify(r,{header:t.length===0,columns:[{key:"path",header:"path"},{key:"size",header:"size"},{key:"mtime",header:"mtime"},...this.targetHash.map(n=>({key:n,header:n}))]});await promises.appendFile(this.filePath,a);}}let i=performance.now()-e;console.log(`Index saved in ${i}ms`),i*10>this.intervalTime&&(this.startAutoSave(i*10),console.log(`Index save interval increased to ${i*10}ms`)),this.indexOpsInProgress=!1;}getCidForFile(e,t,i){let s=I.basename(e),r=this.cache.get(s);if(r){if(r.mtime){if(r.size===t+""&&r.mtime===i)return r}else if(r.size===t+"")return r}return null}addFileCid(e,t,i,s){if(!e||!t||!i||!s)throw new Error("Invalid parameters");let r=t+"",a=I.basename(e),n={path:a,size:r,mtime:i,...s};this.cache.set(a,n),this.hasChanged=!0;}};var g=class{constructor(e,t=["cid_sha1","cid_sha2-256"],i){this.targetHash=t;this.hashComputer=new m(this.targetHash,i),this.hashIndexManager=new p(e);}hashIndexManager;hashComputer;async computeMissingHash(e,t){await this.hashIndexManager.init();let i=await stat(e);if(this.hashIndexManager.getCache().has(I.basename(e))){let s=this.hashIndexManager.getCidForFile(e,i.size,i.mtime.toISOString());if(s)for(let r of this.targetHash)!t[r]&&s[r]&&(t[r]=s[r]);}await this.hashComputer.computeMissingHash(e,t),this.hashIndexManager.addFileCid(e,i.size,i.mtime.toISOString(),t);}};
export { F as CID_ALGORITHM, v as CID_ALGORITHM_CODES, p as CID_ALGORITHM_NAMES, M as ComputeHashIndexCache, m as FileIDComputer, d as HashComputer, l as HashIndexManager, ie as INDEX_HEADERS, u as existsAsync };
export { g as ComputeHashIndexCache, c as FileIDComputer, m as HashComputer, p as HashIndexManager, q as INDEX_HEADERS, l as existsAsync };
//# sourceMappingURL=out.js.map
//# sourceMappingURL=index.js.map