import { FileHandleFile, concatenateArrayBuffers, path, NodeFilesystem, NodeFile } from '@loaders.gl/loader-utils'; import { generateEoCD, parseEoCDRecord, updateEoCD } from "./end-of-central-directory.js"; import { CRC32Hash } from '@loaders.gl/crypto'; import { generateLocalHeader } from "./local-file-header.js"; import { generateCDHeader } from "./cd-file-header.js"; import { fetchFile } from '@loaders.gl/core'; /** * cut off CD and EoCD records from zip file * @param provider zip file * @returns tuple with three values: CD, EoCD record, EoCD information */ async function cutTheTailOff(provider) { // define where the body ends const oldEoCDinfo = await parseEoCDRecord(provider); const oldCDStartOffset = oldEoCDinfo.cdStartOffset; // define cd length const oldCDLength = Number(oldEoCDinfo.offsets.zip64EoCDOffset ? oldEoCDinfo.offsets.zip64EoCDOffset - oldCDStartOffset : oldEoCDinfo.offsets.zipEoCDOffset - oldCDStartOffset); // cut off everything except of archieve body const zipEnding = await provider.slice(oldCDStartOffset, provider.length); await provider.truncate(Number(oldCDStartOffset)); // divide cd body and eocd record const oldCDBody = zipEnding.slice(0, oldCDLength); const eocdBody = zipEnding.slice(oldCDLength, zipEnding.byteLength); return [oldCDBody, eocdBody, oldEoCDinfo]; } /** * generates CD and local headers for the file * @param fileName name of the file * @param fileToAdd buffer with the file * @param localFileHeaderOffset offset of the file local header * @returns tuple with two values: local header and file body, cd header */ async function generateFileHeaders(fileName, fileToAdd, localFileHeaderOffset) { // generating CRC32 of the content const newFileCRC322 = parseInt(await new CRC32Hash().hash(fileToAdd, 'hex'), 16); // generate local header for the file const newFileLocalHeader = generateLocalHeader({ crc32: newFileCRC322, fileName, length: fileToAdd.byteLength }); // generate hash file cd header const newFileCDHeader = generateCDHeader({ crc32: newFileCRC322, fileName, offset: localFileHeaderOffset, length: fileToAdd.byteLength }); return [ new Uint8Array(concatenateArrayBuffers(newFileLocalHeader, fileToAdd)), new Uint8Array(newFileCDHeader) ]; } /** * adds one file in the end of the archieve * @param zipUrl path to the file * @param fileToAdd new file body * @param fileName new file name */ export async function addOneFile(zipUrl, fileToAdd, fileName) { // init file handler const provider = new FileHandleFile(zipUrl, true); const [oldCDBody, eocdBody, oldEoCDinfo] = await cutTheTailOff(provider); // remember the new file local header start offset const newFileOffset = provider.length; const [localPart, cdHeaderPart] = await generateFileHeaders(fileName, fileToAdd, newFileOffset); // write down the file local header await provider.append(localPart); // add the file CD header to the CD const newCDBody = concatenateArrayBuffers(oldCDBody, cdHeaderPart); // remember the CD start offset const newCDStartOffset = provider.length; // write down new CD await provider.append(new Uint8Array(newCDBody)); // remember where eocd starts const eocdOffset = provider.length; await provider.append(updateEoCD(eocdBody, oldEoCDinfo.offsets, newCDStartOffset, eocdOffset, oldEoCDinfo.cdRecordsNumber + 1n)); } /** * creates zip archive with no compression * @note This is a node specific function that works on files * @param inputPath path where files for the achive are stored * @param outputPath path where zip archive will be placed */ export async function createZip(inputPath, outputPath, createAdditionalData) { const fileIterator = getFileIterator(inputPath); const resFile = new NodeFile(outputPath, 'w'); const fileList = []; const cdArray = []; for await (const file of fileIterator) { await addFile(file, resFile, cdArray, fileList); } if (createAdditionalData) { const additionaldata = await createAdditionalData(fileList); await addFile(additionaldata, resFile, cdArray); } const cdOffset = (await resFile.stat()).bigsize; const cd = concatenateArrayBuffers(...cdArray); await resFile.append(new Uint8Array(cd)); const eoCDStart = (await resFile.stat()).bigsize; await resFile.append(new Uint8Array(generateEoCD({ recordsNumber: cdArray.length, cdSize: cd.byteLength, cdOffset, eoCDStart }))); } /** * Adds file to zip parts * @param file file to add * @param resFile zip file body * @param cdArray zip file central directory * @param fileList list of file offsets */ async function addFile(file, resFile, cdArray, fileList) { const size = (await resFile.stat()).bigsize; fileList?.push({ fileName: file.path, localHeaderOffset: size }); const [localPart, cdHeaderPart] = await generateFileHeaders(file.path, file.file, size); await resFile.append(localPart); cdArray.push(cdHeaderPart); } /** * creates iterator providing buffer with file content and path to every file in the input folder * @param inputPath path to the input folder * @returns iterator */ export function getFileIterator(inputPath) { async function* iterable() { const fileList = await getAllFiles(inputPath); for (const filePath of fileList) { const file = await (await fetchFile(path.join(inputPath, filePath))).arrayBuffer(); yield { path: filePath, file }; } } return iterable(); } /** * creates a list of relative paths to all files in the provided folder * @param basePath path of the root folder * @param subfolder relative path from the root folder. * @returns list of paths */ export async function getAllFiles(basePath, subfolder = '', fsPassed) { const fs = fsPassed ? fsPassed : new NodeFilesystem({}); const files = await fs.readdir(pathJoin(basePath, subfolder)); const arrayOfFiles = []; for (const file of files) { const fullPath = pathJoin(basePath, subfolder, file); if ((await fs.stat(fullPath)).isDirectory) { const files = await getAllFiles(basePath, pathJoin(subfolder, file)); arrayOfFiles.push(...files); } else { arrayOfFiles.push(pathJoin(subfolder, file)); } } return arrayOfFiles; } /** * removes empty parts from path array and joins it * @param paths paths to join * @returns joined path */ function pathJoin(...paths) { const resPaths = paths.filter((val) => val.length); return path.join(...resPaths); }