From edc0bf5e068ade0d123a41422af5a20478e3d536 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 11 Mar 2018 21:23:23 -0600 Subject: [PATCH] Split up code a bit in prep for DESCRIPT.ION generator --- core/file_base_user_list_export.js | 292 +++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 core/file_base_user_list_export.js diff --git a/core/file_base_user_list_export.js b/core/file_base_user_list_export.js new file mode 100644 index 00000000..4a2d0c4b --- /dev/null +++ b/core/file_base_user_list_export.js @@ -0,0 +1,292 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +const { MenuModule } = require('./menu_module.js'); +const FileEntry = require('./file_entry.js'); +const FileArea = require('./file_base_area.js'); +const { renderSubstr } = require('./string_util.js'); +const { Errors } = require('./enig_error.js'); +const Events = require('./events.js'); +const Log = require('./logger.js').log; +const DownloadQueue = require('./download_queue.js'); +const exportFileList = require('./file_base_list_export.js'); + +// deps +const _ = require('lodash'); +const async = require('async'); +const fs = require('graceful-fs'); +const fse = require('fs-extra'); +const paths = require('path'); +const moment = require('moment'); +const uuidv4 = require('uuid/v4'); +const yazl = require('yazl'); + +/* + Module config block can contain the following: + templateEncoding - encoding of template files (utf8) + tsFormat - timestamp format (theme 'short') + descWidth - max desc width (45) + progBarChar - progress bar character (▒) + compressThreshold - threshold to kick in comrpession for lists (1.44 MiB) + templates - object containing: + header - filename of header template (misc/file_list_header.asc) + entry - filename of entry template (misc/file_list_entry.asc) + + Header template variables: + nowTs, boardName, totalFileCount, totalFileSize, + filterAreaTag, filterAreaName, filterAreaDesc, + filterTerms, filterHashTags + + Entry template variables: + fileId, areaName, areaDesc, userRating, fileName, + fileSize, fileDesc, fileDescShort, fileSha256, fileCrc32, + fileMd5, fileSha1, uploadBy, fileUploadTs, fileHashTags, + currentFile, progress, +*/ + +exports.moduleInfo = { + name : 'File Base List Export', + desc : 'Exports file base listings for download', + author : 'NuSkooler', +}; + +const FormIds = { + main : 0, +}; + +const MciViewIds = { + main : { + status : 1, + progressBar : 2, + + customRangeStart : 10, + } +}; + +exports.getModule = class FileBaseListExport extends MenuModule { + + constructor(options) { + super(options); + this.config = Object.assign({}, _.get(options, 'menuConfig.config'), options.extraArgs); + + this.config.templateEncoding = this.config.templateEncoding || 'utf8'; + this.config.tsFormat = this.config.tsFormat || this.client.currentTheme.helpers.getDateTimeFormat('short'); + this.config.descWidth = this.config.descWidth || 45; // ie FILE_ID.DIZ + this.config.progBarChar = renderSubstr( (this.config.progBarChar || '▒'), 0, 1); + this.config.compressThreshold = this.config.compressThreshold || (1440000); // >= 1.44M by default :) + } + + mciReady(mciData, cb) { + super.mciReady(mciData, err => { + if(err) { + return cb(err); + } + + async.series( + [ + (callback) => this.prepViewController('main', FormIds.main, mciData.menu, callback), + (callback) => this.prepareList(callback), + ], + err => { + if(err) { + if('NORESULTS' === err.reasonCode) { + return this.gotoMenu(this.menuConfig.config.noResultsMenu || 'fileBaseExportListNoResults'); + } + + return this.prevMenu(); + } + return cb(err); + } + ); + }); + } + + finishedLoading() { + this.prevMenu(); + } + + prepareList(cb) { + const self = this; + + const statusView = self.viewControllers.main.getView(MciViewIds.main.status); + const updateStatus = (status) => { + if(statusView) { + statusView.setText(status); + } + }; + + const progBarView = self.viewControllers.main.getView(MciViewIds.main.progressBar); + const updateProgressBar = (curr, total) => { + if(progBarView) { + const prog = Math.floor( (curr / total) * progBarView.dimens.width ); + progBarView.setText(self.config.progBarChar.repeat(prog)); + } + }; + + let cancel = false; + + const exportListProgress = (state, progNext) => { + switch(state.step) { + case 'preparing' : + case 'gathering' : + updateStatus(state.status); + break; + case 'file' : + updateStatus(state.status); + updateProgressBar(state.current, state.total); + self.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.fileInfo); + break; + default : + break; + } + + return progNext(cancel ? Errors.General('User canceled') : null); + }; + + const keyPressHandler = (ch, key) => { + if('escape' === key.name) { + cancel = true; + self.client.removeListener('key press', keyPressHandler); + } + }; + + async.waterfall( + [ + function buildList(callback) { + // this may take quite a while; temp disable of idle monitor + self.client.stopIdleMonitor(); + + const filterCriteria = Object.assign({}, self.config.filterCriteria); + if(!filterCriteria.areaTag) { + filterCriteria.areaTag = FileArea.getAvailableFileAreaTags(self.client); + } + + const opts = { + templateEncoding : self.config.templateEncoding, + headerTemplate : _.get(self.config, 'templates.header', 'file_list_header.asc'), + entryTemplate : _.get(self.config, 'templates.entry', 'file_list_entry.asc'), + tsFormat : self.config.tsFormat, + descWidth : self.config.descWidth, + progress : exportListProgress, + }; + + exportFileList(filterCriteria, opts, (err, listBody) => { + return callback(err, listBody); + }); + }, + function persistList(listBody, callback) { + updateStatus('Persisting list'); + + const sysTempDownloadArea = FileArea.getFileAreaByTag(FileArea.WellKnownAreaTags.TempDownloads); + const sysTempDownloadDir = FileArea.getAreaDefaultStorageDirectory(sysTempDownloadArea); + + fse.mkdirs(sysTempDownloadDir, err => { + if(err) { + return callback(err); + } + + const outputFileName = paths.join( + sysTempDownloadDir, + `file_list_${uuidv4().substr(-8)}_${moment().format('YYYY-MM-DD')}.txt` + ); + + fs.writeFile(outputFileName, listBody, 'utf8', err => { + if(err) { + return callback(err); + } + + self.getSizeAndCompressIfMeetsSizeThreshold(outputFileName, (err, finalOutputFileName, fileSize) => { + return callback(err, finalOutputFileName, fileSize, sysTempDownloadArea); + }); + }); + }); + }, + function persistFileEntry(outputFileName, fileSize, sysTempDownloadArea, callback) { + const newEntry = new FileEntry({ + areaTag : sysTempDownloadArea.areaTag, + fileName : paths.basename(outputFileName), + storageTag : sysTempDownloadArea.storageTags[0], + meta : { + upload_by_username : self.client.user.username, + upload_by_user_id : self.client.user.userId, + byte_size : fileSize, + session_temp_dl : 1, // download is valid until session is over + } + }); + + newEntry.desc = 'File List Export'; + + newEntry.persist(err => { + if(!err) { + // queue it! + const dlQueue = new DownloadQueue(self.client); + dlQueue.add(newEntry); + + // clean up after ourselves when the session ends + const thisClientId = self.client.session.id; + Events.once(Events.getSystemEvents().ClientDisconnected, evt => { + if(thisClientId === _.get(evt, 'client.session.id')) { + FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => { + if(err) { + Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); + } else { + Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' ); + } + }); + } + }); + } + return callback(err); + }); + }, + function done(callback) { + // re-enable idle monitor + self.client.startIdleMonitor(); + + updateStatus('Exported list has been added to your download queue'); + return callback(null); + } + ], + err => { + self.client.removeListener('key press', keyPressHandler); + return cb(err); + } + ); + } + + getSizeAndCompressIfMeetsSizeThreshold(filePath, cb) { + fse.stat(filePath, (err, stats) => { + if(err) { + return cb(err); + } + + if(stats.size < this.config.compressThreshold) { + // small enough, keep orig + return cb(null, filePath, stats.size); + } + + const zipFilePath = `${filePath}.zip`; + + const zipFile = new yazl.ZipFile(); + zipFile.addFile(filePath, paths.basename(filePath)); + zipFile.end( () => { + const outZipFile = fs.createWriteStream(zipFilePath); + zipFile.outputStream.pipe(outZipFile); + zipFile.outputStream.on('finish', () => { + // delete the original + fse.unlink(filePath, err => { + if(err) { + return cb(err); + } + + // finally stat the new output + fse.stat(zipFilePath, (err, stats) => { + return cb(err, zipFilePath, stats ? stats.size : 0); + }); + }); + }); + }); + }); + } +}; \ No newline at end of file