From d0e2d41c002d1386bcceb44ba05ead0d316f4398 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 21 May 2017 11:45:50 -0600 Subject: [PATCH 1/5] Ability to scan single archive (e.g. compressed .pdf) -> extract useful description --- core/config.js | 5 +- core/file_base_area.js | 232 +++++++++++++++++++++++++++-------------- core/predefined_mci.js | 2 +- package.json | 3 +- util/exiftool2desc.js | 17 ++- 5 files changed, 176 insertions(+), 83 deletions(-) diff --git a/core/config.js b/core/config.js index 148c9aa0..04729903 100644 --- a/core/config.js +++ b/core/config.js @@ -280,7 +280,10 @@ function getDefaultConfig() { cmd : 'exiftool', args : [ '-charset', 'utf8', '{filePath}', - '--directory', '--filepermissions', '--exiftoolversion', '--filename', '--filesize', '--filemodifydate', '--fileaccessdate', '--fileinodechangedate' + // exclude the following: + '--directory', '--filepermissions', '--exiftoolversion', '--filename', '--filesize', + '--filemodifydate', '--fileaccessdate', '--fileinodechangedate', '--createdate', '--modifydate', + '--metadatadate', '--xmptoolkit' ] } }, diff --git a/core/file_base_area.js b/core/file_base_area.js index 7fb123d4..6abf5844 100644 --- a/core/file_base_area.js +++ b/core/file_base_area.js @@ -22,7 +22,7 @@ const crypto = require('crypto'); const paths = require('path'); const temptmp = require('temptmp').createTrackedSession('file_area'); const iconv = require('iconv-lite'); -const exec = require('child_process').exec; +const execFile = require('child_process').execFile; const moment = require('moment'); exports.isInternalArea = isInternalArea; @@ -262,48 +262,16 @@ function logDebug(obj, msg) { } } -function populateFileEntryWithArchive(fileEntry, filePath, stepInfo, iterator, cb) { - const archiveUtil = ArchiveUtil.getInstance(); - const archiveType = fileEntry.meta.archive_type; // we set this previous to populateFileEntryWithArchive() - +function extractAndProcessDescFiles(fileEntry, filePath, archiveEntries, cb) { async.waterfall( [ - function getArchiveFileList(callback) { - stepInfo.step = 'archive_list_start'; - - iterator(err => { - if(err) { - return callback(err); - } - - archiveUtil.listEntries(filePath, archiveType, (err, entries) => { - if(err) { - stepInfo.step = 'archive_list_failed'; - } else { - stepInfo.step = 'archive_list_finish'; - stepInfo.archiveEntries = entries || []; - } - - iterator(iterErr => { - return callback( iterErr, entries || [] ); // ignore original |err| here - }); - }); - }); - }, - function processDescFilesStart(entries, callback) { - stepInfo.step = 'desc_files_start'; - iterator(err => { - return callback(err, entries); - }); - }, - function extractDescFiles(entries, callback) { - + function extractDescFiles(callback) { // :TODO: would be nice if these RegExp's were cached // :TODO: this is long winded... const extractList = []; - const shortDescFile = entries.find( e => { + const shortDescFile = archiveEntries.find( e => { return Config.fileBase.fileNamePatterns.desc.find( pat => new RegExp(pat, 'i').test(e.fileName) ); }); @@ -311,7 +279,7 @@ function populateFileEntryWithArchive(fileEntry, filePath, stepInfo, iterator, c extractList.push(shortDescFile.fileName); } - const longDescFile = entries.find( e => { + const longDescFile = archiveEntries.find( e => { return Config.fileBase.fileNamePatterns.descLong.find( pat => new RegExp(pat, 'i').test(e.fileName) ); }); @@ -328,7 +296,8 @@ function populateFileEntryWithArchive(fileEntry, filePath, stepInfo, iterator, c return callback(err); } - archiveUtil.extractTo(filePath, tempDir, archiveType, extractList, err => { + const archiveUtil = ArchiveUtil.getInstance(); + archiveUtil.extractTo(filePath, tempDir, fileEntry.meta.archive_type, extractList, err => { if(err) { return callback(err); } @@ -384,6 +353,101 @@ function populateFileEntryWithArchive(fileEntry, filePath, stepInfo, iterator, c return callback(null); }); }, + ], + err => { + return cb(err); + } + ); +} + +function extractAndProcessSingleArchiveEntry(fileEntry, filePath, archiveEntries, cb) { + + async.waterfall( + [ + function extractToTemp(callback) { + // :TODO: we may want to skip this if the compressed file is too large... + temptmp.mkdir( { prefix : 'enigextract-' }, (err, tempDir) => { + if(err) { + return callback(err); + } + + const archiveUtil = ArchiveUtil.getInstance(); + + // ensure we only extract one - there should only be one anyway -- we also just need the fileName + const extractList = archiveEntries.slice(0, 1).map(entry => entry.fileName); + + archiveUtil.extractTo(filePath, tempDir, fileEntry.meta.archive_type, extractList, err => { + if(err) { + return callback(err); + } + + return callback(null, paths.join(tempDir, extractList[0])); + }); + }); + }, + function processSingleExtractedFile(extractedFile, callback) { + populateFileEntryInfoFromFile(fileEntry, extractedFile, err => { + if(!fileEntry.desc) { + fileEntry.desc = getDescFromFileName(filePath); + } + return callback(err); + }); + } + ], + err => { + return cb(err); + } + ); +} + +function populateFileEntryWithArchive(fileEntry, filePath, stepInfo, iterator, cb) { + const archiveUtil = ArchiveUtil.getInstance(); + const archiveType = fileEntry.meta.archive_type; // we set this previous to populateFileEntryWithArchive() + + async.waterfall( + [ + function getArchiveFileList(callback) { + stepInfo.step = 'archive_list_start'; + + iterator(err => { + if(err) { + return callback(err); + } + + archiveUtil.listEntries(filePath, archiveType, (err, entries) => { + if(err) { + stepInfo.step = 'archive_list_failed'; + } else { + stepInfo.step = 'archive_list_finish'; + stepInfo.archiveEntries = entries || []; + } + + iterator(iterErr => { + return callback( iterErr, entries || [] ); // ignore original |err| here + }); + }); + }); + }, + function processDescFilesStart(entries, callback) { + stepInfo.step = 'desc_files_start'; + iterator(err => { + return callback(err, entries); + }); + }, + function extractDescFromArchive(entries, callback) { + // + // If we have a -single- entry in the archive, extract that file + // and try retrieving info in the non-archive manor. This should + // work for things like zipped up .pdf files. + // + // Otherwise, try to find particular desc files such as FILE_ID.DIZ + // and README.1ST + // + const archDescHandler = (1 === entries.length) ? extractAndProcessSingleArchiveEntry : extractAndProcessDescFiles; + archDescHandler(fileEntry, filePath, entries, err => { + return callback(err); + }); + }, function attemptReleaseYearEstimation(callback) { attemptSetEstimatedReleaseDate(fileEntry); return callback(null); @@ -413,6 +477,53 @@ function getInfoExtractUtilForDesc(mimeType, descType) { return util; } +function populateFileEntryInfoFromFile(fileEntry, filePath, cb) { + const mimeType = resolveMimeType(filePath); + if(!mimeType) { + return cb(null); + } + + async.eachSeries( [ 'short', 'long' ], (descType, nextDesc) => { + const util = getInfoExtractUtilForDesc(mimeType, descType); + if(!util) { + return nextDesc(null); + } + + const args = (util.args || [ '{filePath}'] ).map( arg => stringFormat(arg, { filePath : filePath } ) ); + + execFile(util.cmd, args, { timeout : 1000 * 30 }, (err, stdout) => { + if(err || !stdout) { + const reason = err ? err.message : 'No description produced'; + logDebug( + { reason : reason, cmd : util.cmd, args : args }, + `${_.upperFirst(descType)} description command failed` + ); + } else { + stdout = (stdout || '').trim(); + if(stdout.length > 0) { + const key = 'short' === descType ? 'desc' : 'descLong'; + if('desc' === key) { + // + // Word wrap short descriptions to FILE_ID.DIZ spec + // + // "...no more than 45 characters long" + // + // See http://www.textfiles.com/computers/fileid.txt + // + stdout = (wordWrapText( stdout, { width : 45 } ).wrapped || []).join('\n'); + } + + fileEntry[key] = stdout; + } + } + + return nextDesc(null); + }); + }, () => { + return cb(null); + }); +} + function populateFileEntryNonArchive(fileEntry, filePath, stepInfo, iterator, cb) { async.series( @@ -422,48 +533,11 @@ function populateFileEntryNonArchive(fileEntry, filePath, stepInfo, iterator, cb return iterator(callback); }, function getDescriptions(callback) { - const mimeType = resolveMimeType(filePath); - if(!mimeType) { - return callback(null); - } - - async.eachSeries( [ 'short', 'long' ], (descType, nextDesc) => { - const util = getInfoExtractUtilForDesc(mimeType, descType); - if(!util) { - return nextDesc(null); + populateFileEntryInfoFromFile(fileEntry, filePath, err => { + if(!fileEntry.desc) { + fileEntry.desc = getDescFromFileName(filePath); } - - const args = (util.args || [ '{filePath} '] ).map( arg => stringFormat(arg, { filePath : filePath } ) ); - - exec(`${util.cmd} ${args.join(' ')}`, (err, stdout) => { - if(err) { - logDebug( - { error : err.message, cmd : util.cmd, args : args }, - `${_.upperFirst(descType)} description command failed` - ); - } else { - stdout = (stdout || '').trim(); - if(stdout.length > 0) { - const key = 'short' === descType ? 'desc' : 'descLong'; - if('desc' === key) { - // - // Word wrap short descriptions to FILE_ID.DIZ spec - // - // "...no more than 45 characters long" - // - // See http://www.textfiles.com/computers/fileid.txt - // - stdout = (wordWrapText( stdout, { width : 45 } ).wrapped || []).join('\n'); - } - - fileEntry[key] = stdout; - } - } - - return nextDesc(null); - }); - }, () => { - return callback(null); + return callback(err); }); }, function processDescFilesFinish(callback) { diff --git a/core/predefined_mci.js b/core/predefined_mci.js index 68998044..34cb6a5c 100644 --- a/core/predefined_mci.js +++ b/core/predefined_mci.js @@ -95,7 +95,7 @@ const PREDEFINED_MCI_GENERATORS = { const byteSize = StatLog.getUserStatNum(client.user, 'dl_total_bytes'); return formatByteSize(byteSize, true); // true=withAbbr }, - UP : function userNumUploadsclient(client) { return userStatAsString(client, 'ul_total_count', 0); }, // Obv/2 + UP : function userNumUploads(client) { return userStatAsString(client, 'ul_total_count', 0); }, // Obv/2 UK : function userByteUpload(client) { // Obv/2 uses UK=uploaded Kbytes const byteSize = StatLog.getUserStatNum(client.user, 'ul_total_bytes'); return formatByteSize(byteSize, true); // true=withAbbr diff --git a/package.json b/package.json index 47ab69d5..2a432e33 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "uuid": "^3.0.1", "uuid-parse": "^1.0.0", "ws" : "^2.3.1", - "graceful-fs" : "^4.1.11" + "graceful-fs" : "^4.1.11", + "exiftool" : "^0.0.3" }, "devDependencies": {}, "engines": { diff --git a/util/exiftool2desc.js b/util/exiftool2desc.js index 5c828718..d94f5d75 100755 --- a/util/exiftool2desc.js +++ b/util/exiftool2desc.js @@ -19,6 +19,11 @@ const FILETYPE_HANDLERS = {}; [ 'PNG', 'JPEG', 'GIF', 'WEBP', 'XCF' ].forEach(ext => FILETYPE_HANDLERS[ext] = imageFile); function audioFile(metadata) { + // nothing if we don't know at least the author or title + if(!metadata.author && !metadata.title) { + return; + } + let desc = `${metadata.artist||'Unknown Artist'} - ${metadata.title||'Unknown'} (`; if(metadata.year) { desc += `${metadata.year}, `; @@ -28,6 +33,11 @@ function audioFile(metadata) { } function documentFile(metadata) { + // nothing if we don't know at least the author or title + if(!metadata.author && !metadata.title) { + return; + } + let desc = `${metadata.author||'Unknown Author'} - ${metadata.title||'Unknown'}`; const created = moment(metadata.createdate); if(created.isValid()) { @@ -86,7 +96,12 @@ function main() { return -1; } - console.info(handler(metadata)); + const info = handler(metadata); + if(!info) { + return -1; + } + + console.info(info); return 0; }); }); From 1c92b349cd919d69b889db8ffc05c124adf39f3b Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 22 May 2017 21:35:06 -0600 Subject: [PATCH 2/5] Clean up oputil help a bit Minor code tidy + notes --- core/file_base_area.js | 1 + core/oputil/oputil_file_base.js | 22 +++++++---- core/oputil/oputil_help.js | 66 ++++++++++++++++----------------- 3 files changed, 48 insertions(+), 41 deletions(-) diff --git a/core/file_base_area.js b/core/file_base_area.js index 6abf5844..79b91ce8 100644 --- a/core/file_base_area.js +++ b/core/file_base_area.js @@ -234,6 +234,7 @@ function attemptSetEstimatedReleaseDate(fileEntry) { // const maxYear = moment().add(2, 'year').year(); const match = getMatch(fileEntry.desc) || getMatch(fileEntry.descLong); + if(match && match[1]) { let year; if(2 === match[1].length) { diff --git a/core/oputil/oputil_file_base.js b/core/oputil/oputil_file_base.js index b18577d0..b67b76e2 100644 --- a/core/oputil/oputil_file_base.js +++ b/core/oputil/oputil_file_base.js @@ -185,18 +185,24 @@ function getSpecificFileEntry(pattern, cb) { const fileEntry = new FileEntry(); fileEntry.load(fileId, () => { - return callback(null, fileEntry); // try sha + return callback(null, fileEntry); // try SHA }); }, function getBySha(fileEntry, callback) { if(fileEntry) { - return callback(null, fileEntry); // already got it by sha + return callback(null, fileEntry); // already got it by SHA } FileEntry.findFileBySha(pattern, (err, fileEntry) => { - return callback(err, fileEntry); + return callback(null, fileEntry); // try by PATH }); - }, + }/*, + function getByPath(fileEntry, callback) { + if(fileEntry) { + return callback(null, fileEntry); // already got by FILE_ID|SHA + } + } + */ ], (err, fileEntry) => { return cb(err, fileEntry); @@ -367,12 +373,11 @@ function moveFiles() { // Each SRC may be PATH|FILE_ID|SHA|AREA_TAG[@STORAGE_TAG] FileEntry = require('../../core/file_entry.js'); - async.eachSeries(src, (areaAndStorage, next) => { - // - // If this entry represents a area tag, it means *all files* in that area - // + async.eachSeries(src, (areaAndStorage, next) => { const areaInfo = fileArea.getFileAreaByTag(areaAndStorage.areaTag); + if(areaInfo) { + // AREA_TAG[@STORAGE_TAG] - all files in area@tag src.areaInfo = areaInfo; const findFilter = { @@ -404,6 +409,7 @@ function moveFiles() { } else { // PATH|FILE_ID|SHA|PARTIAL_SHA + // :TODO: Implement by FILE|PATH support: find first path|file getSpecificFileEntry(areaAndStorage.pattern, (err, fileEntry) => { if(err) { return next(err); diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index 4ee6d7ac..ebbc970e 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -12,60 +12,60 @@ const usageHelp = exports.USAGE_HELP = { [] global args: - -c, --config PATH : specify config path (${getDefaultConfigPath()}) - -n, --no-prompt : assume defaults/don't prompt for input where possible + -c, --config PATH specify config path (${getDefaultConfigPath()}) + -n, --no-prompt assume defaults/don't prompt for input where possible -where is one of: - user : user utilities - config : config file management - fb : file base management +commands: + user user utilities + config config file management + fb file base management `, User : `usage: optutil.js user --user USERNAME valid args: - --user USERNAME : specify username for further actions - --password PASS : set new password - --delete : delete user - --activate : activate user - --deactivate : deactivate user + --user USERNAME specify username for further actions + --password PASS set new password + --delete delete user + --activate activate user + --deactivate deactivate user `, Config : `usage: optutil.js config [] -where is one of: - new : generate a new/initial configuration - import-areas PATH : import areas using fidonet *.NA or AREAS.BBS file from PATH +actions: + new generate a new/initial configuration + import-areas PATH import areas using fidonet *.NA or AREAS.BBS file from PATH -valid import-areas : - --conf CONF_TAG : specify conference tag in which to import areas - --network NETWORK : specify network name/key to associate FTN areas - --uplinks UL1,UL2,... : specify one or more comma separated uplinks - --type TYPE : specifies area import type. valid options are "bbs" and "na" +import-areas args: + --conf CONF_TAG specify conference tag in which to import areas + --network NETWORK specify network name/key to associate FTN areas + --uplinks UL1,UL2,... specify one or more comma separated uplinks + --type TYPE specifies area import type. valid options are "bbs" and "na" `, FileBase : `usage: oputil.js fb [] [] -where is one of: - scan AREA_TAG : scan specified areas - AREA_TAG may be suffixed with @STORAGE_TAG; for example: retro@bbs +actions: + scan AREA_TAG scan specified areas + AREA_TAG may be suffixed with @STORAGE_TAG; for example: retro@bbs - info AREA_TAG|SHA|FILE_ID : display information about areas and/or files - SHA may be a full or partial SHA-256 + info AREA_TAG|SHA|FILE_ID display information about areas and/or files + SHA may be a full or partial SHA-256 - move SRC DST : move entry(s) from SRC to DST where: - SRC may be FILE_ID|SHA|AREA_TAG - DST may be AREA_TAG, optionally suffixed with @STORAGE_TAG; for example: retro@bbs - SHA may be a full or partial SHA-256 - multiple instances of SRC may exist: SRC1 SRC2 ... + move SRC DST move entry(s) from SRC to DST where: + SRC may be FILE_ID|SHA|AREA_TAG + DST may be AREA_TAG, optionally suffixed with @STORAGE_TAG; for example: retro@bbs + SHA may be a full or partial SHA-256 + multiple instances of SRC may exist: SRC1 SRC2 ... -valid scan : - --tags TAG1,TAG2,... : specify tag(s) to assign to discovered entries +scan args: + --tags TAG1,TAG2,... specify tag(s) to assign to discovered entries -valid info : - --show-desc : display short description, if any +info args: + --show-desc display short description, if any ` }; From 3a41a6b2e11693aaced21d9af3b498481bba6998 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 23 May 2017 21:55:22 -0600 Subject: [PATCH 3/5] fb move FILENAME_WC ... DST support: Allow moving entries via their filenames inc. wildcard support --- core/file_entry.js | 35 ++++++++++++++ core/oputil/oputil_file_base.js | 84 ++++++++++++++++++++------------- core/oputil/oputil_help.js | 29 ++++++++---- 3 files changed, 106 insertions(+), 42 deletions(-) diff --git a/core/file_entry.js b/core/file_entry.js index 49e4b8a3..06fb6fe1 100644 --- a/core/file_entry.js +++ b/core/file_entry.js @@ -353,6 +353,41 @@ module.exports = class FileEntry { ); } + static findByFileNameWildcard(wc, cb) { + // convert any * -> % and ? -> _ for SQLite syntax - see https://www.sqlite.org/lang_expr.html + wc = wc.replace(/\*/g, '%').replace(/\?/g, '_'); + + fileDb.all( + `SELECT file_id + FROM file + WHERE file_name LIKE "${wc}" + `, + (err, fileIdRows) => { + if(err) { + return cb(err); + } + + if(!fileIdRows || 0 === fileIdRows.length) { + return cb(Errors.DoesNotExist('No matches')); + } + + const entries = []; + async.each(fileIdRows, (row, nextRow) => { + const fileEntry = new FileEntry(); + fileEntry.load(row.file_id, err => { + if(!err) { + entries.push(fileEntry); + } + return nextRow(err); + }); + }, + err => { + return cb(err, entries); + }); + } + ); + } + static findFiles(filter, cb) { filter = filter || {}; diff --git a/core/oputil/oputil_file_base.js b/core/oputil/oputil_file_base.js index b67b76e2..9b376e7c 100644 --- a/core/oputil/oputil_file_base.js +++ b/core/oputil/oputil_file_base.js @@ -171,41 +171,42 @@ function dumpAreaInfo(areaInfo, areaAndStorageInfo, cb) { return cb(null); } -function getSpecificFileEntry(pattern, cb) { - // spec: FILE_ID|SHA|PARTIAL_SHA +function getFileEntries(pattern, cb) { + // spec: FILENAME_WC|FILE_ID|SHA|PARTIAL_SHA const FileEntry = require('../../core/file_entry.js'); async.waterfall( [ - function getByFileId(callback) { + function tryByFileId(callback) { const fileId = parseInt(pattern); if(!/^[0-9]+$/.test(pattern) || isNaN(fileId)) { - return callback(null, null); + return callback(null, null); // try SHA } const fileEntry = new FileEntry(); - fileEntry.load(fileId, () => { - return callback(null, fileEntry); // try SHA + fileEntry.load(fileId, err => { + return callback(null, err ? null : [ fileEntry ] ); }); }, - function getBySha(fileEntry, callback) { - if(fileEntry) { - return callback(null, fileEntry); // already got it by SHA + function tryByShaOrPartialSha(entries, callback) { + if(entries) { + return callback(null, entries); // already got it by FILE_ID } FileEntry.findFileBySha(pattern, (err, fileEntry) => { - return callback(null, fileEntry); // try by PATH + return callback(null, fileEntry ? [ fileEntry ] : null ); }); - }/*, - function getByPath(fileEntry, callback) { - if(fileEntry) { - return callback(null, fileEntry); // already got by FILE_ID|SHA - } + }, + function tryByFileNameWildcard(entries, callback) { + if(entries) { + return callback(null, entries); // already got by FILE_ID|SHA + } + + return FileEntry.findByFileNameWildcard(pattern, callback); } - */ ], - (err, fileEntry) => { - return cb(err, fileEntry); + (err, entries) => { + return cb(err, entries); } ); } @@ -214,8 +215,12 @@ function dumpFileInfo(shaOrFileId, cb) { async.waterfall( [ function getEntry(callback) { - getSpecificFileEntry(shaOrFileId, (err, fileEntry) => { - return callback(err, fileEntry); + getFileEntries(shaOrFileId, (err, entries) => { + if(err) { + return callback(err); + } + + return callback(null, entries[0]); }); }, function dumpInfo(fileEntry, callback) { @@ -338,7 +343,7 @@ function moveFiles() { // // oputil fb move SRC [SRC2 ...] DST // - // SRC: PATH|FILE_ID|SHA|AREA_TAG[@STORAGE_TAG] + // SRC: FILENAME_WC|FILE_ID|SHA|AREA_TAG[@STORAGE_TAG] // DST: AREA_TAG[@STORAGE_TAG] // if(argv._.length < 4) { @@ -408,13 +413,14 @@ function moveFiles() { }); } else { - // PATH|FILE_ID|SHA|PARTIAL_SHA - // :TODO: Implement by FILE|PATH support: find first path|file - getSpecificFileEntry(areaAndStorage.pattern, (err, fileEntry) => { + // FILENAME_WC|FILE_ID|SHA|PARTIAL_SHA + // :TODO: FULL_PATH -> entries + getFileEntries(areaAndStorage.pattern, (err, entries) => { if(err) { return next(err); } - srcEntries.push(fileEntry); + + srcEntries = srcEntries.concat(entries); return next(null); }); } @@ -454,18 +460,30 @@ function moveFiles() { ); } +function removeFiles() { + // + // REMOVE SHA|FILE_ID [SHA|FILE_ID ...] +} + function handleFileBaseCommand() { + + function errUsage() { + return printUsageAndSetExitCode( + getHelpFor('FileBase') + getHelpFor('FileOpsInfo'), + ExitCodes.ERROR + ); + } + if(true === argv.help) { - return printUsageAndSetExitCode(getHelpFor('FileBase'), ExitCodes.ERROR); + return errUsage(); } const action = argv._[1]; - switch(action) { - case 'info' : return displayFileAreaInfo(); - case 'scan' : return scanFileAreas(); - case 'move' : return moveFiles(); - - default : return printUsageAndSetExitCode(getHelpFor('FileBase'), ExitCodes.ERROR); - } + return ({ + info : displayFileAreaInfo, + scan : scanFileAreas, + move : moveFiles, + remove : removeFiles, + }[action] || errUsage)(); } \ No newline at end of file diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index ebbc970e..0344145b 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -9,7 +9,7 @@ exports.getHelpFor = getHelpFor; const usageHelp = exports.USAGE_HELP = { General : `usage: optutil.js [--version] [--help] - [] + [] global args: -c, --config PATH specify config path (${getDefaultConfigPath()}) @@ -19,7 +19,6 @@ commands: user user utilities config config file management fb file base management - `, User : `usage: optutil.js user --user USERNAME @@ -49,23 +48,35 @@ import-areas args: `usage: oputil.js fb [] [] actions: - scan AREA_TAG scan specified areas - AREA_TAG may be suffixed with @STORAGE_TAG; for example: retro@bbs + scan AREA_TAG[@STORAGE_TAG] scan specified area info AREA_TAG|SHA|FILE_ID display information about areas and/or files SHA may be a full or partial SHA-256 - move SRC DST move entry(s) from SRC to DST where: - SRC may be FILE_ID|SHA|AREA_TAG - DST may be AREA_TAG, optionally suffixed with @STORAGE_TAG; for example: retro@bbs - SHA may be a full or partial SHA-256 - multiple instances of SRC may exist: SRC1 SRC2 ... + move SRC [SRC...]] DST move entry(s) from SRC to DST + * SRC: FILENAME_WC|SHA|FILE_ID|AREA_TAG[@STORAGE_TAG] + * DST: AREA_TAG[@STORAGE_TAG] + + remove SHA|FILE_ID removes a entry from the system scan args: --tags TAG1,TAG2,... specify tag(s) to assign to discovered entries info args: --show-desc display short description, if any + +remove args: + --delete also remove underlying physical file +`, + FileOpsInfo : +` +general information: + AREA_TAG[@STORAGE_TAG] can specify an area tag and optionally, a storage specific tag + example: retro@bbs + + FILENAME_WC filename with * and ? wildcard support. may match 0:n entries + SHA full or partial SHA-256 + FILE_ID a file identifier. see file.sqlite3 ` }; From 2e18833014ac3e8562829882e850b137b2f28db6 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Fri, 26 May 2017 08:25:41 -0600 Subject: [PATCH 4/5] Working WebSocket connections - not yet complete, but working well --- core/login_server_module.js | 2 +- core/servers/login/telnet.js | 50 +-------- core/servers/login/websocket.js | 173 ++++++++++++++++++++++++++++++++ package.json | 2 +- 4 files changed, 177 insertions(+), 50 deletions(-) create mode 100644 core/servers/login/websocket.js diff --git a/core/login_server_module.js b/core/login_server_module.js index 4f003982..212d2e27 100644 --- a/core/login_server_module.js +++ b/core/login_server_module.js @@ -43,7 +43,7 @@ module.exports = class LoginServerModule extends ServerModule { } client.session.serverName = modInfo.name; - client.session.isSecure = modInfo.isSecure || false; + client.session.isSecure = _.isBoolean(client.isSecure) ? client.isSecure : (modInfo.isSecure || false); clientConns.addNewClient(client, clientSock); diff --git a/core/servers/login/telnet.js b/core/servers/login/telnet.js index e340d512..5c471473 100644 --- a/core/servers/login/telnet.js +++ b/core/servers/login/telnet.js @@ -24,6 +24,8 @@ const ModuleInfo = exports.moduleInfo = { packageName : 'codes.l33t.enigma.telnet.server', }; +exports.TelnetClient = TelnetClient; + // // Telnet Protocol Resources // * http://pcmicro.com/netfoss/telnet.html @@ -498,54 +500,6 @@ function TelnetClient(input, output) { this.input.on('data', this.dataHandler); - /* - this.input.on('data', b => { - bufs.push(b); - - let i; - while((i = bufs.indexOf(IAC_BUF)) >= 0) { - - // - // Some clients will send even IAC separate from data - // - if(bufs.length <= (i + 1)) { - i = MORE_DATA_REQUIRED; - break; - } - - assert(bufs.length > (i + 1)); - - if(i > 0) { - self.emit('data', bufs.splice(0, i).toBuffer()); - } - - i = parseBufs(bufs); - - if(MORE_DATA_REQUIRED === i) { - break; - } else { - if(i.option) { - self.emit(i.option, i); // "transmit binary", "echo", ... - } - - self.handleTelnetEvent(i); - - if(i.data) { - self.emit('data', i.data); - } - } - } - - if(MORE_DATA_REQUIRED !== i && bufs.length > 0) { - // - // Standard data payload. This can still be "non-user" data - // such as ANSI control, but we don't handle that here. - // - self.emit('data', bufs.splice(0).toBuffer()); - } - }); - */ - this.input.on('end', () => { self.emit('end'); }); diff --git a/core/servers/login/websocket.js b/core/servers/login/websocket.js new file mode 100644 index 00000000..4b73bcd3 --- /dev/null +++ b/core/servers/login/websocket.js @@ -0,0 +1,173 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +const Config = require('../../config.js').config; +const TelnetClient = require('./telnet.js').TelnetClient; +const Log = require('../../logger.js').log; +const LoginServerModule = require('../../login_server_module.js'); + +// deps +const _ = require('lodash'); +const WebSocketServer = require('ws').Server; +const http = require('http'); +const https = require('https'); +const fs = require('graceful-fs'); +const EventEmitter = require('events'); + +const ModuleInfo = exports.moduleInfo = { + name : 'WebSocket', + desc : 'WebSocket Server', + author : 'NuSkooler', + packageName : 'codes.l33t.enigma.websocket.server', +}; + +function WebSocketClient(ws, req, serverType) { + + Object.defineProperty(this, 'isSecure', { + get : () => 'secure' === serverType ? true : false, + }); + + this.socketBridge = new class SocketBridge extends EventEmitter { + constructor(ws) { + super(); + this.ws = ws; + } + + end() { + return ws.terminate(); + } + + write(data, cb) { + return this.ws.send(data, { binary : true }, cb); + } + + get remoteAddress() { + return req.connection.remoteAddress; + } + }(ws); + + ws.on('message', data => { + this.socketBridge.emit('data', data); + }); + + ws.on('close', () => { + this.end(); + }); + + // + // Montior connection status with ping/pong + // + ws.on('pong', () => { + Log.trace(`Pong from ${this.socketBridge.remoteAddress}`); + ws.isConnectionAlive = true; + }); + + TelnetClient.call(this, this.socketBridge, this.socketBridge); + + // start handshake process + this.banner(); +} + +require('util').inherits(WebSocketClient, TelnetClient); + +const WSS_SERVER_TYPES = [ 'insecure', 'secure' ]; + +exports.getModule = class WebSocketLoginServer extends LoginServerModule { + constructor() { + super(); + } + + createServer() { + // + // We will actually create up to two servers: + // * insecure websocket (ws://) + // * secure (tls) websocket (wss://) + // + const insecureConf = _.get(Config, 'loginServers.webSocket') || { enabled : false }; + const secureConf = _.get(Config, 'loginServers.secureWebSocket') || { enabled : false }; + + if(insecureConf.enabled) { + const httpServer = http.createServer( (req, resp) => { + // dummy handler + resp.writeHead(200); + return resp.end('ENiGMA½ BBS WebSocket Server!'); + }); + + this.insecure = { + httpServer : httpServer, + wsServer : new WebSocketServer( { server : httpServer } ), + }; + } + + if(secureConf.enabled) { + const httpServer = https.createServer({ + key : fs.readFileSync(Config.loginServers.secureWebSocket.keyPem), + cert : fs.readFileSync(Config.loginServers.secureWebSocket.certPem), + }); + + this.secure = { + httpServer : httpServer, + wsServer : new WebSocketServer( { server : httpServer } ), + }; + } + } + + configName(serverType) { + return 'secure' === serverType ? 'secureWebSocket' : 'webSocket'; + } + + listen() { + WSS_SERVER_TYPES.forEach(serverType => { + const server = this[serverType]; + if(!server) { + return; + } + + const serverName = `${ModuleInfo.name} (${serverType})`; + const port = parseInt( _.get( Config, [ 'loginServers', this.configName(serverType), 'port' ] ) ); + + if(isNaN(port)) { + Log.error( { server : serverName, port : port }, 'Cannot load server (invalid port)' ); + return; + } + + server.httpServer.listen(port); + + server.wsServer.on('connection', (ws, req) => { + const webSocketClient = new WebSocketClient(ws, req, serverType); + this.handleNewClient(webSocketClient, webSocketClient.socketBridge, ModuleInfo); + }); + + Log.info( { server : serverName, port : port }, 'Listening for connections' ); + }); + + // + // Send pings every 30s + // + setInterval( () => { + WSS_SERVER_TYPES.forEach(serverType => { + if(this[serverType]) { + this[serverType].wsServer.clients.forEach(ws => { + if(false === ws.isConnectionAlive) { + Log.debug('WebSocket connection seems inactive. Terminating.'); + return ws.terminate(); + } + + ws.isConnectionAlive = false; // pong will reset this + + Log.trace('Ping to remote WebSocket client'); + return ws.ping('', false, true); + }); + } + }); + }, 30000); + + return true; + } + + webSocketConnection(conn) { + const webSocketClient = new WebSocketClient(conn); + this.handleNewClient(webSocketClient, webSocketClient.socketShim, ModuleInfo); + } +}; diff --git a/package.json b/package.json index 2a432e33..c983edd5 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "temptmp": "^1.0.0", "uuid": "^3.0.1", "uuid-parse": "^1.0.0", - "ws" : "^2.3.1", + "ws" : "^3.0.0", "graceful-fs" : "^4.1.11", "exiftool" : "^0.0.3" }, From 74bab3d6c550fb09cf95e0e32a13707890430b93 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 30 May 2017 21:31:35 -0600 Subject: [PATCH 5/5] Better configuration for WebSockets --- core/config.js | 11 +++++------ core/servers/login/websocket.js | 24 +++++++++++++----------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/core/config.js b/core/config.js index 04729903..25e409f4 100644 --- a/core/config.js +++ b/core/config.js @@ -225,13 +225,12 @@ function getDefaultConfig() { firstMenuNewUser : 'sshConnectedNewUser', }, webSocket : { - port : 8810, - enabled : true, // :TODO: default to false - }, - secureWebSocket : { - port : 8811, + port : 8810, // ws:// enabled : false, - } + securePort : 8811, // wss:// - must provide certPem and keyPem + certPem : paths.join(__dirname, './../misc/https_cert.pem'), + keyPem : paths.join(__dirname, './../misc/https_cert_key.pem'), + }, }, contentServers : { diff --git a/core/servers/login/websocket.js b/core/servers/login/websocket.js index 4b73bcd3..3cad496e 100644 --- a/core/servers/login/websocket.js +++ b/core/servers/login/websocket.js @@ -28,6 +28,10 @@ function WebSocketClient(ws, req, serverType) { get : () => 'secure' === serverType ? true : false, }); + // + // This bridge makes accessible various calls that client sub classes + // want to access on I/O socket + // this.socketBridge = new class SocketBridge extends EventEmitter { constructor(ws) { super(); @@ -84,10 +88,12 @@ exports.getModule = class WebSocketLoginServer extends LoginServerModule { // * insecure websocket (ws://) // * secure (tls) websocket (wss://) // - const insecureConf = _.get(Config, 'loginServers.webSocket') || { enabled : false }; - const secureConf = _.get(Config, 'loginServers.secureWebSocket') || { enabled : false }; + const config = _.get(Config, 'loginServers.webSocket') || { enabled : false }; + if(!config || true !== config.enabled || !(config.port || config.securePort)) { + return; + } - if(insecureConf.enabled) { + if(config.port) { const httpServer = http.createServer( (req, resp) => { // dummy handler resp.writeHead(200); @@ -100,10 +106,10 @@ exports.getModule = class WebSocketLoginServer extends LoginServerModule { }; } - if(secureConf.enabled) { + if(config.securePort) { const httpServer = https.createServer({ - key : fs.readFileSync(Config.loginServers.secureWebSocket.keyPem), - cert : fs.readFileSync(Config.loginServers.secureWebSocket.certPem), + key : fs.readFileSync(Config.loginServers.webSocket.keyPem), + cert : fs.readFileSync(Config.loginServers.webSocket.certPem), }); this.secure = { @@ -113,10 +119,6 @@ exports.getModule = class WebSocketLoginServer extends LoginServerModule { } } - configName(serverType) { - return 'secure' === serverType ? 'secureWebSocket' : 'webSocket'; - } - listen() { WSS_SERVER_TYPES.forEach(serverType => { const server = this[serverType]; @@ -125,7 +127,7 @@ exports.getModule = class WebSocketLoginServer extends LoginServerModule { } const serverName = `${ModuleInfo.name} (${serverType})`; - const port = parseInt( _.get( Config, [ 'loginServers', this.configName(serverType), 'port' ] ) ); + const port = parseInt(_.get(Config, [ 'loginServers', 'webSocket', 'secure' === serverType ? 'securePort' : 'port' ] )); if(isNaN(port)) { Log.error( { server : serverName, port : port }, 'Cannot load server (invalid port)' );