diff options
author | Lauri Ojansivu <x@xet7.org> | 2020-05-24 03:13:53 +0300 |
---|---|---|
committer | Lauri Ojansivu <x@xet7.org> | 2020-05-24 03:13:53 +0300 |
commit | 055b5285328f495203a7e225fdecd04bcf5c8b32 (patch) | |
tree | 361621362801c5409a08d60f45ebedb9f1cf9c02 /models | |
parent | fda392e662a842aa7e686ce13870c8adb35c35ab (diff) | |
parent | 921460db4031134db863e32101c0ad60a17416b5 (diff) | |
download | wekan-055b5285328f495203a7e225fdecd04bcf5c8b32.tar.gz wekan-055b5285328f495203a7e225fdecd04bcf5c8b32.tar.bz2 wekan-055b5285328f495203a7e225fdecd04bcf5c8b32.zip |
Merge branch 'lib-change' of https://github.com/PDIS/wekan into
PDIS-lib-change
Diffstat (limited to 'models')
-rw-r--r-- | models/activities.js | 2 | ||||
-rw-r--r-- | models/attachments.js | 342 | ||||
-rw-r--r-- | models/cards.js | 21 | ||||
-rw-r--r-- | models/export.js | 240 | ||||
-rw-r--r-- | models/trelloCreator.js | 1 | ||||
-rw-r--r-- | models/wekanCreator.js | 2 |
6 files changed, 313 insertions, 295 deletions
diff --git a/models/activities.js b/models/activities.js index b5fcb7d8..2663dd29 100644 --- a/models/activities.js +++ b/models/activities.js @@ -217,7 +217,7 @@ if (Meteor.isServer) { } if (activity.attachmentId) { const attachment = activity.attachment(); - params.attachment = attachment.original.name; + params.attachment = attachment.name; params.attachmentId = attachment._id; } if (activity.checklistId) { diff --git a/models/attachments.js b/models/attachments.js index 3fe1d745..03999f55 100644 --- a/models/attachments.js +++ b/models/attachments.js @@ -1,263 +1,127 @@ -const localFSStore = process.env.ATTACHMENTS_STORE_PATH; -const storeName = 'attachments'; -const defaultStoreOptions = { - beforeWrite: fileObj => { - if (!fileObj.isImage()) { - return { - type: 'application/octet-stream', - }; - } - return {}; - }, -}; -let store; -if (localFSStore) { - // have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem - const fs = Npm.require('fs'); - const path = Npm.require('path'); - const mongodb = Npm.require('mongodb'); - const Grid = Npm.require('gridfs-stream'); - // calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :( - let pathname = localFSStore; - /*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */ - - if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) { - pathname = path.join( - __meteor_bootstrap__.serverDir, - `../../../cfs/files/${storeName}`, - ); - } +import { FilesCollection } from 'meteor/ostrio:files'; +const fs = require('fs'); + +const collectionName = 'attachments2'; + +Attachments = new FilesCollection({ + storagePath: storagePath(), + debug: false, +// allowClientCode: true, + collectionName: 'attachments2', + onAfterUpload: onAttachmentUploaded, + onBeforeRemove: onAttachmentRemoving +}); - if (!pathname) - throw new Error('FS.Store.FileSystem unable to determine path'); +if (Meteor.isServer) { + Meteor.startup(() => { + Attachments.collection._ensureIndex({ cardId: 1 }); + }); - // Check if we have '~/foo/bar' - if (pathname.split(path.sep)[0] === '~') { - const homepath = - process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE; - if (homepath) { - pathname = pathname.replace('~', homepath); - } else { - throw new Error('FS.Store.FileSystem unable to resolve "~" in path'); + // TODO: Permission related + Attachments.allow({ + insert() { + return false; + }, + update() { + return true; + }, + remove() { + return true; } - } - - // Set absolute path - const absolutePath = path.resolve(pathname); - - const _FStore = new FS.Store.FileSystem(storeName, { - path: localFSStore, - ...defaultStoreOptions, }); - const GStore = { - fileKey(fileObj) { - const key = { - _id: null, - filename: null, - }; - // If we're passed a fileObj, we retrieve the _id and filename from it. - if (fileObj) { - const info = fileObj._getInfo(storeName, { - updateFileRecordFirst: false, - }); - key._id = info.key || null; - key.filename = - info.name || - fileObj.name({ updateFileRecordFirst: false }) || - `${fileObj.collectionName}-${fileObj._id}`; + Meteor.methods({ + cloneAttachment(file, overrides) { + check(file, Object); + check(overrides, Match.Maybe(Object)); + const path = file.path; + const opts = { + fileName: file.name, + type: file.type, + meta: file.meta, + userId: file.userId + }; + for (let key in overrides) { + if (key === 'meta') { + for (let metaKey in overrides.meta) { + opts.meta[metaKey] = overrides.meta[metaKey]; + } + } else { + opts[key] = overrides[key]; + } } - - // If key._id is null at this point, createWriteStream will let GridFS generate a new ID - return key; - }, - db: undefined, - mongoOptions: { useNewUrlParser: true }, - mongoUrl: process.env.MONGO_URL, - init() { - this._init(err => { - this.inited = !err; - }); - }, - _init(callback) { - const self = this; - mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function( - err, - db, - ) { + const buffer = fs.readFileSync(path); + Attachments.write(buffer, opts, (err, fileRef) => { if (err) { - return callback(err); + console.log('Error when cloning record', err); } - self.db = db; - return callback(null); }); - return; - }, - createReadStream(fileKey, options) { - const self = this; - if (!self.inited) { - self.init(); - return undefined; - } - options = options || {}; - - // Init GridFS - const gfs = new Grid(self.db, mongodb); - - // Set the default streamning settings - const settings = { - _id: new mongodb.ObjectID(fileKey._id), - root: `cfs_gridfs.${storeName}`, - }; - - // Check if this should be a partial read - if ( - typeof options.start !== 'undefined' && - typeof options.end !== 'undefined' - ) { - // Add partial info - settings.range = { - startPos: options.start, - endPos: options.end, - }; - } - return gfs.createReadStream(settings); - }, - }; - GStore.init(); - const CRS = 'createReadStream'; - const _CRS = `_${CRS}`; - const FStore = _FStore._transform; - FStore[_CRS] = FStore[CRS].bind(FStore); - FStore[CRS] = function(fileObj, options) { - let stream; - try { - const localFile = path.join( - absolutePath, - FStore.storage.fileKey(fileObj), - ); - const state = fs.statSync(localFile); - if (state) { - stream = FStore[_CRS](fileObj, options); - } - } catch (e) { - // file is not there, try GridFS ? - stream = undefined; + return true; } - if (stream) return stream; - else { - try { - const stream = GStore[CRS](GStore.fileKey(fileObj), options); - return stream; - } catch (e) { - return undefined; - } - } - }.bind(FStore); - store = _FStore; -} else { - store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, { - // XXX Add a new store for cover thumbnails so we don't load big images in - // the general board view - // If the uploaded document is not an image we need to enforce browser - // download instead of execution. This is particularly important for HTML - // files that the browser will just execute if we don't serve them with the - // appropriate `application/octet-stream` MIME header which can lead to user - // data leaks. I imagine other formats (like PDF) can also be attack vectors. - // See https://github.com/wekan/wekan/issues/99 - // XXX Should we use `beforeWrite` option of CollectionFS instead of - // collection-hooks? - // We should use `beforeWrite`. - ...defaultStoreOptions, }); -} -Attachments = new FS.Collection('attachments', { - stores: [store], -}); - -if (Meteor.isServer) { - Meteor.startup(() => { - Attachments.files._ensureIndex({ cardId: 1 }); - }); - - Attachments.allow({ - insert(userId, doc) { - return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); - }, - update(userId, doc) { - return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); - }, - remove(userId, doc) { - return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); - }, - // We authorize the attachment download either: - // - if the board is public, everyone (even unconnected) can download it - // - if the board is private, only board members can download it - download(userId, doc) { - const board = Boards.findOne(doc.boardId); - if (board.isPublic()) { - return true; - } else { - return board.hasMember(userId); - } - }, - fetch: ['boardId'], + Meteor.publish(collectionName, function() { + return Attachments.find().cursor; }); +} else { + Meteor.subscribe(collectionName); } -// XXX Enforce a schema for the Attachments CollectionFS - -if (Meteor.isServer) { - Attachments.files.after.insert((userId, doc) => { - // If the attachment doesn't have a source field - // or its source is different than import - if (!doc.source || doc.source !== 'import') { - // Add activity about adding the attachment - Activities.insert({ - userId, - type: 'card', - activityType: 'addAttachment', - attachmentId: doc._id, - // this preserves the name so that notifications can be meaningful after - // this file is removed - attachmentName: doc.original.name, - boardId: doc.boardId, - cardId: doc.cardId, - listId: doc.listId, - swimlaneId: doc.swimlaneId, - }); - } else { - // Don't add activity about adding the attachment as the activity - // be imported and delete source field - Attachments.update( - { - _id: doc._id, - }, - { - $unset: { - source: '', - }, - }, - ); - } - }); +function storagePath(defaultPath) { + const storePath = process.env.ATTACHMENTS_STORE_PATH; + return storePath ? storePath : defaultPath; +} - Attachments.files.before.remove((userId, doc) => { +function onAttachmentUploaded(fileRef) { + Attachments.update({_id:fileRef._id}, {$set: {"meta.uploading": false}}); + if (!fileRef.meta.source || fileRef.meta.source !== 'import') { + // Add activity about adding the attachment Activities.insert({ - userId, + userId: fileRef.userId, type: 'card', - activityType: 'deleteAttachment', - attachmentId: doc._id, + activityType: 'addAttachment', + attachmentId: fileRef._id, // this preserves the name so that notifications can be meaningful after - // this file is removed - attachmentName: doc.original.name, - boardId: doc.boardId, - cardId: doc.cardId, - listId: doc.listId, - swimlaneId: doc.swimlaneId, + // this file is removed + attachmentName: fileRef.name, + boardId: fileRef.meta.boardId, + cardId: fileRef.meta.cardId, + listId: fileRef.meta.listId, + swimlaneId: fileRef.meta.swimlaneId, }); + } else { + // Don't add activity about adding the attachment as the activity + // be imported and delete source field + Attachments.collection.update( + { + _id: fileRef._id, + }, + { + $unset: { + 'meta.source': '', + }, + }, + ); + } +} + +function onAttachmentRemoving(cursor) { + const file = cursor.get()[0]; + const meta = file.meta; + Activities.insert({ + userId: this.userId, + type: 'card', + activityType: 'deleteAttachment', + attachmentId: file._id, + // this preserves the name so that notifications can be meaningful after + // this file is removed + attachmentName: file.name, + boardId: meta.boardId, + cardId: meta.cardId, + listId: meta.listId, + swimlaneId: meta.swimlaneId, }); + return true; } export default Attachments; diff --git a/models/cards.js b/models/cards.js index aace2647..f4bb464c 100644 --- a/models/cards.js +++ b/models/cards.js @@ -412,10 +412,14 @@ Cards.helpers({ const _id = Cards.insert(this); // Copy attachments - oldCard.attachments().forEach(att => { - att.cardId = _id; - delete att._id; - return Attachments.insert(att); + oldCard.attachments().forEach((file) => { + Meteor.call('cloneAttachment', file, + { + meta: { + cardId: _id + } + } + ); }); // copy checklists @@ -518,14 +522,15 @@ Cards.helpers({ attachments() { if (this.isLinkedCard()) { return Attachments.find( - { cardId: this.linkedId }, + { 'meta.cardId': this.linkedId }, { sort: { uploadedAt: -1 } }, ); } else { - return Attachments.find( - { cardId: this._id }, + let ret = Attachments.find( + { 'meta.cardId': this._id }, { sort: { uploadedAt: -1 } }, ); + return ret; } }, @@ -533,7 +538,7 @@ Cards.helpers({ const cover = Attachments.findOne(this.coverId); // if we return a cover before it is fully stored, we will get errors when we try to display it // todo XXX we could return a default "upload pending" image in the meantime? - return cover && cover.url() && cover; + return cover && cover.link(); }, checklists() { diff --git a/models/export.js b/models/export.js index c3783679..1eb47e54 100644 --- a/models/export.js +++ b/models/export.js @@ -1,4 +1,3 @@ -import { Exporter } from './exporter'; /* global JsonRoutes */ if (Meteor.isServer) { // todo XXX once we have a real API in place, move that route there @@ -8,10 +7,10 @@ if (Meteor.isServer) { // on the client instead of copy/pasting the route path manually between the // client and the server. /** - * @operation exportJson + * @operation export * @tag Boards * - * @summary This route is used to export the board to a json file format. + * @summary This route is used to export the board. * * @description If user is already logged-in, pass loginToken as param * "authToken": '/api/boards/:boardId/export?authToken=:token' @@ -47,52 +46,199 @@ if (Meteor.isServer) { JsonRoutes.sendResult(res, 403); } }); +} - /** - * @operation exportCSV/TSV - * @tag Boards - * - * @summary This route is used to export the board to a CSV or TSV file format. - * - * @description If user is already logged-in, pass loginToken as param - * - * See https://blog.kayla.com.au/server-side-route-authentication-in-meteor/ - * for detailed explanations - * - * @param {string} boardId the ID of the board we are exporting - * @param {string} authToken the loginToken - * @param {string} delimiter delimiter to use while building export. Default is comma ',' - */ - Picker.route('/api/boards/:boardId/export/csv', function(params, req, res) { - const boardId = params.boardId; - let user = null; - const loginToken = params.query.authToken; - if (loginToken) { - const hashToken = Accounts._hashLoginToken(loginToken); - user = Meteor.users.findOne({ - 'services.resume.loginTokens.hashedToken': hashToken, +// exporter maybe is broken since Gridfs introduced, add fs and path + +export class Exporter { + constructor(boardId) { + this._boardId = boardId; + } + + build() { + const fs = Npm.require('fs'); + const os = Npm.require('os'); + const path = Npm.require('path'); + + const byBoard = { boardId: this._boardId }; + const byBoardNoLinked = { + boardId: this._boardId, + linkedId: { $in: ['', null] }, + }; + // we do not want to retrieve boardId in related elements + const noBoardId = { + fields: { + boardId: 0, + }, + }; + const result = { + _format: 'wekan-board-1.0.0', + }; + _.extend( + result, + Boards.findOne(this._boardId, { + fields: { + stars: 0, + }, + }), + ); + result.lists = Lists.find(byBoard, noBoardId).fetch(); + result.cards = Cards.find(byBoardNoLinked, noBoardId).fetch(); + result.swimlanes = Swimlanes.find(byBoard, noBoardId).fetch(); + result.customFields = CustomFields.find( + { boardIds: { $in: [this.boardId] } }, + { fields: { boardId: 0 } }, + ).fetch(); + result.comments = CardComments.find(byBoard, noBoardId).fetch(); + result.activities = Activities.find(byBoard, noBoardId).fetch(); + result.rules = Rules.find(byBoard, noBoardId).fetch(); + result.checklists = []; + result.checklistItems = []; + result.subtaskItems = []; + result.triggers = []; + result.actions = []; + result.cards.forEach(card => { + result.checklists.push( + ...Checklists.find({ + cardId: card._id, + }).fetch(), + ); + result.checklistItems.push( + ...ChecklistItems.find({ + cardId: card._id, + }).fetch(), + ); + result.subtaskItems.push( + ...Cards.find({ + parentId: card._id, + }).fetch(), + ); + }); + result.rules.forEach(rule => { + result.triggers.push( + ...Triggers.find( + { + _id: rule.triggerId, + }, + noBoardId, + ).fetch(), + ); + result.actions.push( + ...Actions.find( + { + _id: rule.actionId, + }, + noBoardId, + ).fetch(), + ); + }); + + // [Old] for attachments we only export IDs and absolute url to original doc + // [New] Encode attachment to base64 + + const getBase64Data = function(doc, callback) { + let buffer = Buffer.allocUnsafe(0); + buffer.fill(0); + + // callback has the form function (err, res) {} + const tmpFile = path.join( + os.tmpdir(), + `tmpexport${process.pid}${Math.random()}`, + ); + const tmpWriteable = fs.createWriteStream(tmpFile); + const readStream = fs.createReadStream(doc.path); + readStream.on('data', function(chunk) { + buffer = Buffer.concat([buffer, chunk]); }); - } else if (!Meteor.settings.public.sandstorm) { - Authentication.checkUserId(req.userId); - user = Users.findOne({ - _id: req.userId, - isAdmin: true, + + readStream.on('error', function(err) { + callback(null, null); }); - } - const exporter = new Exporter(boardId); - if (exporter.canExport(user)) { - body = params.query.delimiter - ? exporter.buildCsv(params.query.delimiter) - : exporter.buildCsv(); - res.writeHead(200, { - 'Content-Length': body[0].length, - 'Content-Type': params.query.delimiter ? 'text/csv' : 'text/tsv', + readStream.on('end', function() { + // done + fs.unlink(tmpFile, () => { + //ignored + }); + + callback(null, buffer.toString('base64')); }); - res.write(body[0]); - res.end(); - } else { - res.writeHead(403); - res.end('Permission Error'); - } - }); + readStream.pipe(tmpWriteable); + }; + const getBase64DataSync = Meteor.wrapAsync(getBase64Data); + result.attachments = Attachments.find({ 'meta.boardId': byBoard.boardId }) + .fetch() + .map(attachment => { + let filebase64 = null; + filebase64 = getBase64DataSync(attachment); + + return { + _id: attachment._id, + cardId: attachment.meta.cardId, + //url: FlowRouter.url(attachment.url()), + file: filebase64, + name: attachment.name, + type: attachment.type, + }; + }); + + // we also have to export some user data - as the other elements only + // include id but we have to be careful: + // 1- only exports users that are linked somehow to that board + // 2- do not export any sensitive information + const users = {}; + result.members.forEach(member => { + users[member.userId] = true; + }); + result.lists.forEach(list => { + users[list.userId] = true; + }); + result.cards.forEach(card => { + users[card.userId] = true; + if (card.members) { + card.members.forEach(memberId => { + users[memberId] = true; + }); + } + }); + result.comments.forEach(comment => { + users[comment.userId] = true; + }); + result.activities.forEach(activity => { + users[activity.userId] = true; + }); + result.checklists.forEach(checklist => { + users[checklist.userId] = true; + }); + const byUserIds = { + _id: { + $in: Object.getOwnPropertyNames(users), + }, + }; + // we use whitelist to be sure we do not expose inadvertently + // some secret fields that gets added to User later. + const userFields = { + fields: { + _id: 1, + username: 1, + 'profile.fullname': 1, + 'profile.initials': 1, + 'profile.avatarUrl': 1, + }, + }; + result.users = Users.find(byUserIds, userFields) + .fetch() + .map(user => { + // user avatar is stored as a relative url, we export absolute + if ((user.profile || {}).avatarUrl) { + user.profile.avatarUrl = FlowRouter.url(user.profile.avatarUrl); + } + return user; + }); + return result; + } + + canExport(user) { + const board = Boards.findOne(this._boardId); + return board && board.isVisibleBy(user); + } } diff --git a/models/trelloCreator.js b/models/trelloCreator.js index 1c5bcd93..c4be140b 100644 --- a/models/trelloCreator.js +++ b/models/trelloCreator.js @@ -369,6 +369,7 @@ export class TrelloCreator { // so we make it server only, and let UI catch up once it is done, forget about latency comp. const self = this; if (Meteor.isServer) { + // FIXME: Change to new model file.attachData(att.url, function(error) { file.boardId = boardId; file.cardId = cardId; diff --git a/models/wekanCreator.js b/models/wekanCreator.js index 9914f817..c5591a0b 100644 --- a/models/wekanCreator.js +++ b/models/wekanCreator.js @@ -415,6 +415,7 @@ export class WekanCreator { const self = this; if (Meteor.isServer) { if (att.url) { + // FIXME: Change to new file library file.attachData(att.url, function(error) { file.boardId = boardId; file.cardId = cardId; @@ -440,6 +441,7 @@ export class WekanCreator { } }); } else if (att.file) { + // FIXME: Change to new file library file.attachData( Buffer.from(att.file, 'base64'), { |