summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLauri Ojansivu <x@xet7.org>2020-05-25 17:54:51 +0300
committerLauri Ojansivu <x@xet7.org>2020-05-25 17:54:51 +0300
commitd52affe65893f17bab59bb43aa9f5afbb54993d3 (patch)
tree82b88066d4fb8efb6610ed77177d3747d34f0480
parent23dcd084a49de4e6568527d8f97ed25adb04dafd (diff)
downloadwekan-d52affe65893f17bab59bb43aa9f5afbb54993d3.tar.gz
wekan-d52affe65893f17bab59bb43aa9f5afbb54993d3.tar.bz2
wekan-d52affe65893f17bab59bb43aa9f5afbb54993d3.zip
Move In Progress ostrio-files changes to separate branch, and revert ostrio-files changes, so that:
- Export to CSV/TSV with custom fields works - Attachments are not exported to disk - It is possible to build arm64/s390x versions again. Thanks to xet7 ! Related #3110
-rw-r--r--.meteor/packages1
-rw-r--r--.meteor/versions1
-rw-r--r--client/components/activities/activities.js12
-rw-r--r--client/components/cards/attachments.jade13
-rw-r--r--client/components/cards/attachments.js103
-rw-r--r--client/components/cards/attachments.styl11
-rw-r--r--client/components/cards/minicard.jade2
-rw-r--r--client/components/cards/minicard.js3
-rwxr-xr-xclient/components/main/editor.js50
-rw-r--r--client/lib/utils.js44
-rw-r--r--models/activities.js2
-rw-r--r--models/attachments.js342
-rw-r--r--models/cards.js21
-rw-r--r--models/export.js240
-rw-r--r--models/trelloCreator.js1
-rw-r--r--models/wekanCreator.js2
-rw-r--r--sandstorm-pkgdef.capnp7
-rw-r--r--server/migrations.js45
-rw-r--r--server/old-attachments-migration.js212
-rw-r--r--server/publications/boards.js2
-rwxr-xr-xsnap-src/bin/config4
-rwxr-xr-xsnap-src/bin/mongodb-control8
22 files changed, 377 insertions, 749 deletions
diff --git a/.meteor/packages b/.meteor/packages
index b0df2be8..ba278f34 100644
--- a/.meteor/packages
+++ b/.meteor/packages
@@ -98,4 +98,3 @@ percolate:synced-cron
easylogic:summernote
cfs:filesystem
ostrio:cookies
-ostrio:files
diff --git a/.meteor/versions b/.meteor/versions
index 6b6842ad..722547de 100644
--- a/.meteor/versions
+++ b/.meteor/versions
@@ -134,7 +134,6 @@ observe-sequence@1.0.16
ongoworks:speakingurl@1.1.0
ordered-dict@1.1.0
ostrio:cookies@2.6.0
-ostrio:files@1.14.2
peerlibrary:assert@0.3.0
peerlibrary:base-component@0.16.0
peerlibrary:blaze-components@0.15.1
diff --git a/client/components/activities/activities.js b/client/components/activities/activities.js
index 4f15d421..5d356f6e 100644
--- a/client/components/activities/activities.js
+++ b/client/components/activities/activities.js
@@ -151,23 +151,21 @@ BlazeComponent.extendComponent({
},
attachmentLink() {
- const activity = this.currentData().activity;
- const attachment = activity.attachment();
- const link = attachment ? attachment.link('original', '/') : null;
+ const attachment = this.currentData().activity.attachment();
// trying to display url before file is stored generates js errors
return (
(attachment &&
- link &&
+ attachment.url({ download: true }) &&
Blaze.toHTML(
HTML.A(
{
- href: link,
+ href: attachment.url({ download: true }),
target: '_blank',
},
- attachment.name,
+ attachment.name(),
),
)) ||
- activity.attachmentName
+ this.currentData().activity.attachmentName
);
},
diff --git a/client/components/cards/attachments.jade b/client/components/cards/attachments.jade
index eda6d118..61454fa7 100644
--- a/client/components/cards/attachments.jade
+++ b/client/components/cards/attachments.jade
@@ -18,19 +18,12 @@ template(name="attachmentDeletePopup")
p {{_ "attachment-delete-pop"}}
button.js-confirm.negate.full(type="submit") {{_ 'delete'}}
-template(name="uploadingPopup")
- .uploading-info
- span.upload-percentage {{progress}}%
- .upload-progress-frame
- .upload-progress-bar(style="width: {{progress}}%;")
- span.upload-size {{fileSize}}
-
template(name="attachmentsGalery")
.attachments-galery
each attachments
.attachment-item
- a.attachment-thumbnail.swipebox(href="{{url}}" download="{{name}}" title="{{name}}")
- if isUploaded
+ a.attachment-thumbnail.swipebox(href="{{url}}" title="{{name}}")
+ if isUploaded
if isImage
img.attachment-thumbnail-img(src="{{url}}")
else
@@ -40,7 +33,7 @@ template(name="attachmentsGalery")
p.attachment-details
= name
span.attachment-details-actions
- a.js-download(href="{{url download=true}}" download="{{name}}")
+ a.js-download(href="{{url download=true}}")
i.fa.fa-download
| {{_ 'download'}}
if currentUser.isBoardMember
diff --git a/client/components/cards/attachments.js b/client/components/cards/attachments.js
index 81f6c6e1..e4439155 100644
--- a/client/components/cards/attachments.js
+++ b/client/components/cards/attachments.js
@@ -13,10 +13,10 @@ Template.attachmentsGalery.events({
event.stopPropagation();
},
'click .js-add-cover'() {
- Cards.findOne(this.meta.cardId).setCover(this._id);
+ Cards.findOne(this.cardId).setCover(this._id);
},
'click .js-remove-cover'() {
- Cards.findOne(this.meta.cardId).unsetCover();
+ Cards.findOne(this.cardId).unsetCover();
},
'click .js-preview-image'(event) {
Popup.open('previewAttachedImage').call(this, event);
@@ -45,63 +45,22 @@ Template.attachmentsGalery.events({
},
});
-Template.attachmentsGalery.helpers({
- url() {
- return Attachments.link(this, 'original', '/');
- },
- isUploaded() {
- return !this.meta.uploading;
- },
- isImage() {
- return !!this.isImage;
- },
-});
-
Template.previewAttachedImagePopup.events({
'click .js-large-image-clicked'() {
Popup.close();
},
});
-Template.previewAttachedImagePopup.helpers({
- url() {
- return Attachments.link(this, 'original', '/');
- }
-});
-
-// For uploading popup
-
-let uploadFileSize = new ReactiveVar('');
-let uploadProgress = new ReactiveVar(0);
-
Template.cardAttachmentsPopup.events({
- 'change .js-attach-file'(event, instance) {
+ 'change .js-attach-file'(event) {
const card = this;
- const callbacks = {
- onBeforeUpload: (err, fileData) => {
- Popup.open('uploading')(this.clickEvent);
- uploadFileSize.set('...');
- uploadProgress.set(0);
- return true;
- },
- onUploaded: (err, attachment) => {
- if (attachment && attachment._id && attachment.isImage) {
- card.setCover(attachment._id);
- }
- Popup.close();
- },
- onStart: (error, fileData) => {
- uploadFileSize.set(formatBytes(fileData.size));
- },
- onError: (err, fileObj) => {
- console.log('Error!', err);
- },
- onProgress: (progress, fileData) => {
- uploadProgress.set(progress);
- }
- };
const processFile = f => {
- Utils.processUploadedAttachment(card, f, callbacks);
+ Utils.processUploadedAttachment(card, f, attachment => {
+ if (attachment && attachment._id && attachment.isImage()) {
+ card.setCover(attachment._id);
+ }
+ Popup.close();
+ });
};
FS.Utility.eachFile(event, f => {
@@ -141,22 +100,12 @@ Template.cardAttachmentsPopup.events({
});
},
'click .js-computer-upload'(event, templateInstance) {
- this.clickEvent = event;
templateInstance.find('.js-attach-file').click();
event.preventDefault();
},
'click .js-upload-clipboard-image': Popup.open('previewClipboardImage'),
});
-Template.uploadingPopup.helpers({
- fileSize: () => {
- return uploadFileSize.get();
- },
- progress: () => {
- return uploadProgress.get();
- }
-});
-
const MAX_IMAGE_PIXEL = Utils.MAX_IMAGE_PIXEL;
const COMPRESS_RATIO = Utils.IMAGE_COMPRESS_RATIO;
let pastedResults = null;
@@ -200,26 +149,20 @@ Template.previewClipboardImagePopup.events({
if (results && results.file) {
window.oPasted = pastedResults;
const card = this;
- const settings = {
- file: results.file,
- streams: 'dynamic',
- chunkSize: 'dynamic',
- };
+ const file = new FS.File(results.file);
if (!results.name) {
// if no filename, it's from clipboard. then we give it a name, with ext name from MIME type
if (typeof results.file.type === 'string') {
- settings.fileName =
- new Date().getTime() + results.file.type.replace('.+/', '');
+ file.name(results.file.type.replace('image/', 'clipboard.'));
}
}
- settings.meta = {};
- settings.meta.updatedAt = new Date().getTime();
- settings.meta.boardId = card.boardId;
- settings.meta.cardId = card._id;
- settings.meta.userId = Meteor.userId();
- const attachment = Attachments.insert(settings);
+ file.updatedAt(new Date());
+ file.boardId = card.boardId;
+ file.cardId = card._id;
+ file.userId = Meteor.userId();
+ const attachment = Attachments.insert(file);
- if (attachment && attachment._id && attachment.isImage) {
+ if (attachment && attachment._id && attachment.isImage()) {
card.setCover(attachment._id);
}
@@ -229,15 +172,3 @@ Template.previewClipboardImagePopup.events({
}
},
});
-
-function formatBytes(bytes, decimals = 2) {
- if (bytes === 0) return '0 Bytes';
-
- const k = 1024;
- const dm = decimals < 0 ? 0 : decimals;
- const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
-
- const i = Math.floor(Math.log(bytes) / Math.log(k));
-
- return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
-}
diff --git a/client/components/cards/attachments.styl b/client/components/cards/attachments.styl
index 61ea8232..4a22fd8a 100644
--- a/client/components/cards/attachments.styl
+++ b/client/components/cards/attachments.styl
@@ -64,17 +64,6 @@
border: 1px solid black
box-shadow: 0 1px 2px rgba(0,0,0,.2)
-.uploading-info
- .upload-progress-frame
- background-color: grey;
- border: 1px solid;
- height: 22px;
-
- .upload-progress-bar
- background-color: blue;
- height: 20px;
- padding: 1px;
-
@media screen and (max-width: 800px)
.attachments-galery
flex-direction
diff --git a/client/components/cards/minicard.jade b/client/components/cards/minicard.jade
index e9de2fea..79dd9127 100644
--- a/client/components/cards/minicard.jade
+++ b/client/components/cards/minicard.jade
@@ -11,7 +11,7 @@ template(name="minicard")
.handle
.fa.fa-arrows
if cover
- .minicard-cover(style="background-image: url('{{coverUrl}}');")
+ .minicard-cover(style="background-image: url('{{cover.url}}');")
if labels
.minicard-labels
each labels
diff --git a/client/components/cards/minicard.js b/client/components/cards/minicard.js
index 200e019d..da36b87f 100644
--- a/client/components/cards/minicard.js
+++ b/client/components/cards/minicard.js
@@ -52,7 +52,4 @@ Template.minicard.helpers({
return false;
}
},
- coverUrl() {
- return Attachments.findOne(this.coverId).link('original', '/');
- },
});
diff --git a/client/components/main/editor.js b/client/components/main/editor.js
index abe4160f..0c2e3186 100755
--- a/client/components/main/editor.js
+++ b/client/components/main/editor.js
@@ -152,31 +152,33 @@ Template.editor.onRendered(() => {
const processData = function(fileObj) {
Utils.processUploadedAttachment(
currentCard,
- fileObj,
- { onUploaded:
- attachment => {
- if (attachment && attachment._id && attachment.isImage) {
- attachment.one('uploaded', function() {
- const maxTry = 3;
- const checkItvl = 500;
- let retry = 0;
- const checkUrl = function() {
- // even though uploaded event fired, attachment.url() is still null somehow //TODO
- const url = Attachments.link(attachment, 'original', '/');
- if (url) {
- insertImage(
- `${location.protocol}//${location.host}${url}`,
- );
- } else {
- retry++;
- if (retry < maxTry) {
- setTimeout(checkUrl, checkItvl);
- }
+ fileObj,
+ attachment => {
+ if (
+ attachment &&
+ attachment._id &&
+ attachment.isImage()
+ ) {
+ attachment.one('uploaded', function() {
+ const maxTry = 3;
+ const checkItvl = 500;
+ let retry = 0;
+ const checkUrl = function() {
+ // even though uploaded event fired, attachment.url() is still null somehow //TODO
+ const url = attachment.url();
+ if (url) {
+ insertImage(
+ `${location.protocol}//${location.host}${url}`,
+ );
+ } else {
+ retry++;
+ if (retry < maxTry) {
+ setTimeout(checkUrl, checkItvl);
}
- };
- checkUrl();
- });
- }
+ }
+ };
+ checkUrl();
+ });
}
},
);
diff --git a/client/lib/utils.js b/client/lib/utils.js
index e72f177e..c921fddc 100644
--- a/client/lib/utils.js
+++ b/client/lib/utils.js
@@ -61,38 +61,30 @@ Utils = {
},
MAX_IMAGE_PIXEL: Meteor.settings.public.MAX_IMAGE_PIXEL,
COMPRESS_RATIO: Meteor.settings.public.IMAGE_COMPRESS_RATIO,
- processUploadedAttachment(card, fileObj, callbacks) {
+ processUploadedAttachment(card, fileObj, callback) {
+ const next = attachment => {
+ if (typeof callback === 'function') {
+ callback(attachment);
+ }
+ };
if (!card) {
- return onUploaded();
+ return next();
}
- let settings = {
- file: fileObj,
- streams: 'dynamic',
- chunkSize: 'dynamic',
- };
- settings.meta = {
- uploading: true
- };
+ const file = new FS.File(fileObj);
if (card.isLinkedCard()) {
- settings.meta.boardId = Cards.findOne(card.linkedId).boardId;
- settings.meta.cardId = card.linkedId;
+ file.boardId = Cards.findOne(card.linkedId).boardId;
+ file.cardId = card.linkedId;
} else {
- settings.meta.boardId = card.boardId;
- settings.meta.swimlaneId = card.swimlaneId;
- settings.meta.listId = card.listId;
- settings.meta.cardId = card._id;
+ file.boardId = card.boardId;
+ file.swimlaneId = card.swimlaneId;
+ file.listId = card.listId;
+ file.cardId = card._id;
}
- settings.meta.userId = Meteor.userId();
- if (typeof callbacks === 'function') {
- settings.onEnd = callbacks;
- } else {
- for (const key in callbacks) {
- if (key.substring(0, 2) === 'on') {
- settings[key] = callbacks[key];
- }
- }
+ file.userId = Meteor.userId();
+ if (file.original) {
+ file.original.name = fileObj.name;
}
- Attachments.insert(settings);
+ return next(Attachments.insert(file));
},
shrinkImage(options) {
// shrink image to certain size
diff --git a/models/activities.js b/models/activities.js
index 2663dd29..b5fcb7d8 100644
--- a/models/activities.js
+++ b/models/activities.js
@@ -217,7 +217,7 @@ if (Meteor.isServer) {
}
if (activity.attachmentId) {
const attachment = activity.attachment();
- params.attachment = attachment.name;
+ params.attachment = attachment.original.name;
params.attachmentId = attachment._id;
}
if (activity.checklistId) {
diff --git a/models/attachments.js b/models/attachments.js
index 03999f55..3fe1d745 100644
--- a/models/attachments.js
+++ b/models/attachments.js
@@ -1,127 +1,263 @@
-import { FilesCollection } from 'meteor/ostrio:files';
-const fs = require('fs');
-
-const collectionName = 'attachments2';
-
-Attachments = new FilesCollection({
- storagePath: storagePath(),
- debug: false,
-// allowClientCode: true,
- collectionName: 'attachments2',
- onAfterUpload: onAttachmentUploaded,
- onBeforeRemove: onAttachmentRemoving
-});
+const localFSStore = process.env.ATTACHMENTS_STORE_PATH;
+const storeName = 'attachments';
+const defaultStoreOptions = {
+ beforeWrite: fileObj => {
+ if (!fileObj.isImage()) {
+ return {
+ type: 'application/octet-stream',
+ };
+ }
+ return {};
+ },
+};
+let store;
+if (localFSStore) {
+ // have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem
+ const fs = Npm.require('fs');
+ const path = Npm.require('path');
+ const mongodb = Npm.require('mongodb');
+ const Grid = Npm.require('gridfs-stream');
+ // calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :(
+ let pathname = localFSStore;
+ /*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */
-if (Meteor.isServer) {
- Meteor.startup(() => {
- Attachments.collection._ensureIndex({ cardId: 1 });
- });
+ if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) {
+ pathname = path.join(
+ __meteor_bootstrap__.serverDir,
+ `../../../cfs/files/${storeName}`,
+ );
+ }
- // TODO: Permission related
- Attachments.allow({
- insert() {
- return false;
- },
- update() {
- return true;
- },
- remove() {
- return true;
+ if (!pathname)
+ throw new Error('FS.Store.FileSystem unable to determine path');
+
+ // Check if we have '~/foo/bar'
+ if (pathname.split(path.sep)[0] === '~') {
+ const homepath =
+ process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
+ if (homepath) {
+ pathname = pathname.replace('~', homepath);
+ } else {
+ throw new Error('FS.Store.FileSystem unable to resolve "~" in path');
}
- });
+ }
- Meteor.methods({
- cloneAttachment(file, overrides) {
- check(file, Object);
- check(overrides, Match.Maybe(Object));
- const path = file.path;
- const opts = {
- fileName: file.name,
- type: file.type,
- meta: file.meta,
- userId: file.userId
+ // Set absolute path
+ const absolutePath = path.resolve(pathname);
+
+ const _FStore = new FS.Store.FileSystem(storeName, {
+ path: localFSStore,
+ ...defaultStoreOptions,
+ });
+ const GStore = {
+ fileKey(fileObj) {
+ const key = {
+ _id: null,
+ filename: null,
};
- for (let key in overrides) {
- if (key === 'meta') {
- for (let metaKey in overrides.meta) {
- opts.meta[metaKey] = overrides.meta[metaKey];
- }
- } else {
- opts[key] = overrides[key];
- }
+
+ // If we're passed a fileObj, we retrieve the _id and filename from it.
+ if (fileObj) {
+ const info = fileObj._getInfo(storeName, {
+ updateFileRecordFirst: false,
+ });
+ key._id = info.key || null;
+ key.filename =
+ info.name ||
+ fileObj.name({ updateFileRecordFirst: false }) ||
+ `${fileObj.collectionName}-${fileObj._id}`;
}
- const buffer = fs.readFileSync(path);
- Attachments.write(buffer, opts, (err, fileRef) => {
+
+ // If key._id is null at this point, createWriteStream will let GridFS generate a new ID
+ return key;
+ },
+ db: undefined,
+ mongoOptions: { useNewUrlParser: true },
+ mongoUrl: process.env.MONGO_URL,
+ init() {
+ this._init(err => {
+ this.inited = !err;
+ });
+ },
+ _init(callback) {
+ const self = this;
+ mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function(
+ err,
+ db,
+ ) {
if (err) {
- console.log('Error when cloning record', err);
+ return callback(err);
}
+ self.db = db;
+ return callback(null);
});
- return true;
+ return;
+ },
+ createReadStream(fileKey, options) {
+ const self = this;
+ if (!self.inited) {
+ self.init();
+ return undefined;
+ }
+ options = options || {};
+
+ // Init GridFS
+ const gfs = new Grid(self.db, mongodb);
+
+ // Set the default streamning settings
+ const settings = {
+ _id: new mongodb.ObjectID(fileKey._id),
+ root: `cfs_gridfs.${storeName}`,
+ };
+
+ // Check if this should be a partial read
+ if (
+ typeof options.start !== 'undefined' &&
+ typeof options.end !== 'undefined'
+ ) {
+ // Add partial info
+ settings.range = {
+ startPos: options.start,
+ endPos: options.end,
+ };
+ }
+ return gfs.createReadStream(settings);
+ },
+ };
+ GStore.init();
+ const CRS = 'createReadStream';
+ const _CRS = `_${CRS}`;
+ const FStore = _FStore._transform;
+ FStore[_CRS] = FStore[CRS].bind(FStore);
+ FStore[CRS] = function(fileObj, options) {
+ let stream;
+ try {
+ const localFile = path.join(
+ absolutePath,
+ FStore.storage.fileKey(fileObj),
+ );
+ const state = fs.statSync(localFile);
+ if (state) {
+ stream = FStore[_CRS](fileObj, options);
+ }
+ } catch (e) {
+ // file is not there, try GridFS ?
+ stream = undefined;
}
+ if (stream) return stream;
+ else {
+ try {
+ const stream = GStore[CRS](GStore.fileKey(fileObj), options);
+ return stream;
+ } catch (e) {
+ return undefined;
+ }
+ }
+ }.bind(FStore);
+ store = _FStore;
+} else {
+ store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, {
+ // XXX Add a new store for cover thumbnails so we don't load big images in
+ // the general board view
+ // If the uploaded document is not an image we need to enforce browser
+ // download instead of execution. This is particularly important for HTML
+ // files that the browser will just execute if we don't serve them with the
+ // appropriate `application/octet-stream` MIME header which can lead to user
+ // data leaks. I imagine other formats (like PDF) can also be attack vectors.
+ // See https://github.com/wekan/wekan/issues/99
+ // XXX Should we use `beforeWrite` option of CollectionFS instead of
+ // collection-hooks?
+ // We should use `beforeWrite`.
+ ...defaultStoreOptions,
});
+}
+Attachments = new FS.Collection('attachments', {
+ stores: [store],
+});
- Meteor.publish(collectionName, function() {
- return Attachments.find().cursor;
+if (Meteor.isServer) {
+ Meteor.startup(() => {
+ Attachments.files._ensureIndex({ cardId: 1 });
});
-} else {
- Meteor.subscribe(collectionName);
-}
-function storagePath(defaultPath) {
- const storePath = process.env.ATTACHMENTS_STORE_PATH;
- return storePath ? storePath : defaultPath;
+ Attachments.allow({
+ insert(userId, doc) {
+ return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
+ },
+ update(userId, doc) {
+ return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
+ },
+ remove(userId, doc) {
+ return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
+ },
+ // We authorize the attachment download either:
+ // - if the board is public, everyone (even unconnected) can download it
+ // - if the board is private, only board members can download it
+ download(userId, doc) {
+ const board = Boards.findOne(doc.boardId);
+ if (board.isPublic()) {
+ return true;
+ } else {
+ return board.hasMember(userId);
+ }
+ },
+
+ fetch: ['boardId'],
+ });
}
-function onAttachmentUploaded(fileRef) {
- Attachments.update({_id:fileRef._id}, {$set: {"meta.uploading": false}});
- if (!fileRef.meta.source || fileRef.meta.source !== 'import') {
- // Add activity about adding the attachment
+// XXX Enforce a schema for the Attachments CollectionFS
+
+if (Meteor.isServer) {
+ Attachments.files.after.insert((userId, doc) => {
+ // If the attachment doesn't have a source field
+ // or its source is different than import
+ if (!doc.source || doc.source !== 'import') {
+ // Add activity about adding the attachment
+ Activities.insert({
+ userId,
+ type: 'card',
+ activityType: 'addAttachment',
+ attachmentId: doc._id,
+ // this preserves the name so that notifications can be meaningful after
+ // this file is removed
+ attachmentName: doc.original.name,
+ boardId: doc.boardId,
+ cardId: doc.cardId,
+ listId: doc.listId,
+ swimlaneId: doc.swimlaneId,
+ });
+ } else {
+ // Don't add activity about adding the attachment as the activity
+ // be imported and delete source field
+ Attachments.update(
+ {
+ _id: doc._id,
+ },
+ {
+ $unset: {
+ source: '',
+ },
+ },
+ );
+ }
+ });
+
+ Attachments.files.before.remove((userId, doc) => {
Activities.insert({
- userId: fileRef.userId,
+ userId,
type: 'card',
- activityType: 'addAttachment',
- attachmentId: fileRef._id,
+ activityType: 'deleteAttachment',
+ attachmentId: doc._id,
// this preserves the name so that notifications can be meaningful after
- // this file is removed
- attachmentName: fileRef.name,
- boardId: fileRef.meta.boardId,
- cardId: fileRef.meta.cardId,
- listId: fileRef.meta.listId,
- swimlaneId: fileRef.meta.swimlaneId,
+ // this file is removed
+ attachmentName: doc.original.name,
+ boardId: doc.boardId,
+ cardId: doc.cardId,
+ listId: doc.listId,
+ swimlaneId: doc.swimlaneId,
});
- } else {
- // Don't add activity about adding the attachment as the activity
- // be imported and delete source field
- Attachments.collection.update(
- {
- _id: fileRef._id,
- },
- {
- $unset: {
- 'meta.source': '',
- },
- },
- );
- }
-}
-
-function onAttachmentRemoving(cursor) {
- const file = cursor.get()[0];
- const meta = file.meta;
- Activities.insert({
- userId: this.userId,
- type: 'card',
- activityType: 'deleteAttachment',
- attachmentId: file._id,
- // this preserves the name so that notifications can be meaningful after
- // this file is removed
- attachmentName: file.name,
- boardId: meta.boardId,
- cardId: meta.cardId,
- listId: meta.listId,
- swimlaneId: meta.swimlaneId,
});
- return true;
}
export default Attachments;
diff --git a/models/cards.js b/models/cards.js
index 757772f3..6d5e23cc 100644
--- a/models/cards.js
+++ b/models/cards.js
@@ -412,14 +412,10 @@ Cards.helpers({
const _id = Cards.insert(this);
// Copy attachments
- oldCard.attachments().forEach((file) => {
- Meteor.call('cloneAttachment', file,
- {
- meta: {
- cardId: _id
- }
- }
- );
+ oldCard.attachments().forEach(att => {
+ att.cardId = _id;
+ delete att._id;
+ return Attachments.insert(att);
});
// copy checklists
@@ -522,15 +518,14 @@ Cards.helpers({
attachments() {
if (this.isLinkedCard()) {
return Attachments.find(
- { 'meta.cardId': this.linkedId },
+ { cardId: this.linkedId },
{ sort: { uploadedAt: -1 } },
);
} else {
- let ret = Attachments.find(
- { 'meta.cardId': this._id },
+ return Attachments.find(
+ { cardId: this._id },
{ sort: { uploadedAt: -1 } },
);
- return ret;
}
},
@@ -539,7 +534,7 @@ Cards.helpers({
const cover = Attachments.findOne(this.coverId);
// if we return a cover before it is fully stored, we will get errors when we try to display it
// todo XXX we could return a default "upload pending" image in the meantime?
- return cover && cover.link();
+ return cover && cover.url() && cover;
},
checklists() {
diff --git a/models/export.js b/models/export.js
index 1eb47e54..c3783679 100644
--- a/models/export.js
+++ b/models/export.js
@@ -1,3 +1,4 @@
+import { Exporter } from './exporter';
/* global JsonRoutes */
if (Meteor.isServer) {
// todo XXX once we have a real API in place, move that route there
@@ -7,10 +8,10 @@ if (Meteor.isServer) {
// on the client instead of copy/pasting the route path manually between the
// client and the server.
/**
- * @operation export
+ * @operation exportJson
* @tag Boards
*
- * @summary This route is used to export the board.
+ * @summary This route is used to export the board to a json file format.
*
* @description If user is already logged-in, pass loginToken as param
* "authToken": '/api/boards/:boardId/export?authToken=:token'
@@ -46,199 +47,52 @@ if (Meteor.isServer) {
JsonRoutes.sendResult(res, 403);
}
});
-}
-
-// exporter maybe is broken since Gridfs introduced, add fs and path
-
-export class Exporter {
- constructor(boardId) {
- this._boardId = boardId;
- }
-
- build() {
- const fs = Npm.require('fs');
- const os = Npm.require('os');
- const path = Npm.require('path');
-
- const byBoard = { boardId: this._boardId };
- const byBoardNoLinked = {
- boardId: this._boardId,
- linkedId: { $in: ['', null] },
- };
- // we do not want to retrieve boardId in related elements
- const noBoardId = {
- fields: {
- boardId: 0,
- },
- };
- const result = {
- _format: 'wekan-board-1.0.0',
- };
- _.extend(
- result,
- Boards.findOne(this._boardId, {
- fields: {
- stars: 0,
- },
- }),
- );
- result.lists = Lists.find(byBoard, noBoardId).fetch();
- result.cards = Cards.find(byBoardNoLinked, noBoardId).fetch();
- result.swimlanes = Swimlanes.find(byBoard, noBoardId).fetch();
- result.customFields = CustomFields.find(
- { boardIds: { $in: [this.boardId] } },
- { fields: { boardId: 0 } },
- ).fetch();
- result.comments = CardComments.find(byBoard, noBoardId).fetch();
- result.activities = Activities.find(byBoard, noBoardId).fetch();
- result.rules = Rules.find(byBoard, noBoardId).fetch();
- result.checklists = [];
- result.checklistItems = [];
- result.subtaskItems = [];
- result.triggers = [];
- result.actions = [];
- result.cards.forEach(card => {
- result.checklists.push(
- ...Checklists.find({
- cardId: card._id,
- }).fetch(),
- );
- result.checklistItems.push(
- ...ChecklistItems.find({
- cardId: card._id,
- }).fetch(),
- );
- result.subtaskItems.push(
- ...Cards.find({
- parentId: card._id,
- }).fetch(),
- );
- });
- result.rules.forEach(rule => {
- result.triggers.push(
- ...Triggers.find(
- {
- _id: rule.triggerId,
- },
- noBoardId,
- ).fetch(),
- );
- result.actions.push(
- ...Actions.find(
- {
- _id: rule.actionId,
- },
- noBoardId,
- ).fetch(),
- );
- });
-
- // [Old] for attachments we only export IDs and absolute url to original doc
- // [New] Encode attachment to base64
-
- const getBase64Data = function(doc, callback) {
- let buffer = Buffer.allocUnsafe(0);
- buffer.fill(0);
-
- // callback has the form function (err, res) {}
- const tmpFile = path.join(
- os.tmpdir(),
- `tmpexport${process.pid}${Math.random()}`,
- );
- const tmpWriteable = fs.createWriteStream(tmpFile);
- const readStream = fs.createReadStream(doc.path);
- readStream.on('data', function(chunk) {
- buffer = Buffer.concat([buffer, chunk]);
- });
-
- readStream.on('error', function(err) {
- callback(null, null);
- });
- readStream.on('end', function() {
- // done
- fs.unlink(tmpFile, () => {
- //ignored
- });
- callback(null, buffer.toString('base64'));
+ /**
+ * @operation exportCSV/TSV
+ * @tag Boards
+ *
+ * @summary This route is used to export the board to a CSV or TSV file format.
+ *
+ * @description If user is already logged-in, pass loginToken as param
+ *
+ * See https://blog.kayla.com.au/server-side-route-authentication-in-meteor/
+ * for detailed explanations
+ *
+ * @param {string} boardId the ID of the board we are exporting
+ * @param {string} authToken the loginToken
+ * @param {string} delimiter delimiter to use while building export. Default is comma ','
+ */
+ Picker.route('/api/boards/:boardId/export/csv', function(params, req, res) {
+ const boardId = params.boardId;
+ let user = null;
+ const loginToken = params.query.authToken;
+ if (loginToken) {
+ const hashToken = Accounts._hashLoginToken(loginToken);
+ user = Meteor.users.findOne({
+ 'services.resume.loginTokens.hashedToken': hashToken,
});
- readStream.pipe(tmpWriteable);
- };
- const getBase64DataSync = Meteor.wrapAsync(getBase64Data);
- result.attachments = Attachments.find({ 'meta.boardId': byBoard.boardId })
- .fetch()
- .map(attachment => {
- let filebase64 = null;
- filebase64 = getBase64DataSync(attachment);
-
- return {
- _id: attachment._id,
- cardId: attachment.meta.cardId,
- //url: FlowRouter.url(attachment.url()),
- file: filebase64,
- name: attachment.name,
- type: attachment.type,
- };
+ } else if (!Meteor.settings.public.sandstorm) {
+ Authentication.checkUserId(req.userId);
+ user = Users.findOne({
+ _id: req.userId,
+ isAdmin: true,
});
-
- // we also have to export some user data - as the other elements only
- // include id but we have to be careful:
- // 1- only exports users that are linked somehow to that board
- // 2- do not export any sensitive information
- const users = {};
- result.members.forEach(member => {
- users[member.userId] = true;
- });
- result.lists.forEach(list => {
- users[list.userId] = true;
- });
- result.cards.forEach(card => {
- users[card.userId] = true;
- if (card.members) {
- card.members.forEach(memberId => {
- users[memberId] = true;
- });
- }
- });
- result.comments.forEach(comment => {
- users[comment.userId] = true;
- });
- result.activities.forEach(activity => {
- users[activity.userId] = true;
- });
- result.checklists.forEach(checklist => {
- users[checklist.userId] = true;
- });
- const byUserIds = {
- _id: {
- $in: Object.getOwnPropertyNames(users),
- },
- };
- // we use whitelist to be sure we do not expose inadvertently
- // some secret fields that gets added to User later.
- const userFields = {
- fields: {
- _id: 1,
- username: 1,
- 'profile.fullname': 1,
- 'profile.initials': 1,
- 'profile.avatarUrl': 1,
- },
- };
- result.users = Users.find(byUserIds, userFields)
- .fetch()
- .map(user => {
- // user avatar is stored as a relative url, we export absolute
- if ((user.profile || {}).avatarUrl) {
- user.profile.avatarUrl = FlowRouter.url(user.profile.avatarUrl);
- }
- return user;
+ }
+ const exporter = new Exporter(boardId);
+ if (exporter.canExport(user)) {
+ body = params.query.delimiter
+ ? exporter.buildCsv(params.query.delimiter)
+ : exporter.buildCsv();
+ res.writeHead(200, {
+ 'Content-Length': body[0].length,
+ 'Content-Type': params.query.delimiter ? 'text/csv' : 'text/tsv',
});
- return result;
- }
-
- canExport(user) {
- const board = Boards.findOne(this._boardId);
- return board && board.isVisibleBy(user);
- }
+ res.write(body[0]);
+ res.end();
+ } else {
+ res.writeHead(403);
+ res.end('Permission Error');
+ }
+ });
}
diff --git a/models/trelloCreator.js b/models/trelloCreator.js
index c4be140b..1c5bcd93 100644
--- a/models/trelloCreator.js
+++ b/models/trelloCreator.js
@@ -369,7 +369,6 @@ export class TrelloCreator {
// so we make it server only, and let UI catch up once it is done, forget about latency comp.
const self = this;
if (Meteor.isServer) {
- // FIXME: Change to new model
file.attachData(att.url, function(error) {
file.boardId = boardId;
file.cardId = cardId;
diff --git a/models/wekanCreator.js b/models/wekanCreator.js
index c5591a0b..9914f817 100644
--- a/models/wekanCreator.js
+++ b/models/wekanCreator.js
@@ -415,7 +415,6 @@ export class WekanCreator {
const self = this;
if (Meteor.isServer) {
if (att.url) {
- // FIXME: Change to new file library
file.attachData(att.url, function(error) {
file.boardId = boardId;
file.cardId = cardId;
@@ -441,7 +440,6 @@ export class WekanCreator {
}
});
} else if (att.file) {
- // FIXME: Change to new file library
file.attachData(
Buffer.from(att.file, 'base64'),
{
diff --git a/sandstorm-pkgdef.capnp b/sandstorm-pkgdef.capnp
index a902cb47..3dcf9e5f 100644
--- a/sandstorm-pkgdef.capnp
+++ b/sandstorm-pkgdef.capnp
@@ -22,10 +22,10 @@ const pkgdef :Spk.PackageDefinition = (
appTitle = (defaultText = "Wekan"),
# The name of the app as it is displayed to the user.
- appVersion = 404,
+ appVersion = 403,
# Increment this for every release.
- appMarketingVersion = (defaultText = "4.04.0~2020-05-24"),
+ appMarketingVersion = (defaultText = "4.03.0~2020-05-16"),
# Human-readable presentation of the app version.
minUpgradableAppVersion = 0,
@@ -261,7 +261,6 @@ const myCommand :Spk.Manifest.Command = (
(key = "LDAP_ENABLE", value="false"),
(key = "PASSWORD_LOGIN_ENABLED", value="true"),
(key = "SANDSTORM", value="1"),
- (key = "METEOR_SETTINGS", value = "{\"public\": {\"sandstorm\": true}}"),
- (key = "ATTACHMENTS_STORE_PATH", value = "/var/attachments/")
+ (key = "METEOR_SETTINGS", value = "{\"public\": {\"sandstorm\": true}}")
]
);
diff --git a/server/migrations.js b/server/migrations.js
index 72b39ea7..5655bd1d 100644
--- a/server/migrations.js
+++ b/server/migrations.js
@@ -80,7 +80,7 @@ Migrations.add('lowercase-board-permission', () => {
Migrations.add('change-attachments-type-for-non-images', () => {
const newTypeForNonImage = 'application/octet-stream';
Attachments.find().forEach(file => {
- if (!file.isImage) {
+ if (!file.isImage()) {
Attachments.update(
file._id,
{
@@ -1044,46 +1044,3 @@ Migrations.add('add-sort-field-to-boards', () => {
}
});
});
-
-import { MongoInternals } from 'meteor/mongo';
-
-Migrations.add('change-attachment-library', () => {
- const fs = require('fs');
- CFSAttachments.find().forEach(file => {
- const bucket = new MongoInternals.NpmModule.GridFSBucket(MongoInternals.defaultRemoteCollectionDriver().mongo.db, {bucketName: 'cfs_gridfs.attachments'});
- const gfsId = new MongoInternals.NpmModule.ObjectID(file.copies.attachments.key);
- const reader = bucket.openDownloadStream(gfsId);
- let store = Attachments.storagePath();
- if (store.charAt(store.length - 1) === '/') {
- store = store.substring(0, store.length - 1);
- }
- const path = `${store}/${file.name()}`;
- const fd = fs.createWriteStream(path);
- reader.pipe(fd);
- reader.on('end', () => {
- let opts = {
- fileName: file.name(),
- type: file.type(),
- size: file.size(),
- fileId: file._id,
- meta: {
- userId: file.userId,
- boardId: file.boardId,
- cardId: file.cardId
- }
- };
- if (file.listId) {
- opts.meta.listId = file.listId;
- }
- if (file.swimlaneId) {
- opts.meta.swimlaneId = file.swimlaneId;
- }
- Attachments.addFile(path, opts, (err, fileRef) => {
- if (err) {
- console.log('error when migrating', file.name(), err);
- }
- });
- });
- });
-});
-
diff --git a/server/old-attachments-migration.js b/server/old-attachments-migration.js
deleted file mode 100644
index 3a6aa85d..00000000
--- a/server/old-attachments-migration.js
+++ /dev/null
@@ -1,212 +0,0 @@
-const localFSStore = process.env.ATTACHMENTS_STORE_PATH;
-const storeName = 'attachments';
-const defaultStoreOptions = {
- beforeWrite: fileObj => {
- if (!fileObj.isImage()) {
- return {
- type: 'application/octet-stream',
- };
- }
- return {};
- },
-};
-let store;
-if (localFSStore) {
- // have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem
- const fs = Npm.require('fs');
- const path = Npm.require('path');
- const mongodb = Npm.require('mongodb');
- const Grid = Npm.require('gridfs-stream');
- // calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :(
- let pathname = localFSStore;
- /*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */
-
- if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) {
- pathname = path.join(
- __meteor_bootstrap__.serverDir,
- `../../../cfs/files/${storeName}`,
- );
- }
-
- if (!pathname)
- throw new Error('FS.Store.FileSystem unable to determine path');
-
- // Check if we have '~/foo/bar'
- if (pathname.split(path.sep)[0] === '~') {
- const homepath =
- process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
- if (homepath) {
- pathname = pathname.replace('~', homepath);
- } else {
- throw new Error('FS.Store.FileSystem unable to resolve "~" in path');
- }
- }
-
- // Set absolute path
- const absolutePath = path.resolve(pathname);
-
- const _FStore = new FS.Store.FileSystem(storeName, {
- path: localFSStore,
- ...defaultStoreOptions,
- });
- const GStore = {
- fileKey(fileObj) {
- const key = {
- _id: null,
- filename: null,
- };
-
- // If we're passed a fileObj, we retrieve the _id and filename from it.
- if (fileObj) {
- const info = fileObj._getInfo(storeName, {
- updateFileRecordFirst: false,
- });
- key._id = info.key || null;
- key.filename =
- info.name ||
- fileObj.name({ updateFileRecordFirst: false }) ||
- `${fileObj.collectionName}-${fileObj._id}`;
- }
-
- // If key._id is null at this point, createWriteStream will let GridFS generate a new ID
- return key;
- },
- db: undefined,
- mongoOptions: { useNewUrlParser: true },
- mongoUrl: process.env.MONGO_URL,
- init() {
- this._init(err => {
- this.inited = !err;
- });
- },
- _init(callback) {
- const self = this;
- mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function(
- err,
- db,
- ) {
- if (err) {
- return callback(err);
- }
- self.db = db;
- return callback(null);
- });
- return;
- },
- createReadStream(fileKey, options) {
- const self = this;
- if (!self.inited) {
- self.init();
- return undefined;
- }
- options = options || {};
-
- // Init GridFS
- const gfs = new Grid(self.db, mongodb);
-
- // Set the default streamning settings
- const settings = {
- _id: new mongodb.ObjectID(fileKey._id),
- root: `cfs_gridfs.${storeName}`,
- };
-
- // Check if this should be a partial read
- if (
- typeof options.start !== 'undefined' &&
- typeof options.end !== 'undefined'
- ) {
- // Add partial info
- settings.range = {
- startPos: options.start,
- endPos: options.end,
- };
- }
- return gfs.createReadStream(settings);
- },
- };
- GStore.init();
- const CRS = 'createReadStream';
- const _CRS = `_${CRS}`;
- const FStore = _FStore._transform;
- FStore[_CRS] = FStore[CRS].bind(FStore);
- FStore[CRS] = function(fileObj, options) {
- let stream;
- try {
- const localFile = path.join(
- absolutePath,
- FStore.storage.fileKey(fileObj),
- );
- const state = fs.statSync(localFile);
- if (state) {
- stream = FStore[_CRS](fileObj, options);
- }
- } catch (e) {
- // file is not there, try GridFS ?
- stream = undefined;
- }
- if (stream) return stream;
- else {
- try {
- const stream = GStore[CRS](GStore.fileKey(fileObj), options);
- return stream;
- } catch (e) {
- return undefined;
- }
- }
- }.bind(FStore);
- store = _FStore;
-} else {
- store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, {
- // XXX Add a new store for cover thumbnails so we don't load big images in
- // the general board view
- // If the uploaded document is not an image we need to enforce browser
- // download instead of execution. This is particularly important for HTML
- // files that the browser will just execute if we don't serve them with the
- // appropriate `application/octet-stream` MIME header which can lead to user
- // data leaks. I imagine other formats (like PDF) can also be attack vectors.
- // See https://github.com/wekan/wekan/issues/99
- // XXX Should we use `beforeWrite` option of CollectionFS instead of
- // collection-hooks?
- // We should use `beforeWrite`.
- ...defaultStoreOptions,
- });
-}
-CFSAttachments = new FS.Collection('attachments', {
- stores: [store],
-});
-
-if (Meteor.isServer) {
- Meteor.startup(() => {
- CFSAttachments.files._ensureIndex({ cardId: 1 });
- });
-
- CFSAttachments.allow({
- insert(userId, doc) {
- return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
- },
- update(userId, doc) {
- return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
- },
- remove(userId, doc) {
- return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
- },
- // We authorize the attachment download either:
- // - if the board is public, everyone (even unconnected) can download it
- // - if the board is private, only board members can download it
- download(userId, doc) {
- if (Meteor.isServer) {
- return true;
- }
- const board = Boards.findOne(doc.boardId);
- if (board.isPublic()) {
- return true;
- } else {
- return board.hasMember(userId);
- }
- },
-
- fetch: ['boardId'],
- });
-}
-
-export default CFSAttachments;
diff --git a/server/publications/boards.js b/server/publications/boards.js
index ae82d28c..b80a6b23 100644
--- a/server/publications/boards.js
+++ b/server/publications/boards.js
@@ -131,7 +131,7 @@ Meteor.publishRelations('board', function(boardId, isArchived) {
// Gather queries and send in bulk
const cardComments = this.join(CardComments);
cardComments.selector = _ids => ({ cardId: _ids });
- const attachments = this.join(Attachments.collection);
+ const attachments = this.join(Attachments);
attachments.selector = _ids => ({ cardId: _ids });
const checklists = this.join(Checklists);
checklists.selector = _ids => ({ cardId: _ids });
diff --git a/snap-src/bin/config b/snap-src/bin/config
index ec0d3a0b..065b8685 100755
--- a/snap-src/bin/config
+++ b/snap-src/bin/config
@@ -93,9 +93,11 @@ DEFAULT_ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURE_WINDOW="15"
KEY_ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURE_WINDOW="accounts-lockout-unknown-users-failure-window"
DESCRIPTION_ATTACHMENTS_STORE_PATH="Allow wekan ower to specify where uploaded files to store on the server instead of the mongodb"
-DEFAULT_ATTACHMENTS_STORE_PATH="/var/snap/wekan/common/uploads/"
+DEFAULT_ATTACHMENTS_STORE_PATH=""
KEY_ATTACHMENTS_STORE_PATH="attachments-store-path"
+# Example, not in use: /var/snap/wekan/common/uploads/
+
DESCRIPTION_MAX_IMAGE_PIXEL="Max image pixel: Allow to shrink attached/pasted image https://github.com/wekan/wekan/pull/2544"
DEFAULT_MAX_IMAGE_PIXEL=""
KEY_MAX_IMAGE_PIXEL="max-image-pixel"
diff --git a/snap-src/bin/mongodb-control b/snap-src/bin/mongodb-control
index 558aecd0..a78f7dfc 100755
--- a/snap-src/bin/mongodb-control
+++ b/snap-src/bin/mongodb-control
@@ -24,11 +24,11 @@ if test -f "$SNAP_COMMON/mongodb.log"; then
rm -f "$SNAP_COMMON/mongodb.log"
fi
-# If uploads directory does not exist, create it.
+# Not in use. If uploads directory does not exist, create it.
# Wekan will store attachments there.
-if [ ! -d "$SNAP_COMMON/uploads" ]; then
- mkdir "$SNAP_COMMON/uploads"
-fi
+#if [ ! -d "$SNAP_COMMON/uploads" ]; then
+# mkdir "$SNAP_COMMON/uploads"
+#fi
# Alternative: When starting MongoDB, and using logfile, truncate log to last 1000 lines of text.
# 1) If file exists: