Merge branch 'lib-change' of https://github.com/PDIS/wekan into

PDIS-lib-change

6 files changed, 313 insertions, 295 deletions

diff --git a/models/activities.js b/models/activities.js
index b5fcb7d8..2663dd29 100644
--- a/models/activities.js
+++ b/models/activities.js
@@ -217,7 +217,7 @@ if (Meteor.isServer) {
     }
     if (activity.attachmentId) {
       const attachment = activity.attachment();
-      params.attachment = attachment.original.name;
+      params.attachment = attachment.name;
       params.attachmentId = attachment._id;
     }
     if (activity.checklistId) {
diff --git a/models/attachments.js b/models/attachments.js
index 3fe1d745..03999f55 100644
--- a/models/attachments.js
+++ b/models/attachments.js
@@ -1,263 +1,127 @@
-const localFSStore = process.env.ATTACHMENTS_STORE_PATH;
-const storeName = 'attachments';
-const defaultStoreOptions = {
-  beforeWrite: fileObj => {
-    if (!fileObj.isImage()) {
-      return {
-        type: 'application/octet-stream',
-      };
-    }
-    return {};
-  },
-};
-let store;
-if (localFSStore) {
-  // have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem
-  const fs = Npm.require('fs');
-  const path = Npm.require('path');
-  const mongodb = Npm.require('mongodb');
-  const Grid = Npm.require('gridfs-stream');
-  // calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :(
-  let pathname = localFSStore;
-  /*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */
-
-  if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) {
-    pathname = path.join(
-      __meteor_bootstrap__.serverDir,
-      `../../../cfs/files/${storeName}`,
-    );
-  }
+import { FilesCollection } from 'meteor/ostrio:files';
+const fs = require('fs');
+
+const collectionName = 'attachments2';
+
+Attachments = new FilesCollection({
+  storagePath: storagePath(),
+  debug: false, 
+//  allowClientCode: true,
+  collectionName: 'attachments2',
+  onAfterUpload: onAttachmentUploaded,
+  onBeforeRemove: onAttachmentRemoving
+});
 
-  if (!pathname)
-    throw new Error('FS.Store.FileSystem unable to determine path');
+if (Meteor.isServer) {
+  Meteor.startup(() => {
+    Attachments.collection._ensureIndex({ cardId: 1 });
+  });
 
-  // Check if we have '~/foo/bar'
-  if (pathname.split(path.sep)[0] === '~') {
-    const homepath =
-      process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
-    if (homepath) {
-      pathname = pathname.replace('~', homepath);
-    } else {
-      throw new Error('FS.Store.FileSystem unable to resolve "~" in path');
+  // TODO: Permission related
+  Attachments.allow({
+    insert() {
+      return false;
+    },
+    update() {
+      return true;
+    },
+    remove() {
+      return true;
     }
-  }
-
-  // Set absolute path
-  const absolutePath = path.resolve(pathname);
-
-  const _FStore = new FS.Store.FileSystem(storeName, {
-    path: localFSStore,
-    ...defaultStoreOptions,
   });
-  const GStore = {
-    fileKey(fileObj) {
-      const key = {
-        _id: null,
-        filename: null,
-      };
 
-      // If we're passed a fileObj, we retrieve the _id and filename from it.
-      if (fileObj) {
-        const info = fileObj._getInfo(storeName, {
-          updateFileRecordFirst: false,
-        });
-        key._id = info.key || null;
-        key.filename =
-          info.name ||
-          fileObj.name({ updateFileRecordFirst: false }) ||
-          `${fileObj.collectionName}-${fileObj._id}`;
+  Meteor.methods({
+    cloneAttachment(file, overrides) {
+      check(file, Object);
+      check(overrides, Match.Maybe(Object));
+      const path = file.path;
+      const opts = {
+          fileName: file.name,
+          type: file.type,
+          meta: file.meta,
+          userId: file.userId
+      };
+      for (let key in overrides) {
+        if (key === 'meta') {
+          for (let metaKey in overrides.meta) {
+            opts.meta[metaKey] = overrides.meta[metaKey];
+          }
+        } else {
+          opts[key] = overrides[key];
+        }
       }
-
-      // If key._id is null at this point, createWriteStream will let GridFS generate a new ID
-      return key;
-    },
-    db: undefined,
-    mongoOptions: { useNewUrlParser: true },
-    mongoUrl: process.env.MONGO_URL,
-    init() {
-      this._init(err => {
-        this.inited = !err;
-      });
-    },
-    _init(callback) {
-      const self = this;
-      mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function(
-        err,
-        db,
-      ) {
+      const buffer = fs.readFileSync(path);
+      Attachments.write(buffer, opts, (err, fileRef) => {
         if (err) {
-          return callback(err);
+          console.log('Error when cloning record', err);
         }
-        self.db = db;
-        return callback(null);
       });
-      return;
-    },
-    createReadStream(fileKey, options) {
-      const self = this;
-      if (!self.inited) {
-        self.init();
-        return undefined;
-      }
-      options = options || {};
-
-      // Init GridFS
-      const gfs = new Grid(self.db, mongodb);
-
-      // Set the default streamning settings
-      const settings = {
-        _id: new mongodb.ObjectID(fileKey._id),
-        root: `cfs_gridfs.${storeName}`,
-      };
-
-      // Check if this should be a partial read
-      if (
-        typeof options.start !== 'undefined' &&
-        typeof options.end !== 'undefined'
-      ) {
-        // Add partial info
-        settings.range = {
-          startPos: options.start,
-          endPos: options.end,
-        };
-      }
-      return gfs.createReadStream(settings);
-    },
-  };
-  GStore.init();
-  const CRS = 'createReadStream';
-  const _CRS = `_${CRS}`;
-  const FStore = _FStore._transform;
-  FStore[_CRS] = FStore[CRS].bind(FStore);
-  FStore[CRS] = function(fileObj, options) {
-    let stream;
-    try {
-      const localFile = path.join(
-        absolutePath,
-        FStore.storage.fileKey(fileObj),
-      );
-      const state = fs.statSync(localFile);
-      if (state) {
-        stream = FStore[_CRS](fileObj, options);
-      }
-    } catch (e) {
-      // file is not there, try GridFS ?
-      stream = undefined;
+      return true;
     }
-    if (stream) return stream;
-    else {
-      try {
-        const stream = GStore[CRS](GStore.fileKey(fileObj), options);
-        return stream;
-      } catch (e) {
-        return undefined;
-      }
-    }
-  }.bind(FStore);
-  store = _FStore;
-} else {
-  store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, {
-    // XXX Add a new store for cover thumbnails so we don't load big images in
-    // the general board view
-    // If the uploaded document is not an image we need to enforce browser
-    // download instead of execution. This is particularly important for HTML
-    // files that the browser will just execute if we don't serve them with the
-    // appropriate `application/octet-stream` MIME header which can lead to user
-    // data leaks. I imagine other formats (like PDF) can also be attack vectors.
-    // See https://github.com/wekan/wekan/issues/99
-    // XXX Should we use `beforeWrite` option of CollectionFS instead of
-    // collection-hooks?
-    // We should use `beforeWrite`.
-    ...defaultStoreOptions,
   });
-}
-Attachments = new FS.Collection('attachments', {
-  stores: [store],
-});
-
-if (Meteor.isServer) {
-  Meteor.startup(() => {
-    Attachments.files._ensureIndex({ cardId: 1 });
-  });
-
-  Attachments.allow({
-    insert(userId, doc) {
-      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
-    },
-    update(userId, doc) {
-      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
-    },
-    remove(userId, doc) {
-      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
-    },
-    // We authorize the attachment download either:
-    // - if the board is public, everyone (even unconnected) can download it
-    // - if the board is private, only board members can download it
-    download(userId, doc) {
-      const board = Boards.findOne(doc.boardId);
-      if (board.isPublic()) {
-        return true;
-      } else {
-        return board.hasMember(userId);
-      }
-    },
 
-    fetch: ['boardId'],
+  Meteor.publish(collectionName, function() {
+    return Attachments.find().cursor;
   });
+} else {
+  Meteor.subscribe(collectionName);
 }
 
-// XXX Enforce a schema for the Attachments CollectionFS
-
-if (Meteor.isServer) {
-  Attachments.files.after.insert((userId, doc) => {
-    // If the attachment doesn't have a source field
-    // or its source is different than import
-    if (!doc.source || doc.source !== 'import') {
-      // Add activity about adding the attachment
-      Activities.insert({
-        userId,
-        type: 'card',
-        activityType: 'addAttachment',
-        attachmentId: doc._id,
-        // this preserves the name so that notifications can be meaningful after
-        // this file is removed
-        attachmentName: doc.original.name,
-        boardId: doc.boardId,
-        cardId: doc.cardId,
-        listId: doc.listId,
-        swimlaneId: doc.swimlaneId,
-      });
-    } else {
-      // Don't add activity about adding the attachment as the activity
-      // be imported and delete source field
-      Attachments.update(
-        {
-          _id: doc._id,
-        },
-        {
-          $unset: {
-            source: '',
-          },
-        },
-      );
-    }
-  });
+function storagePath(defaultPath) {
+  const storePath = process.env.ATTACHMENTS_STORE_PATH;
+  return storePath ? storePath : defaultPath;
+}
 
-  Attachments.files.before.remove((userId, doc) => {
+function onAttachmentUploaded(fileRef) {
+  Attachments.update({_id:fileRef._id}, {$set: {"meta.uploading": false}});
+  if (!fileRef.meta.source || fileRef.meta.source !== 'import') {
+    // Add activity about adding the attachment
     Activities.insert({
-      userId,
+      userId: fileRef.userId,
       type: 'card',
-      activityType: 'deleteAttachment',
-      attachmentId: doc._id,
+      activityType: 'addAttachment',
+      attachmentId: fileRef._id,
       // this preserves the name so that notifications can be meaningful after
-      // this file is removed
-      attachmentName: doc.original.name,
-      boardId: doc.boardId,
-      cardId: doc.cardId,
-      listId: doc.listId,
-      swimlaneId: doc.swimlaneId,
+      // this file is removed 
+      attachmentName: fileRef.name,
+      boardId: fileRef.meta.boardId,
+      cardId: fileRef.meta.cardId,
+      listId: fileRef.meta.listId,
+      swimlaneId: fileRef.meta.swimlaneId,
     });
+  } else {
+    // Don't add activity about adding the attachment as the activity
+    // be imported and delete source field
+    Attachments.collection.update(
+      {
+        _id: fileRef._id,
+      },
+      {
+        $unset: {
+          'meta.source': '',
+        },
+      },
+    );
+  }
+}
+
+function onAttachmentRemoving(cursor) {
+  const file = cursor.get()[0];
+  const meta = file.meta;
+  Activities.insert({
+    userId: this.userId,
+    type: 'card',
+    activityType: 'deleteAttachment',
+    attachmentId: file._id,
+    // this preserves the name so that notifications can be meaningful after
+    // this file is removed
+    attachmentName: file.name,
+    boardId: meta.boardId,
+    cardId: meta.cardId,
+    listId: meta.listId,
+    swimlaneId: meta.swimlaneId,
   });
+  return true;
 }
 
 export default Attachments;
diff --git a/models/cards.js b/models/cards.js
index aace2647..f4bb464c 100644
--- a/models/cards.js
+++ b/models/cards.js
@@ -412,10 +412,14 @@ Cards.helpers({
     const _id = Cards.insert(this);
 
     // Copy attachments
-    oldCard.attachments().forEach(att => {
-      att.cardId = _id;
-      delete att._id;
-      return Attachments.insert(att);
+    oldCard.attachments().forEach((file) => {
+      Meteor.call('cloneAttachment', file, 
+        {
+          meta: {
+            cardId: _id
+          }
+        }
+      );
     });
 
     // copy checklists
@@ -518,14 +522,15 @@ Cards.helpers({
   attachments() {
     if (this.isLinkedCard()) {
       return Attachments.find(
-        { cardId: this.linkedId },
+        { 'meta.cardId': this.linkedId },
         { sort: { uploadedAt: -1 } },
       );
     } else {
-      return Attachments.find(
-        { cardId: this._id },
+      let ret = Attachments.find(
+        { 'meta.cardId': this._id },
         { sort: { uploadedAt: -1 } },
       );
+      return ret;
     }
   },
 
@@ -533,7 +538,7 @@ Cards.helpers({
     const cover = Attachments.findOne(this.coverId);
     // if we return a cover before it is fully stored, we will get errors when we try to display it
     // todo XXX we could return a default "upload pending" image in the meantime?
-    return cover && cover.url() && cover;
+    return cover && cover.link();
   },
 
   checklists() {
diff --git a/models/export.js b/models/export.js
index c3783679..1eb47e54 100644
--- a/models/export.js
+++ b/models/export.js
@@ -1,4 +1,3 @@
-import { Exporter } from './exporter';
 /* global JsonRoutes */
 if (Meteor.isServer) {
   // todo XXX once we have a real API in place, move that route there
@@ -8,10 +7,10 @@ if (Meteor.isServer) {
   // on the client instead of copy/pasting the route path manually between the
   // client and the server.
   /**
-   * @operation exportJson
+   * @operation export
    * @tag Boards
    *
-   * @summary This route is used to export the board to a json file format.
+   * @summary This route is used to export the board.
    *
    * @description If user is already logged-in, pass loginToken as param
    * "authToken": '/api/boards/:boardId/export?authToken=:token'
@@ -47,52 +46,199 @@ if (Meteor.isServer) {
       JsonRoutes.sendResult(res, 403);
     }
   });
+}
 
-  /**
-   * @operation exportCSV/TSV
-   * @tag Boards
-   *
-   * @summary This route is used to export the board to a CSV or TSV file format.
-   *
-   * @description If user is already logged-in, pass loginToken as param
-   *
-   * See https://blog.kayla.com.au/server-side-route-authentication-in-meteor/
-   * for detailed explanations
-   *
-   * @param {string} boardId the ID of the board we are exporting
-   * @param {string} authToken the loginToken
-   * @param {string} delimiter delimiter to use while building export. Default is comma ','
-   */
-  Picker.route('/api/boards/:boardId/export/csv', function(params, req, res) {
-    const boardId = params.boardId;
-    let user = null;
-    const loginToken = params.query.authToken;
-    if (loginToken) {
-      const hashToken = Accounts._hashLoginToken(loginToken);
-      user = Meteor.users.findOne({
-        'services.resume.loginTokens.hashedToken': hashToken,
+// exporter maybe is broken since Gridfs introduced, add fs and path
+
+export class Exporter {
+  constructor(boardId) {
+    this._boardId = boardId;
+  }
+
+  build() {
+    const fs = Npm.require('fs');
+    const os = Npm.require('os');
+    const path = Npm.require('path');
+
+    const byBoard = { boardId: this._boardId };
+    const byBoardNoLinked = {
+      boardId: this._boardId,
+      linkedId: { $in: ['', null] },
+    };
+    // we do not want to retrieve boardId in related elements
+    const noBoardId = {
+      fields: {
+        boardId: 0,
+      },
+    };
+    const result = {
+      _format: 'wekan-board-1.0.0',
+    };
+    _.extend(
+      result,
+      Boards.findOne(this._boardId, {
+        fields: {
+          stars: 0,
+        },
+      }),
+    );
+    result.lists = Lists.find(byBoard, noBoardId).fetch();
+    result.cards = Cards.find(byBoardNoLinked, noBoardId).fetch();
+    result.swimlanes = Swimlanes.find(byBoard, noBoardId).fetch();
+    result.customFields = CustomFields.find(
+      { boardIds: { $in: [this.boardId] } },
+      { fields: { boardId: 0 } },
+    ).fetch();
+    result.comments = CardComments.find(byBoard, noBoardId).fetch();
+    result.activities = Activities.find(byBoard, noBoardId).fetch();
+    result.rules = Rules.find(byBoard, noBoardId).fetch();
+    result.checklists = [];
+    result.checklistItems = [];
+    result.subtaskItems = [];
+    result.triggers = [];
+    result.actions = [];
+    result.cards.forEach(card => {
+      result.checklists.push(
+        ...Checklists.find({
+          cardId: card._id,
+        }).fetch(),
+      );
+      result.checklistItems.push(
+        ...ChecklistItems.find({
+          cardId: card._id,
+        }).fetch(),
+      );
+      result.subtaskItems.push(
+        ...Cards.find({
+          parentId: card._id,
+        }).fetch(),
+      );
+    });
+    result.rules.forEach(rule => {
+      result.triggers.push(
+        ...Triggers.find(
+          {
+            _id: rule.triggerId,
+          },
+          noBoardId,
+        ).fetch(),
+      );
+      result.actions.push(
+        ...Actions.find(
+          {
+            _id: rule.actionId,
+          },
+          noBoardId,
+        ).fetch(),
+      );
+    });
+
+    // [Old] for attachments we only export IDs and absolute url to original doc
+    // [New] Encode attachment to base64
+
+    const getBase64Data = function(doc, callback) {
+      let buffer = Buffer.allocUnsafe(0);
+      buffer.fill(0);
+
+      // callback has the form function (err, res) {}
+      const tmpFile = path.join(
+        os.tmpdir(),
+        `tmpexport${process.pid}${Math.random()}`,
+      );
+      const tmpWriteable = fs.createWriteStream(tmpFile);
+      const readStream = fs.createReadStream(doc.path);
+      readStream.on('data', function(chunk) {
+        buffer = Buffer.concat([buffer, chunk]);
       });
-    } else if (!Meteor.settings.public.sandstorm) {
-      Authentication.checkUserId(req.userId);
-      user = Users.findOne({
-        _id: req.userId,
-        isAdmin: true,
+
+      readStream.on('error', function(err) {
+        callback(null, null);
       });
-    }
-    const exporter = new Exporter(boardId);
-    if (exporter.canExport(user)) {
-      body = params.query.delimiter
-        ? exporter.buildCsv(params.query.delimiter)
-        : exporter.buildCsv();
-      res.writeHead(200, {
-        'Content-Length': body[0].length,
-        'Content-Type': params.query.delimiter ? 'text/csv' : 'text/tsv',
+      readStream.on('end', function() {
+        // done
+        fs.unlink(tmpFile, () => {
+          //ignored
+        });
+
+        callback(null, buffer.toString('base64'));
       });
-      res.write(body[0]);
-      res.end();
-    } else {
-      res.writeHead(403);
-      res.end('Permission Error');
-    }
-  });
+      readStream.pipe(tmpWriteable);
+    };
+    const getBase64DataSync = Meteor.wrapAsync(getBase64Data);
+    result.attachments = Attachments.find({ 'meta.boardId': byBoard.boardId })
+      .fetch()
+      .map(attachment => {
+        let filebase64 = null;
+        filebase64 = getBase64DataSync(attachment);
+
+        return {
+          _id: attachment._id,
+          cardId: attachment.meta.cardId,
+          //url: FlowRouter.url(attachment.url()),
+          file: filebase64,
+          name: attachment.name,
+          type: attachment.type,
+        };
+      });
+
+    // we also have to export some user data - as the other elements only
+    // include id but we have to be careful:
+    // 1- only exports users that are linked somehow to that board
+    // 2- do not export any sensitive information
+    const users = {};
+    result.members.forEach(member => {
+      users[member.userId] = true;
+    });
+    result.lists.forEach(list => {
+      users[list.userId] = true;
+    });
+    result.cards.forEach(card => {
+      users[card.userId] = true;
+      if (card.members) {
+        card.members.forEach(memberId => {
+          users[memberId] = true;
+        });
+      }
+    });
+    result.comments.forEach(comment => {
+      users[comment.userId] = true;
+    });
+    result.activities.forEach(activity => {
+      users[activity.userId] = true;
+    });
+    result.checklists.forEach(checklist => {
+      users[checklist.userId] = true;
+    });
+    const byUserIds = {
+      _id: {
+        $in: Object.getOwnPropertyNames(users),
+      },
+    };
+    // we use whitelist to be sure we do not expose inadvertently
+    // some secret fields that gets added to User later.
+    const userFields = {
+      fields: {
+        _id: 1,
+        username: 1,
+        'profile.fullname': 1,
+        'profile.initials': 1,
+        'profile.avatarUrl': 1,
+      },
+    };
+    result.users = Users.find(byUserIds, userFields)
+      .fetch()
+      .map(user => {
+        // user avatar is stored as a relative url, we export absolute
+        if ((user.profile || {}).avatarUrl) {
+          user.profile.avatarUrl = FlowRouter.url(user.profile.avatarUrl);
+        }
+        return user;
+      });
+    return result;
+  }
+
+  canExport(user) {
+    const board = Boards.findOne(this._boardId);
+    return board && board.isVisibleBy(user);
+  }
 }
diff --git a/models/trelloCreator.js b/models/trelloCreator.js
index 1c5bcd93..c4be140b 100644
--- a/models/trelloCreator.js
+++ b/models/trelloCreator.js
@@ -369,6 +369,7 @@ export class TrelloCreator {
           // so we make it server only, and let UI catch up once it is done, forget about latency comp.
           const self = this;
           if (Meteor.isServer) {
+            // FIXME: Change to new model
             file.attachData(att.url, function(error) {
               file.boardId = boardId;
               file.cardId = cardId;
diff --git a/models/wekanCreator.js b/models/wekanCreator.js
index 9914f817..c5591a0b 100644
--- a/models/wekanCreator.js
+++ b/models/wekanCreator.js
@@ -415,6 +415,7 @@ export class WekanCreator {
           const self = this;
           if (Meteor.isServer) {
             if (att.url) {
+              // FIXME: Change to new file library
               file.attachData(att.url, function(error) {
                 file.boardId = boardId;
                 file.cardId = cardId;
@@ -440,6 +441,7 @@ export class WekanCreator {
                 }
               });
             } else if (att.file) {
+              // FIXME: Change to new file library
               file.attachData(
                 Buffer.from(att.file, 'base64'),
                 {