Bugfix: 2560, 2604 - enable mixed mode mongodb attachment and filesystem attachment while reading

6 files changed, 180 insertions, 24 deletions

diff --git a/client/components/main/editor.js b/client/components/main/editor.js
index 735aac1e..e217e113 100755
--- a/client/components/main/editor.js
+++ b/client/components/main/editor.js
@@ -1,6 +1,4 @@
 import _sanitizeXss from 'xss';
-const enableRicherEditor =
-  Meteor.settings.public.RICHER_CARD_COMMENT_EDITOR || true;
 const sanitizeXss = (input, options) => {
   const defaultAllowedIframeSrc = /^(https:){0,1}\/\/.*?(youtube|vimeo|dailymotion|youku)/i;
   const allowedIframeSrcRegex = (function() {
@@ -104,7 +102,7 @@ Template.editor.onRendered(() => {
     autosize($textarea);
     $textarea.escapeableTextComplete(mentions);
   };
-  if (enableRicherEditor) {
+  if (Meteor.settings.public.RICHER_CARD_COMMENT_EDITOR !== false) {
     const isSmall = Utils.isMiniScreen();
     const toolbar = isSmall
       ? [
diff --git a/docker-compose.yml b/docker-compose.yml
index b8a7760c..e436f806 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -232,6 +232,10 @@ services:
       #- ACCOUNTS_LOCKOUT_UNKNOWN_USERS_LOCKOUT_PERIOD=60
       #- ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURE_WINDOW=15
       #---------------------------------------------------------------
+      # ==== STORE ATTACHMENT ON SERVER FILESYSTEM INSTEAD OF MONGODB ====
+      # https://github.com/wekan/wekan/pull/2603
+      #- ATTACHMENTS_STORE_PATH = <pathname> # pathname can be relative or fullpath
+      #---------------------------------------------------------------
       # ==== RICH TEXT EDITOR IN CARD COMMENTS ====
       # https://github.com/wekan/wekan/pull/2560
       - RICHER_CARD_COMMENT_EDITOR=true
diff --git a/models/attachments.js b/models/attachments.js
index 95265422..9b8ec04f 100644
--- a/models/attachments.js
+++ b/models/attachments.js
@@ -10,27 +10,169 @@ const defaultStoreOptions = {
     return {};
   },
 };
-const Store = localFSStore
-  ? new FS.Store.FileSystem(storeName, {
-      path: localFSStore,
-      ...defaultStoreOptions,
-    })
-  : new FS.Store.GridFS(storeName, {
-      // XXX Add a new store for cover thumbnails so we don't load big images in
-      // the general board view
-      // If the uploaded document is not an image we need to enforce browser
-      // download instead of execution. This is particularly important for HTML
-      // files that the browser will just execute if we don't serve them with the
-      // appropriate `application/octet-stream` MIME header which can lead to user
-      // data leaks. I imagine other formats (like PDF) can also be attack vectors.
-      // See https://github.com/wekan/wekan/issues/99
-      // XXX Should we use `beforeWrite` option of CollectionFS instead of
-      // collection-hooks?
-      // We should use `beforeWrite`.
-      ...defaultStoreOptions,
-    });
+let store;
+if (localFSStore) {
+  // have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem
+  const fs = Npm.require('fs');
+  const path = Npm.require('path');
+  const mongodb = Npm.require('mongodb');
+  const Grid = Npm.require('gridfs-stream');
+  // calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :(
+  let pathname = localFSStore;
+  /*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */
+
+  if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) {
+    pathname = path.join(
+      __meteor_bootstrap__.serverDir,
+      `../../../cfs/files/${storeName}`,
+    );
+  }
+
+  if (!pathname)
+    throw new Error('FS.Store.FileSystem unable to determine path');
+
+  // Check if we have '~/foo/bar'
+  if (pathname.split(path.sep)[0] === '~') {
+    const homepath =
+      process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
+    if (homepath) {
+      pathname = pathname.replace('~', homepath);
+    } else {
+      throw new Error('FS.Store.FileSystem unable to resolve "~" in path');
+    }
+  }
+
+  // Set absolute path
+  const absolutePath = path.resolve(pathname);
+
+  const _FStore = new FS.Store.FileSystem(storeName, {
+    path: localFSStore,
+    ...defaultStoreOptions,
+  });
+  const GStore = {
+    fileKey(fileObj) {
+      const key = {
+        _id: null,
+        filename: null,
+      };
+
+      // If we're passed a fileObj, we retrieve the _id and filename from it.
+      if (fileObj) {
+        const info = fileObj._getInfo(storeName, {
+          updateFileRecordFirst: false,
+        });
+        key._id = info.key || null;
+        key.filename =
+          info.name ||
+          fileObj.name({ updateFileRecordFirst: false }) ||
+          `${fileObj.collectionName}-${fileObj._id}`;
+      }
+
+      // If key._id is null at this point, createWriteStream will let GridFS generate a new ID
+      return key;
+    },
+    db: undefined,
+    mongoOptions: { useNewUrlParser: true },
+    mongoUrl: process.env.MONGO_URL,
+    init() {
+      this._init(err => {
+        this.inited = !err;
+      });
+    },
+    _init(callback) {
+      const self = this;
+      mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function(
+        err,
+        db,
+      ) {
+        if (err) {
+          return callback(err);
+        }
+        self.db = db;
+        return callback(null);
+      });
+      return;
+    },
+    createReadStream(fileKey, options) {
+      const self = this;
+      if (!self.inited) {
+        self.init();
+        return undefined;
+      }
+      options = options || {};
+
+      // Init GridFS
+      const gfs = new Grid(self.db, mongodb);
+
+      // Set the default streamning settings
+      const settings = {
+        _id: new mongodb.ObjectID(fileKey._id),
+        root: `cfs_gridfs.${storeName}`,
+      };
+
+      // Check if this should be a partial read
+      if (
+        typeof options.start !== 'undefined' &&
+        typeof options.end !== 'undefined'
+      ) {
+        // Add partial info
+        settings.range = {
+          startPos: options.start,
+          endPos: options.end,
+        };
+      }
+      return gfs.createReadStream(settings);
+    },
+  };
+  GStore.init();
+  const CRS = 'createReadStream';
+  const _CRS = `_${CRS}`;
+  const FStore = _FStore._transform;
+  FStore[_CRS] = FStore[CRS].bind(FStore);
+  FStore[CRS] = function(fileObj, options) {
+    let stream;
+    try {
+      const localFile = path.join(
+        absolutePath,
+        FStore.storage.fileKey(fileObj),
+      );
+      const state = fs.statSync(localFile);
+      if (state) {
+        stream = FStore[_CRS](fileObj, options);
+      }
+    } catch (e) {
+      // file is not there, try GridFS ?
+      stream = undefined;
+    }
+    if (stream) return stream;
+    else {
+      try {
+        const stream = GStore[CRS](GStore.fileKey(fileObj), options);
+        return stream;
+      } catch (e) {
+        return undefined;
+      }
+    }
+  }.bind(FStore);
+  store = _FStore;
+} else {
+  store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, {
+    // XXX Add a new store for cover thumbnails so we don't load big images in
+    // the general board view
+    // If the uploaded document is not an image we need to enforce browser
+    // download instead of execution. This is particularly important for HTML
+    // files that the browser will just execute if we don't serve them with the
+    // appropriate `application/octet-stream` MIME header which can lead to user
+    // data leaks. I imagine other formats (like PDF) can also be attack vectors.
+    // See https://github.com/wekan/wekan/issues/99
+    // XXX Should we use `beforeWrite` option of CollectionFS instead of
+    // collection-hooks?
+    // We should use `beforeWrite`.
+    ...defaultStoreOptions,
+  });
+}
 Attachments = new FS.Collection('attachments', {
-  stores: [Store],
+  stores: [store],
 });
 
 if (Meteor.isServer) {
diff --git a/package.json b/package.json
index 1ec9f127..67142418 100644
--- a/package.json
+++ b/package.json
@@ -59,8 +59,10 @@
     "bson": "^4.0.0",
     "bunyan": "^1.8.12",
     "es6-promise": "^4.2.4",
+    "gridfs-stream": "^0.5.3",
     "ldapjs": "^1.0.2",
     "meteor-node-stubs": "^0.4.1",
+    "mongodb": "^2.2.19",
     "os": "^0.1.1",
     "page": "^1.8.6",
     "qs": "^6.5.2",
diff --git a/snap-src/bin/config b/snap-src/bin/config
index 63a95d19..22e872f9 100755
--- a/snap-src/bin/config
+++ b/snap-src/bin/config
@@ -3,7 +3,7 @@
 # All supported keys are defined here together with descriptions and default values
 
 # list of supported keys
-keys="DEBUG MONGO_URL MONGODB_BIND_UNIX_SOCKET MONGO_URL MONGODB_BIND_IP MONGODB_PORT MAIL_URL MAIL_FROM ROOT_URL PORT DISABLE_MONGODB CADDY_ENABLED CADDY_BIND_PORT WITH_API RICHER_CARD_COMMENT_EDITOR ACCOUNTS_LOCKOUT_KNOWN_USERS_FAILURES_BEFORE ACCOUNTS_LOCKOUT_KNOWN_USERS_PERIOD ACCOUNTS_LOCKOUT_KNOWN_USERS_FAILURE_WINDOW ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURES_BERORE ACCOUNTS_LOCKOUT_UNKNOWN_USERS_LOCKOUT_PERIOD ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURE_WINDOW  MAX_IMAGE_PIXEL IMAGE_COMPRESS_RATIO BIGEVENTS_PATTERN NOTIFY_DUE_DAYS_BEFORE_AND_AFTER NOTIFY_DUE_AT_HOUR_OF_DAY EMAIL_NOTIFICATION_TIMEOUT CORS CORS_ALLOW_HEADERS CORS_EXPOSE_HEADERS MATOMO_ADDRESS MATOMO_SITE_ID MATOMO_DO_NOT_TRACK MATOMO_WITH_USERNAME BROWSER_POLICY_ENABLED TRUSTED_URL WEBHOOKS_ATTRIBUTES OAUTH2_ENABLED OAUTH2_LOGIN_STYLE OAUTH2_CLIENT_ID OAUTH2_SECRET OAUTH2_SERVER_URL OAUTH2_AUTH_ENDPOINT OAUTH2_USERINFO_ENDPOINT OAUTH2_TOKEN_ENDPOINT OAUTH2_ID_MAP OAUTH2_USERNAME_MAP OAUTH2_FULLNAME_MAP OAUTH2_ID_TOKEN_WHITELIST_FIELDS OAUTH2_EMAIL_MAP OAUTH2_REQUEST_PERMISSIONS LDAP_ENABLE LDAP_PORT LDAP_HOST LDAP_BASEDN LDAP_LOGIN_FALLBACK LDAP_RECONNECT LDAP_TIMEOUT LDAP_IDLE_TIMEOUT LDAP_CONNECT_TIMEOUT LDAP_AUTHENTIFICATION LDAP_AUTHENTIFICATION_USERDN LDAP_AUTHENTIFICATION_PASSWORD LDAP_LOG_ENABLED LDAP_BACKGROUND_SYNC LDAP_BACKGROUND_SYNC_INTERVAL LDAP_BACKGROUND_SYNC_KEEP_EXISTANT_USERS_UPDATED LDAP_BACKGROUND_SYNC_IMPORT_NEW_USERS LDAP_ENCRYPTION LDAP_CA_CERT LDAP_REJECT_UNAUTHORIZED LDAP_USER_AUTHENTICATION LDAP_USER_AUTHENTICATION_FIELD LDAP_USER_SEARCH_FILTER LDAP_USER_SEARCH_SCOPE LDAP_USER_SEARCH_FIELD LDAP_SEARCH_PAGE_SIZE LDAP_SEARCH_SIZE_LIMIT LDAP_GROUP_FILTER_ENABLE LDAP_GROUP_FILTER_OBJECTCLASS LDAP_GROUP_FILTER_GROUP_ID_ATTRIBUTE LDAP_GROUP_FILTER_GROUP_MEMBER_ATTRIBUTE LDAP_GROUP_FILTER_GROUP_MEMBER_FORMAT LDAP_GROUP_FILTER_GROUP_NAME LDAP_UNIQUE_IDENTIFIER_FIELD LDAP_UTF8_NAMES_SLUGIFY LDAP_USERNAME_FIELD LDAP_FULLNAME_FIELD LDAP_MERGE_EXISTING_USERS LDAP_SYNC_USER_DATA LDAP_SYNC_USER_DATA_FIELDMAP LDAP_SYNC_GROUP_ROLES LDAP_DEFAULT_DOMAIN LDAP_EMAIL_MATCH_ENABLE LDAP_EMAIL_MATCH_REQUIRE LDAP_EMAIL_MATCH_VERIFIED LDAP_EMAIL_FIELD LDAP_SYNC_ADMIN_STATUS LDAP_SYNC_ADMIN_GROUPS HEADER_LOGIN_ID HEADER_LOGIN_FIRSTNAME HEADER_LOGIN_LASTNAME HEADER_LOGIN_EMAIL LOGOUT_WITH_TIMER LOGOUT_IN LOGOUT_ON_HOURS LOGOUT_ON_MINUTES DEFAULT_AUTHENTICATION_METHOD"
+keys="DEBUG MONGO_URL MONGODB_BIND_UNIX_SOCKET MONGO_URL MONGODB_BIND_IP MONGODB_PORT MAIL_URL MAIL_FROM ROOT_URL PORT DISABLE_MONGODB CADDY_ENABLED CADDY_BIND_PORT WITH_API RICHER_CARD_COMMENT_EDITOR ACCOUNTS_LOCKOUT_KNOWN_USERS_FAILURES_BEFORE ACCOUNTS_LOCKOUT_KNOWN_USERS_PERIOD ACCOUNTS_LOCKOUT_KNOWN_USERS_FAILURE_WINDOW ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURES_BERORE ACCOUNTS_LOCKOUT_UNKNOWN_USERS_LOCKOUT_PERIOD ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURE_WINDOW  MAX_IMAGE_PIXEL IMAGE_COMPRESS_RATIO BIGEVENTS_PATTERN NOTIFY_DUE_DAYS_BEFORE_AND_AFTER NOTIFY_DUE_AT_HOUR_OF_DAY EMAIL_NOTIFICATION_TIMEOUT CORS CORS_ALLOW_HEADERS CORS_EXPOSE_HEADERS MATOMO_ADDRESS MATOMO_SITE_ID MATOMO_DO_NOT_TRACK MATOMO_WITH_USERNAME BROWSER_POLICY_ENABLED TRUSTED_URL WEBHOOKS_ATTRIBUTES OAUTH2_ENABLED OAUTH2_LOGIN_STYLE OAUTH2_CLIENT_ID OAUTH2_SECRET OAUTH2_SERVER_URL OAUTH2_AUTH_ENDPOINT OAUTH2_USERINFO_ENDPOINT OAUTH2_TOKEN_ENDPOINT OAUTH2_ID_MAP OAUTH2_USERNAME_MAP OAUTH2_FULLNAME_MAP OAUTH2_ID_TOKEN_WHITELIST_FIELDS OAUTH2_EMAIL_MAP OAUTH2_REQUEST_PERMISSIONS LDAP_ENABLE LDAP_PORT LDAP_HOST LDAP_BASEDN LDAP_LOGIN_FALLBACK LDAP_RECONNECT LDAP_TIMEOUT LDAP_IDLE_TIMEOUT LDAP_CONNECT_TIMEOUT LDAP_AUTHENTIFICATION LDAP_AUTHENTIFICATION_USERDN LDAP_AUTHENTIFICATION_PASSWORD LDAP_LOG_ENABLED LDAP_BACKGROUND_SYNC LDAP_BACKGROUND_SYNC_INTERVAL LDAP_BACKGROUND_SYNC_KEEP_EXISTANT_USERS_UPDATED LDAP_BACKGROUND_SYNC_IMPORT_NEW_USERS LDAP_ENCRYPTION LDAP_CA_CERT LDAP_REJECT_UNAUTHORIZED LDAP_USER_AUTHENTICATION LDAP_USER_AUTHENTICATION_FIELD LDAP_USER_SEARCH_FILTER LDAP_USER_SEARCH_SCOPE LDAP_USER_SEARCH_FIELD LDAP_SEARCH_PAGE_SIZE LDAP_SEARCH_SIZE_LIMIT LDAP_GROUP_FILTER_ENABLE LDAP_GROUP_FILTER_OBJECTCLASS LDAP_GROUP_FILTER_GROUP_ID_ATTRIBUTE LDAP_GROUP_FILTER_GROUP_MEMBER_ATTRIBUTE LDAP_GROUP_FILTER_GROUP_MEMBER_FORMAT LDAP_GROUP_FILTER_GROUP_NAME LDAP_UNIQUE_IDENTIFIER_FIELD LDAP_UTF8_NAMES_SLUGIFY LDAP_USERNAME_FIELD LDAP_FULLNAME_FIELD LDAP_MERGE_EXISTING_USERS LDAP_SYNC_USER_DATA LDAP_SYNC_USER_DATA_FIELDMAP LDAP_SYNC_GROUP_ROLES LDAP_DEFAULT_DOMAIN LDAP_EMAIL_MATCH_ENABLE LDAP_EMAIL_MATCH_REQUIRE LDAP_EMAIL_MATCH_VERIFIED LDAP_EMAIL_FIELD LDAP_SYNC_ADMIN_STATUS LDAP_SYNC_ADMIN_GROUPS HEADER_LOGIN_ID HEADER_LOGIN_FIRSTNAME HEADER_LOGIN_LASTNAME HEADER_LOGIN_EMAIL LOGOUT_WITH_TIMER LOGOUT_IN LOGOUT_ON_HOURS LOGOUT_ON_MINUTES DEFAULT_AUTHENTICATION_METHOD ATTACHMENTS_STORE_PATH METEOR_SETTINGS"
 
 # default values
 DESCRIPTION_DEBUG="Debug OIDC OAuth2 etc. Example: sudo snap set wekan debug='true'"
@@ -60,6 +60,10 @@ DESCRIPTION_WITH_API="Enable/disable the api of wekan"
 DEFAULT_WITH_API="true"
 KEY_WITH_API="with-api"
 
+DESCRIPTION_METEOR_SETTINGS="Meteor client settings, a JSON string, {\"public\":{}}"
+DEFAULT_METEOR_SETTINGS=""
+KEY_METEOR_SETTINGS="meteor-settings"
+
 DESCRIPTION_RICHER_CARD_COMMENT_EDITOR="Rich text editor in card comments. Default: true"
 DEFAULT_RICHER_CARD_COMMENT_EDITOR="true"
 KEY_RICHER_CARD_COMMENT_EDITOR="richer-card-comment-editor"
diff --git a/snap-src/bin/wekan-help b/snap-src/bin/wekan-help
index c662bf7e..5c405e1f 100755
--- a/snap-src/bin/wekan-help
+++ b/snap-src/bin/wekan-help
@@ -81,6 +81,12 @@ echo -e "\t$ snap set $SNAP_NAME image-compress-ratio='80'"
 echo -e "Disabled:"
 echo -e "\t$ snap set $SNAP_NAME image-compress-ratio=''"
 echo -e "\n"
+echo -e "Allow to set attachment upload into specified server location https://github.com/wekan/wekan/pull/2603"
+echo -e "Example:"
+echo -e "\t$ snap set $SNAP_NAME attachments-store-path='/tmp/attachments'"
+echo -e "Disabled:"
+echo -e "\t$ snap set $SNAP_NAME attachments-store-path=''"
+echo -e "\n"
 echo -e "BIGEVENTS DUE ETC NOTIFICATIONS https://github.com/wekan/wekan/pull/2541"
 echo -e "Big events pattern: Notify always due etc regardless of notification settings. Default: due, All: received|start|due|end, Disabled: NONE"
 echo -e "Default:"