More changes for db migration

migrate/migrate-activations.js

@@ -0,0 +1,75 @@
+import co from 'co';
+
+import models from '../models';
+import { mongo } from '../lib/mongo';
+
+import debugname from 'debug';
+const debug = debugname('hostr:db');
+let db;
+
+
+co(function *sync() {
+  debug('Syncing schema');
+  yield models.sequelize.sync();
+  debug('Schema synced');
+  db = yield mongo;
+  const users = yield db.Users.find({}, { sort: [['joined', 'asc']] }).toArray();
+  for (const user of users) {
+    if (user.joined === '0') {
+      const file = yield db.Files.findOne({
+        owner: user._id,
+      }, {
+        limit: 1,
+        sort: [['time_added', 'asc']],
+      });
+      if (file && file.time_added > 0) {
+        user.createdAt = new Date(file.time_added * 1000).getTime();
+      } else {
+        user.createdAt = new Date().getTime();
+      }
+    } else {
+      user.createdAt = new Date(user.joined * 1000).getTime();
+    }
+  }
+
+  users.sort((a, b) => (a.createdAt < b.createdAt ? -1 : 1));
+
+  for (const user of users) {
+    if (!user.email) {
+      continue;
+    }
+
+    const exists = yield models.user.findOne({
+      where: {
+        email: user.email,
+      },
+    });
+
+    if (exists) {
+      debug('User exists, continue');
+      continue;
+    }
+
+    const oldId = user._id.toString();
+
+    const newUser = yield models.user.create({
+      email: user.email,
+      password: user.salted_password,
+      name: user.first_name ? `${user.first_name} ${user.last_name}` : null,
+      plan: user.type || 'Free',
+      activated: !user.activationCode,
+      banned: !!user.banned,
+      deletedAt: user.status === 'deleted' ? new Date().getTime() : null,
+      createdAt: user.createdAt,
+      updatedAt: user.createdAt,
+      oldId,
+    });
+    yield newUser.save({ silent: true });
+  }
+  models.sequelize.close();
+  db.close();
+}).catch((err) => {
+  models.sequelize.close();
+  db.close();
+  debug(err);
+});

migrate/migrate-files.js

@@ -0,0 +1,80 @@
+import co from 'co';
+import validateIp from 'validate-ip';
+
+import models from '../models';
+import { mongo } from '../lib/mongo';
+
+import debugname from 'debug';
+const debug = debugname('hostr:db');
+let db;
+
+
+co(function *sync() {
+  debug('Syncing schema');
+  yield models.sequelize.sync();
+  debug('Schema synced');
+  db = yield mongo;
+  const users = yield models.user.findAll({});
+  const userIds = {};
+  debug('remap');
+  for (const user of users) {
+    userIds[user._id] = user.id;
+  }
+  debug('remap done');
+  let files;
+  try {
+    files = db.Files.find({}, {
+      sort: [['time_added', 'desc']],
+      skip: 0,
+    });
+  } catch (err) {
+    debug(err);
+  }
+  debug('fetched files');
+
+  while (true) {
+    const file = yield files.next();
+    if (!file) {
+      break;
+    }
+    if (!file.time_added || !file.file_size) {
+      continue;
+    }
+    let ip = file.ip ? file.ip.split(',').pop().trim() : null;
+
+    if (typeof ip !== 'string' || !validateIp(ip)) {
+      ip = null;
+    }
+    const processed = file.status !== 'uploading';
+    const accessedAt = file.last_accessed ? new Date(file.last_accessed * 1000) : null;
+
+    yield models.file.upsert({
+      id: file._id.toString(),
+      name: file.file_name,
+      originalName: file.original_name || file.file_name,
+      size: file.file_size,
+      downloads: file.downloads,
+      deletedAt: file.status === 'deleted' ? new Date() : null,
+      createdAt: new Date(file.time_added * 1000),
+      updatedAt: new Date(file.time_added * 1000),
+      accessedAt,
+      processed,
+      type: file.type !== 'file' ? file.type : 'other',
+      width: Number.isInteger(file.width) ? file.width : null,
+      height: Number.isInteger(file.height) ? file.height : null,
+      userId: file.owner !== undefined ? userIds[file.owner] : null,
+      ip,
+      legacyId: file.system_name !== file._id ? file.system_name : null,
+      md5: file.md5,
+      malwarePositives: file.virustotal && file.virustotal.positives > 0 ?
+      file.virustotal.positives : null,
+    }, { /* logging: false */ });
+  }
+
+  models.sequelize.close();
+  db.close();
+}).catch((err) => {
+  models.sequelize.close();
+  db.close();
+  debug(err);
+});

migrate/migrate-logins.js

@@ -0,0 +1,54 @@
+import co from 'co';
+import validateIp from 'validate-ip';
+
+import models from '../models';
+import { mongo } from '../lib/mongo';
+
+import debugname from 'debug';
+const debug = debugname('hostr:db');
+let db;
+
+
+co(function *sync() {
+  debug('Syncing schema');
+  yield models.sequelize.sync();
+  debug('Schema synced');
+  db = yield mongo;
+  const users = yield models.user.findAll({});
+  const userIds = {};
+  debug('remap');
+  for (const user of users) {
+    userIds[user._id] = user.id;
+  }
+  debug('remap done');
+  let logins;
+  try {
+    logins = db.Logins.find({}, {
+      skip: 0,
+    });
+  } catch (err) {
+    debug(err);
+  }
+  debug('fetched logins');
+
+  while (true) {
+    const login = yield logins.next();
+    if (!login) {
+      break;
+    }
+
+    const newLogin = yield models.login.create({
+      ip: login.ip,
+      createdAt: login.at * 1000,
+      successful: login.successful,
+    }, { /* logging: false */ });
+    newLogin.save();
+  }
+
+  models.sequelize.close();
+  db.close();
+}).catch((err) => {
+  models.sequelize.close();
+  db.close();
+  debug(err);
+});

migrate/migrate-malwares.js

@@ -0,0 +1,46 @@
+import co from 'co';
+
+import models from '../models';
+import { mongo } from '../lib/mongo';
+
+import debugname from 'debug';
+const debug = debugname('hostr:db');
+let db;
+
+
+co(function *sync() {
+  debug('Syncing schema');
+  yield models.sequelize.sync();
+  debug('Schema synced');
+  db = yield mongo;
+
+  const files = db.Files.find({}, {
+    sort: [['time_added', 'desc']],
+    skip: 0,
+  });
+  debug('fetched files');
+
+  while (true) {
+    const file = yield files.next();
+    if (!file) {
+      break;
+    }
+
+    if (!file.time_added || !file.file_size || !file.malware) {
+      continue;
+    }
+
+    yield models.malware.upsert({
+      fileId: file._id,
+      positives: file.virustotal ? file.virustotal.positives : 100,
+      virustotal: file.virustotal || null,
+    }, { /* logging: false */ });
+  }
+
+  models.sequelize.close();
+  db.close();
+}).catch((err) => {
+  models.sequelize.close();
+  db.close();
+  debug(err);
+});

migrate/migrate-users.js

@@ -0,0 +1,77 @@
+import co from 'co';
+
+import models from '../models';
+import { mongo } from '../lib/mongo';
+
+import debugname from 'debug';
+const debug = debugname('hostr:db');
+let db;
+
+
+co(function *sync() {
+  debug('Syncing schema');
+  yield models.sequelize.sync();
+  debug('Schema synced');
+  db = yield mongo;
+  const users = yield db.Users.find({}, { sort: [['joined', 'asc']] }).toArray();
+  for (const user of users) {
+    if (user.joined === '0') {
+      const file = yield db.Files.findOne({
+        owner: user._id,
+      }, {
+        limit: 1,
+        sort: [['time_added', 'asc']],
+      });
+      if (file && file.time_added > 0) {
+        user.createdAt = new Date(file.time_added * 1000).getTime();
+      } else {
+        user.createdAt = new Date().getTime();
+      }
+    } else {
+      user.createdAt = new Date(user.joined * 1000).getTime();
+    }
+  }
+
+  users.sort((a, b) => (a.createdAt < b.createdAt ? -1 : 1));
+
+  for (const user of users) {
+    if (!user.email) {
+      continue;
+    }
+
+    const exists = yield models.user.findOne({
+      where: {
+        email: user.email,
+      },
+    });
+
+    if (exists) {
+      debug('User exists, continue');
+      continue;
+    }
+
+    const oldId = user._id.toString();
+
+    const newUser = yield models.user.create({
+      email: user.email,
+      password: user.salted_password,
+      name: user.first_name ? `${user.first_name} ${user.last_name}` : null,
+      plan: user.type || 'Free',
+      activated: !user.activationCode,
+      banned: !!user.banned,
+      deletedAt: user.status === 'deleted' ? new Date().getTime() : null,
+      createdAt: user.createdAt,
+      updatedAt: user.createdAt,
+      oldId,
+    }, {
+      include: [models.activation],
+    });
+    yield newUser.save({ silent: true });
+  }
+  models.sequelize.close();
+  db.close();
+}).catch((err) => {
+  models.sequelize.close();
+  db.close();
+  debug(err);
+});