Merge pull request #2 from memezinga/development

v1.0.0-MVP

README.md

@@ -1,2 +1,74 @@
-# memegeddon
-Herramienta para adquisición y enriquecimiento de plantillas de meme :muscle:
+![header](https://raw.githubusercontent.com/OSWeekends/agile-project-template/master/other/img/OSW-project-GitHub-template-header.jpg)
+
+
+![travis](https://img.shields.io/travis/memezinga/memegeddon.svg)
+![issues abiertos](https://img.shields.io/github/issues/memezinga/memegeddon.svg)
+![PR Abiertos](https://img.shields.io/github/issues-pr/memezinga/memegeddon.svg)
+![último commit](https://img.shields.io/github/last-commit/memezinga/memegeddon.svg)
+![TOP Lang](https://img.shields.io/github/languages/top/memezinga/memegeddon.svg)
+![total lang](https://img.shields.io/github/languages/count/memezinga/memegeddon.svg)
+
+# Memegeddon
+
+> Herramienta para adquisición y enriquecimiento de plantillas de memes para Memezinga :muscle::muscle::muscle:
+
+
+**Motivación**
+
+Generar una heramienta que permita recopilar memes de internet e introduccirlos en la base de datos y ficheros de memezinga de una forma independiente
+
+### Equipo
+
+ - [Ulises Gascón(@ulisesgascon)](https://github.com/ulisesgascon) (Leader)
+ - [Theba Gomez (@KoolTheba)](https://github.com/integrante1) (Contributor)
+ - [Fabiola Vieyra (@Fa-v)](https://github.com/Fa-v) (Contributor)
+
+
+
+##### Necesitamos
+
+_No tenemos posiciones abiertas por el momento_
+
+### Demo
+
+_No tenemos demo disponible por el momento_
+
+### Tecnología utilizada
+
+#### Dependencias
+- **Plataforma de Github**: Para facilitar el clonado
+- **Markdown**: Para mejorar la sintaxis y la semantica
+
+### Cómo contribuir en el proyecto
+
+
+_Nás información en breve_
+
+### ¿Cómo usarlo?.
+
+#### TL:DR;
+
+1. Es necesario modificar `/secrets/secrets.json` y `config.js` para que funcione. 
+2. Deberías tener un proyecto en Firebase habilitado para usar Memegeddon.
+
+_Nás información en breve_
+
+### Estado del proyecto.
+
+_Desarrollo activo. Entrega prevista para el Lunes 01 de Febrero de 2019_
+
+
+### Releases anteriores
+
+_Primera release en desarrollo activo_
+
+### Licencia
+
+GNU General Public License v3.0
+
+
+![footer](https://raw.githubusercontent.com/OSWeekends/agile-project-template/master/other/img/OSW-project-GitHub-template-footer.jpg)
+
+
+
+

config.js

@@ -0,0 +1,22 @@
+module.exports = {
+    paths: {
+        memes: `${__dirname}/memes`,
+        data: `${__dirname}/data`,
+        backup: `${__dirname}/backup`,
+    },
+    firebase: {
+        apiKey: "",
+        authDomain: "",
+        databaseURL: "",
+        projectId: "",
+        storageBucket: "",
+        messagingSenderId: ""
+    },
+    gcs: {
+       projectId: '',
+       bucketName: '',
+       memesPath: ''
+    },
+    delayPerPage: 4000,
+    cleanUpAndBackup: true
+};

lib/database.js

@@ -0,0 +1,19 @@
+const firebase = require("firebase"),
+    config = require('../config');
+
+firebase.initializeApp(config.firebase);
+const db = firebase.database();
+const memesDbRef = db.ref("memes");
+
+function saveMeme (meme){
+    return new Promise((resolve, reject) => {
+        memesDbRef.child(meme.id).set(meme, err => {
+            console.log(`[Memegeddon][Firebase][INFO] processed ${meme.id}`);
+            err ? reject(err) : resolve(meme);
+        });
+    });
+}
+
+exports.saveMemes = memes => {
+    return Promise.all(memes.map(saveMeme));
+};

lib/digester.js

@@ -0,0 +1,83 @@
+const request = require('request'),
+    {filter} = require('asyncro'),
+    EventEmitter = require('events'),
+    {sleep, isBetween, idGenerator} = require('./utils');
+
+const downloadEmitter = new EventEmitter();
+
+function harmonizeMemes (memes) {
+    return memes.map(meme => {
+        meme.id_src = meme.ID;
+        meme.id = idGenerator();
+        meme.img_src = meme.image;
+        meme.tags = meme.tags ? meme.tags.split(", ") : [];
+        delete meme.ID;
+        delete meme.image;
+        return meme;
+    });
+}
+
+function validUrl (url){
+    return new Promise ((resolve) => {
+        request(url, (error, response, body) => {
+            if(error || !isBetween(response.statusCode, 200, 299)) {
+                console.log(`[Memegeddon][Download][Check-url][INFO] discarted ${url}`);
+                resolve(false);
+            } else {
+                console.log(`[Memegeddon][Download][Check-url][INFO] validated ${url}`);
+                resolve(true);
+            }
+        });
+    });
+}
+
+function getMemePage (baseUrl, page=1) {
+    const url = `${baseUrl}/${page}`;
+    console.log(`[Memegeddon][Download][INFO] Start meme data for page ${page}`);
+    return new Promise((resolve, reject) => {
+        request(url, (error, response, body) => {
+            if(error || !isBetween(response.statusCode, 200, 299)) {
+                reject(error);
+            } else {
+                const data = JSON.parse(body);
+                console.log(`[Memegeddon][Download][INFO] meme data downloaded. Total: ${data.data.length}. Page: ${page}`);
+                resolve(data);
+            }
+        });
+    });
+}
+
+exports.getMemesDb =  function (url, delay=0) {
+    setTimeout(async ()=>{
+        let page = 1;
+        let res = false;
+        let all = [];
+        do {
+            try {
+                if(delay){
+                    console.log(`[Memegeddon][Download][INFO] Sleep for ${delay}ms`);
+                    await sleep(delay);
+                }
+                res = await getMemePage(url, page);
+                const data = res.data;
+                if(data.length > 0) {
+                    const validMemes = await filter(res.data, async meme => validUrl(meme.image));
+                    const memes = harmonizeMemes(validMemes)
+                    all = all.concat(memes);
+                    console.log(`[Memegeddon][Download][INFO] Memes valid data for ${validMemes.length} items`);
+                    downloadEmitter.emit("data", memes);
+                }
+
+                page++;
+
+            } catch (e) {
+                console.log(`[Memegeddon][Download][ERROR] ${e}`);
+                downloadEmitter.emit("error", e);
+            }
+        } while(res.next);
+
+        console.log(`[Memegeddon][Download][INFO] Meme download tasks has ended`);
+        downloadEmitter.emit("end", all);
+    }, delay);
+    return downloadEmitter;
+};

lib/downloader.js

@@ -0,0 +1,48 @@
+const request = require('request'),
+    mime = require('mime'),
+    fs = require('fs'),
+    {paths} = require('../config');
+
+function getMetadata (meme){
+    return new Promise((resolve, reject) => {
+        console.log(`[Memegeddon][Download][IMG][INFO] Metadata adquisition ${meme.img_src}`);
+
+        request.head(meme.img_src, (err, res, body) => {
+            if(err) {
+                reject(err);
+            } else {
+                const contentType = res.headers['content-type'];
+                meme["content-type"] = contentType;
+                meme.extension = mime.getExtension(contentType);
+                resolve(meme);
+            }
+        });
+
+
+    });
+}
+
+function getRequest(meme){
+    const filePath = `${paths.memes}/${meme.id}.${meme.extension}`;
+    return new Promise((resolve, reject) => {
+        console.log(`[Memegeddon][Download][IMG][INFO] File adquisition ${meme.img_src} to ${filePath}`);
+        request(meme.img_src)
+            .pipe(fs.createWriteStream(filePath))
+            .on('close', () => {
+                resolve(meme);
+            })
+            .on('error', () => reject());
+    });
+}
+
+
+
+function downloadAllMemes (memes) {
+    return Promise.all(memes.map(downloadMeme));
+}
+
+function downloadMeme (meme) {
+    return getMetadata(meme).then(getRequest);
+}
+
+module.exports = {downloadMeme, downloadAllMemes};

lib/uploader.js

@@ -0,0 +1,49 @@
+const {Storage} = require('@google-cloud/storage'),
+    {gcs, paths} = require('../config');
+
+const projectId = gcs.projectId,
+    bucketName = gcs.bucketName,
+    bucketUrl = `gs://${bucketName}`,
+    storage = new Storage({projectId});
+
+async function makePublicFile (meme) {
+    return new Promise((resolve, reject) => {
+    storage
+        .bucket(bucketUrl)
+        .file(`${gcs.memesPath}/${meme.id}`)
+        .makePublic()
+        .then(()=>{
+            meme.url = `https://storage.googleapis.com/${bucketName}/${gcs.memesPath}/${meme.id}`;
+            console.log(`[Memegeddon][UPLOADER][INFO] Meme ${meme.id} is public now as ${meme.url}`);
+            resolve(meme);
+        })
+        .catch(reject);
+    });
+}
+
+
+function uploadFile (meme) {
+    const filePath = `${paths.memes}/${meme.id}.${meme.extension}`;
+    const destination = `${gcs.memesPath}/${meme.id}`;
+    return new Promise((resolve, reject) => {
+        storage
+            .bucket(bucketUrl)
+            .upload(filePath, {destination})
+            .then(() => {
+                console.log(`[Memegeddon][UPLOADER][INFO] Uploaded Meme ${meme.id} to ${bucketUrl}`);
+                resolve(meme);
+            })
+            .catch(reject);
+    });
+}
+
+
+function uploadAllFiles (memes){
+    return Promise.all(memes.map(uploadFile));
+}
+
+function publicAllFiles (memes) {
+    return Promise.all(memes.map(makePublicFile));
+}
+
+module.exports = {uploadAllFiles, publicAllFiles};

lib/utils.js

@@ -0,0 +1,49 @@
+const archiver = require('archiver'),
+  rimraf = require('rimraf'),
+  uuidv4 = require('uuid/v4'),
+  { promisify } = require("util"),
+  fs = require('fs');
+
+exports.timestamp = () => new Date().getTime();
+
+exports.saveFile = promisify(fs.writeFile);
+
+exports.listFiles = promisify(fs.readdir);
+
+exports.getFile = promisify(fs.readFile);
+
+exports.removeFolder = promisify(rimraf);
+
+/**
+ * @param {String} source
+ * @param {String} out
+ * @returns {Promise}
+ * @see https://stackoverflow.com/a/51518100
+ */
+exports.zipDirectory = (source, out) => {
+  const archive = archiver('zip', { zlib: { level: 9 }});
+  const stream = fs.createWriteStream(out);
+
+  return new Promise((resolve, reject) => {
+    archive
+      .directory(source, false)
+      .on('error', err => reject(err))
+      .pipe(stream);
+
+    stream.on('close', () => resolve());
+    archive.finalize();
+  });
+};
+
+exports.idGenerator = function (){
+    return uuidv4(); 
+};
+
+exports.sleep = function (ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+};
+
+
+exports.isBetween = function (nbr, min, max) {
+  return nbr >= min && nbr <= max;
+};