Modify /clusters service to manipulate CSV data

index.js

@@ -8,6 +8,8 @@ const app = express();
 
 const _datasetsFilePath = 'data/datasets.csv';
 const _dataFilePath = 'data/data.csv';
+const _clusteringInputFilePath = 'data/clustering/input_all.csv';
+const _clusteringModifiedFilePath = 'data/clustering/input_modified.csv';
 const _clustersFilePath = 'data/clustering/out_file.csv';
 var _datasets = undefined;
 var _ruralData = undefined;
@@ -31,6 +33,7 @@ app.get('/refresh', (req, res, next) => {
         nutsData.loadRuralData(_dataFilePath, _datasets, function (rd) {
             //console.log('Rural data loaded succesfully');
             _ruralData = rd;
+            res.send('Data refreshed');
         });
     });
 });
@@ -132,6 +135,7 @@ app.post('/scores', (req, res, next) => {
 app.get('/runR', (req, res, next) => {
     //console.log(console);
     console.log('calling R...')
+    console.log(req)
     R('./r/selected_data.r').call(
         function(err, data) {
             console.log('R done');
@@ -149,27 +153,39 @@ app.get('/runR', (req, res, next) => {
 });
 
 /*
-    Calls R script, loads the resulting CSV file and returns it
+    Just informative response. POST with JSON data is required.
 */
 app.get('/clusters', (req, res, next) => {
-    //console.log(console);
-    console.log('calling R...')
-    R('./r/selected_data.r').call(
-        function(err, data) {
-            console.log('R done');
-            if (err) {
-                console.log(err.toString('utf8'));
-                data = { result: err.toString('utf8') };
-            }
-            else {
-                console.log(data);
-                nutsData.loadClusters(_clustersFilePath, function(clusterData) {
-                    data = clusterData;
-                    helpers.formatResponse({ response: data }, req, res);
-                });
-            }
+    const data = { response: '/clusters method is only available under POST' }
+    helpers.formatResponse(data, req, res);
+});
+
+/*
+    Modifies input CSV file, calls R script, loads the resulting CSV file and returns it
+*/
+app.post('/clusters', async (req, res, next) => {
+    try {
+        if (!_datasets) {
+            //TODO: promisify all functions to avoid callback hell and make this work properly
+            await nutsData.loadDatasets(_datasetsFilePath, function (ds) {
+                //console.log('Datasets loaded succesfully');
+                _datasets = ds;
+            })
         }
-    );
+        //console.log(req.body);
+        const clusteringData = await nutsData.loadClusteringInput(
+            _clusteringInputFilePath
+        );
+        await nutsData.modifyClusteringData({
+            datasets: _datasets,
+            data: clusteringData,
+            params: req.body,
+            outputFileName: _clusteringModifiedFilePath
+        });
+        handleRCall(req, res);
+    } catch (error) { // Catch errors in async functions
+        next(error);
+    }
 });
 
 // start the service on the port xxxx
@@ -216,3 +232,23 @@ function returnRegionScores(nuts, req, res) {
         // NUTS region not found
         res.status(404).send('NUTS region not found.');
 }
+
+function handleRCall(req, res) {
+    //console.log('calling R...')
+    R('./r/selected_data.r').call(
+        function(err, data) {
+            //console.log('R done');
+            if (err) {
+                console.log(err.toString('utf8'));
+                data = { result: err.toString('utf8') };
+            }
+            else {
+                //console.log(data);
+                nutsData.loadClusters(_clustersFilePath, function(clusterData) {
+                    data = clusterData;
+                    helpers.formatResponse({ response: data }, req, res);
+                });
+            }
+        }
+    );
+}

nuts-data.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
 const csv = require('csv-parse');
+const stringify = require('csv-stringify');
 
 /* Helper method to load the datasets from CSV and store it in server object */
 module.exports.loadDatasets = function(filePath, dataLoadedCallback) {
@@ -75,10 +76,83 @@ module.exports.loadRuralData = function (filePath, datasets, dataLoadedCallback)
         });
 }
 
-/* Reads the out_file.csv created by R script and saves it into an object
+/**
+ * Resolves with an array representing rows of CSV file
+ * @param {string} inputFileName path to the CSV file with input data for clustering calculation
+ */
+module.exports.loadClusteringInput = async function (inputFileName) {
+    const clusteringData = [];
+
+    /*
+     * The parsed CSV array keeps the native csv-parser structure
+     * for future easier serialization back to CSV file
+     */
+    return new Promise((resolve, reject) => {
+        fs.createReadStream(inputFileName)
+            .pipe(csv())
+            .on('data', (row) => {
+                clusteringData.push(row);
+            })
+            .on('end', () => {
+                resolve(clusteringData);
+            })
+            .on('error', reject);
+    });
+}
+
+/**
+ * Resolves once the modified CSV file is written to fs
+ */
+module.exports.modifyClusteringData = async function ({datasets, data, params, outputFileName}) {
+    let allowedDatasets = ['NUTS_ID']; // NUTS_ID must be copied to the output as well
+    for (const factor of params.factors) {
+        allowedDatasets = [...allowedDatasets, ...factor.datasets];
+    }
+    const factorMultipliers = data[0].map((dataset) => {
+        const factor = datasets.find(ds => ds.Name === dataset);
+        if (!factor) {
+            /* If the factor is unknown for this dataset, it will effectivelly turn it off */
+            console.log(`Undefined factor for dataset ${dataset}`);
+            allowedDatasets.filter(ds => ds !== dataset);
+            return 0;
+        } else if (!allowedDatasets.includes(dataset)) {
+            return 0;
+        } else {
+            return params.factors.find(f => f.factor === factor.Factor).weight;
+        }
+    })
+    //console.log(factorMultipliers);
+    /* The actual modification logic resides here */
+    const modifiedData = data.map((row, idx) => {
+        return row.map((value, i) => {
+            if (idx == 0) {
+                /* These are the headers */
+                return allowedDatasets.includes(value) ? value : null;
+            } else if (isNaN(value)) {
+                /* This is the NUTS ID record */
+                return value;
+            }
+            return factorMultipliers[i] === 0 ? null : value*factorMultipliers[i];
+        }).filter(val => val !== null);
+    });
+    //console.log(modifiedData);
+    return new Promise((resolve, reject) => {
+        stringify(modifiedData, (err, output) => {
+            if (err) return reject(err);
+            fs.writeFile(outputFileName, output, (err) => {
+                if (err) reject(err);
+                else resolve();
+                //console.log('Data modification finished.');
+            })
+        })
+    });
+}
+
+/** 
+ * Reads the out_file.csv created by R script and saves it into an object
 */
 module.exports.loadClusters = function (filePath, dataLoadedCallback) {
-    console.log('Reading clustering data file processing started.');
+    //console.log('Reading clustering data file processing started.');
     let clusters = [];
 
     let columns = undefined;
@@ -126,7 +200,6 @@ module.exports.getFactorIndex = function (region, factor) {
 }
 
 function getDataSetFactor(datasets, colName) {
-
     for (let i = 0; i < datasets.length; i++) {
         if (datasets[i].Name.toLowerCase() == colName)
             return datasets[i].Factor;

r/selected_data.r

@@ -2,8 +2,8 @@ setwd("./data/clustering/") # Nastavení pracovního adresáře (relativní k ro
 
 library(cluster)
 
-input <- read.csv(file = 'input_all.csv',header=TRUE,sep=",") # Načtení CSV souboru
-head(input) # Výpis prvních šesti řádek CSV souboru
+input <- read.csv(file = 'input_modified.csv',header=TRUE,sep=",") # Načtení CSV souboru
+#head(input) # Výpis prvních šesti řádek CSV souboru
 mydata <- input[, -1] # Úprava dat, která funguje, ale nevím proč... (ale jen pro data s více než jedním číselným sloupcem)
 rownames(mydata) <- input[, 1]
 mydata <- scale(mydata) # Standardizace dat

test.html

@@ -6,7 +6,7 @@ function loadDoc() {
   var xhttp = new XMLHttpRequest();
   xhttp.onreadystatechange = function() {
     if (this.readyState == 4 && this.status == 200) {
-     document.getElementById("demo").innerHTML = this.responseText;
+      document.getElementById("demo").innerHTML = this.responseText;
     }
   };
   xhttp.open("GET", "https://publish.lesprojekt.cz/nodejs/datasets", true);