Refactor datasets/ and scores/ requests

...for usage with ontology and variable schemas

index.js

@@ -42,10 +42,10 @@ app.get('/', (req, res) => {
 app.get('/refresh', async (req, res, next) => {
   try {
     _datasetsEU = await nutsData.loadDatasets(_datasetsEuFilePath)
-    _datasetsCZ = await nutsData.loadDatasets(_datasetsCzFilePath)
+    _datasetsCZ = await nutsData.loadDatasets(_dataCzFilePath)
     //console.log('Datasets loaded succesfully');
-    _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath, _datasetsEU)
-    _ruralDataCZ = await nutsData.loadRuralData(_datasetsCzFilePath, _datasetsCZ)
+    _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath)
+    _ruralDataCZ = await nutsData.loadRuralData(_dataCzFilePath)
     //console.log('Rural data loaded succesfully');
   } catch (e) {
     res.send(e.toString() || e)
@@ -71,7 +71,7 @@ app.get('/:aoi?/scores/:nuts', async (req, res, next) => {
   }
   if (!_ruralDataEU) {
     try {
-      _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath, _datasetsEU)
+      _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath)
     } catch (e) {
       res.send(e.toString() || e)
     }
@@ -94,7 +94,8 @@ app.post('/:aoi?/scores', async (req, res, next) => {
   const aoi = req.params.aoi || 'eu'
   await loadDatasetsIfNeeded(req, res, aoi)
   await loadRuralDataIfNeeded(req, res, aoi)
-  returnAllScores(req, res, aoi)
+  const resData = returnAllScores(req, res, aoi)
+  helpers.formatResponse(resData, req, res);
 });
 
 /*
@@ -175,7 +176,7 @@ app.get('/georeport/:nuts', async (req, res, next) => {
   }
   if (!_ruralDataEU) {
     try {
-      _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath, _datasetsEU)
+      _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath)
     } catch (e) {
       res.send(e.toString() || e)
     }
@@ -223,21 +224,21 @@ function returnAllScores(req, res, aoi = 'eu') {
     let sumValue = 0;
     let regionIndexes = { code: region[regionIdString] };
 
-    req.body.factors.forEach(f => {
-      let fi = nutsData.getFactorIndex(region, f);
-      //console.log("f: " + JSON.stringify(f));
-      //console.log("fi: " + JSON.stringify(fi));
+    req.body.factors.forEach(factor => {
+      let factIndex = nutsData.getFactorIndex(region, factor);
+      //console.log("f: " + JSON.stringify(factor));
+      //console.log("fi: " + JSON.stringify(factIndex));
 
-      regionIndexes[f.factor] = fi.index;
-      sumValue += fi.sumValue * f.weight;
-      sumWeight += fi.sumWeight;
+      regionIndexes[factor.factor] = factIndex.index;
+      sumValue += factIndex.sumValue * factor.weight;
+      sumWeight += factIndex.sumWeight;
     });
 
     regionIndexes.aggregate = sumValue / sumWeight;
     resData.push(regionIndexes);
   });
 
-  helpers.formatResponse(resData, req, res);
+  return resData;
 }
 
 function returnRegionScores(nuts, req, res) {
@@ -286,7 +287,7 @@ async function loadDatasetsIfNeeded(req, res, aoi) {
   if (!datasets) {
     try {
       if (aoi == 'cz') {
-        _datasetsCZ = await nutsData.loadDatasets(_datasetsCzFilePath)
+        _datasetsCZ = await nutsData.loadDatasets(_dataCzFilePath)
         datasets = _datasetsCZ
       } else {
         _datasetsEU = await nutsData.loadDatasets(_datasetsEuFilePath)
@@ -296,19 +297,19 @@ async function loadDatasetsIfNeeded(req, res, aoi) {
       res.send(e.toString() || e)
     }
   }
+  console.log(datasets)
   return datasets
 }
 
 async function loadRuralDataIfNeeded(req, res, aoi) {
   let ruralData = aoi == 'cz' ? _ruralDataCZ : _ruralDataEU
-  const datasets = aoi == 'cz' ? _datasetsCZ : _datasetsEU
   if (!ruralData) {
     try {
       if (aoi == 'cz') {
-        _ruralDataCZ = await nutsData.loadRuralData(_dataCzFilePath, datasets)
+        _ruralDataCZ = await nutsData.loadRuralData(_dataCzFilePath)
         ruralData = _ruralDataCZ
       } else {
-        _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath, datasets)
+        _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath)
         ruralData = _ruralDataEU
       }
     } catch (e) {

nuts-data.js

@@ -4,80 +4,65 @@ const stringify = require('csv-stringify');
 
 /* Helper method to load the datasets from CSV and store it in server object */
 module.exports.loadDatasets = async function(filePath) {
-    //console.log('Datasets structure loading.');
-    let datasets = [];
-
-    let columns = undefined;
+    console.log('Datasets structure loading.')
+    let datasets = undefined
 
     return new Promise((resolve, reject) => {
-    fs.createReadStream(filePath)
-        .pipe(csv({ separator: ';' }))
+    const stream = fs.createReadStream(filePath).pipe(csv())
+    stream
         .on('data', (row) => {
-            if (!columns) {
-                columns = row;
-            }
-            else {
-                let ds = {};
-
-                for (let i = 0; i < columns.length; i++) {
-                    ds[columns[i]] = row[i];
-                }
-                datasets.push(ds);
+            if (!datasets) {
+                datasets = row
             }
+            stream.end()
         })
         .on('end', () => {
-            //console.log('Datasets structure loaded.');
-            resolve(datasets);
+            datasets = datasets.slice(3)
+            console.log('Datasets structure loaded.')
+            resolve(datasets)
         })
-        .on('error', reject);
+        .on('error', reject)
     })
 }
 
 /* Load all the attractivness data from CSV into a server object.
    The data don't need to be loaded for each request then. */
-module.exports.loadRuralData = async function (filePath, datasets) {
-    console.log('Reading rural data file processing started.');
-    let ruralData = [];
-
-    let columns = undefined;
+module.exports.loadRuralData = async function (filePath) {
+    console.log('Reading rural data file processing started.')
+    let ruralData = []
+    let columns
 
     return new Promise((resolve, reject) => {
     fs.createReadStream(filePath)
         .pipe(csv())
         .on('data', (row) => {
             if (!columns) {
-                columns = row;
+                columns = row
+                return
             }
-            else {
-                let item = {};
-
-                for (let i = 0; i < columns.length; i++) {
-                    let colName = columns[i].toLowerCase();
-
-                    if (colName == "nuts_id")       // ID of the NUTS region
-                        item.nuts = row[i];
-                    else if (colName == "datasets") // empty datasets count
-                        item.availableDS = datasets.length - row[i];
-                    else if (colName == "quality")
-                        item.quality = row[i];
-                    else if (colName == "lau2")
-                        item.lau2 = row[i];
-                    else {
-                        let factor = getDataSetFactor(datasets, colName);
-                        if (factor) {
-                            if (!item[factor])
-                                item[factor] = {};
-
-                            //item[factor].push({ dataset: columns[i], value: row[i] });
-                            item[factor][columns[i]] = Number(row[i]);
-                        }
-                    }
+            let item = {
+                values: {}
+            }
+            for (let i = 0; i < columns.length; i++) {
+                let colName = columns[i].toLowerCase()
+                if (colName == "nuts_id")       // ID of the NUTS region
+                    item.nuts = row[i]
+                // else if (colName == "datasets") // empty datasets count
+                //     item.availableDS = datasets.length - row[i];
+                // else if (colName == "quality")
+                //     item.quality = row[i];
+                else if (colName == "lau2")
+                    item.lau2 = row[i]
+                else if (colName == "eurostat_code" || colName == "name")
+                    continue
+                else {
+                    item.values[colName] = Number(row[i])
                 }
-                ruralData.push(item);
             }
+            ruralData.push(item)
         })
         .on('end', () => {
-            //console.log('Rural data file processing finished.');
+            console.log('Rural data file processing finished.');
             resolve(ruralData);
         })
         .on('error', reject);
@@ -203,16 +188,17 @@ module.exports.loadClusters = function (filePath, idString, dataLoadedCallback)
 
 module.exports.getFactorIndex = function (region, factor) {
     //console.log('getFactorIndex');
-    //console.log('region: ' + region.nuts);
+    //console.log('region: ' + JSON.stringify(region, null, 4));
     //console.log('factor: ' + JSON.stringify(factor, null, 4));
 
     let sumValue = 0;
     let count = 0;
 
     factor.datasets.forEach(ds => {
+        const dataset = ds.split('/').slice(-1).pop()
         //console.log('factor: ' + factor.factor);
 
-        let value = region[factor.factor][ds];
+        const value = region.values[dataset];
         if (value) {
             sumValue += value;
             count++;
@@ -222,7 +208,8 @@ module.exports.getFactorIndex = function (region, factor) {
     return { index: sumValue / count, sumValue: sumValue, sumWeight: count * factor.weight };
 }
 
-function getDataSetFactor(datasets, colName) {
+/* Unused */
+function getDatasetFactor(datasets, colName) {
     for (let i = 0; i < datasets.length; i++) {
         if (datasets[i].Name.toLowerCase() == colName)
             return datasets[i].Factor;

package.json

@@ -4,7 +4,8 @@
   "description": "Web service for Rural Attractivness app",
   "main": "index.js",
   "scripts": {
-    "start": "nodemon --exec babel-node index.js"
+    "build": "babel index.js -d dist",
+    "start": "nodemon index.js"
   },
   "author": "František Zadražil",
   "license": "ISC",