rural-attractivness-service/nuts-data.js

const fs = require('fs');
const csv = require('csv-parse');

/* Helper method to load the datasets from CSV and store it in server object */
module.exports.loadDatasets = function(filePath, dataLoadedCallback) {
    //console.log('Datasets structure loading.');
    var datasets = [];

    let columns = undefined;

    fs.createReadStream(filePath)
        .pipe(csv({ separator: ';' }))
        .on('data', (row) => {
            if (!columns) {
                columns = row;
            }
            else {
                let ds = {};

                for (let i = 0; i < columns.length; i++) {
                    ds[columns[i]] = row[i];
                }
                datasets.push(ds);
            }
        })
        .on('end', () => {
            //console.log('Datasets structure loaded.');
            dataLoadedCallback(datasets);
        });
}

/* Load all the attractivness data from CSV into a server object.
   The data don't need to be loaded for each request then. */
module.exports.loadRuralData = function (filePath, datasets, dataLoadedCallback) {
    console.log('Reading rural data file processing started.');
    var ruralData = [];

    let columns = undefined;

    fs.createReadStream(filePath)
        .pipe(csv())
        .on('data', (row) => {
            if (!columns) {
                columns = row;
            }
            else {
                let item = {};

                for (let i = 0; i < columns.length; i++) {
                    let colName = columns[i].toLowerCase();

                    if (colName == "nuts_id")       // ID of the NUTS region
                        item.nuts = row[i];
                    else if (colName == "datasets") // empty datasets count
                        item.availableDS = datasets.length - row[i];
                    else if (colName == "quality")
                        item.quality = row[i];
                    else {
                        let factor = getDataSetFactor(datasets, colName);
                        if (factor) {
                            if (!item[factor])
                                item[factor] = {};

                            //item[factor].push({ dataset: columns[i], value: row[i] });
                            item[factor][columns[i]] = Number(row[i]);
                        }
                    }
                }
                ruralData.push(item);
            }
        })
        .on('end', () => {
            //console.log('Rural data file processing finished.');
            dataLoadedCallback(ruralData);
        });
}

/* Reads the out_file.csv created by R script and saves it into an object
*/
module.exports.loadClusters = function (filePath, dataLoadedCallback) {
    console.log('Reading clustering data file processing started.');
    let clusters = [];

    let columns = undefined;

    fs.createReadStream(filePath)
        .pipe(csv())
        .on('data', (row) => {
            if (!columns) {
                columns = row;
            }
            else {
                let item = {};
                for (let i = 0; i < columns.length; i++) {
                    const colName = columns[i].length > 0 ? columns[i].toLowerCase() : 'nuts_id';
                    item[colName] = row[i];
                }
                clusters.push(item);
            }
        })
        .on('end', () => {
            //console.log('Rural data file processing finished.');
            dataLoadedCallback(clusters);
        });
}

module.exports.getFactorIndex = function (region, factor) {
    //console.log('getFactorIndex');
    //console.log('region: ' + region.nuts);
    //console.log('factor: ' + JSON.stringify(factor, null, 4));

    sumValue = 0;
    count = 0;

    factor.datasets.forEach(ds => {
        //console.log('factor: ' + factor.factor);

        let value = region[factor.factor][ds];
        if (value) {
            sumValue += value;
            count++;
        }
    });

    return { index: sumValue / count, sumValue: sumValue, sumWeight: count * factor.weight };
}

function getDataSetFactor(datasets, colName) {

    for (let i = 0; i < datasets.length; i++) {
        if (datasets[i].Name.toLowerCase() == colName)
            return datasets[i].Factor;
    }

    return undefined;
}