rural-attractivness-service/nuts-data.js

const fs = require('fs');
const csv = require('csv-parse');
const stringify = require('csv-stringify');

/* Helper method to load the datasets from CSV and store it in server object */
module.exports.loadDatasets = function(filePath, dataLoadedCallback) {
    //console.log('Datasets structure loading.');
    var datasets = [];

    let columns = undefined;

    fs.createReadStream(filePath)
        .pipe(csv({ separator: ';' }))
        .on('data', (row) => {
            if (!columns) {
                columns = row;
            }
            else {
                let ds = {};

                for (let i = 0; i < columns.length; i++) {
                    ds[columns[i]] = row[i];
                }
                datasets.push(ds);
            }
        })
        .on('end', () => {
            //console.log('Datasets structure loaded.');
            dataLoadedCallback(datasets);
        });
}

/* Load all the attractivness data from CSV into a server object.
   The data don't need to be loaded for each request then. */
module.exports.loadRuralData = function (filePath, datasets, dataLoadedCallback) {
    console.log('Reading rural data file processing started.');
    var ruralData = [];

    let columns = undefined;

    fs.createReadStream(filePath)
        .pipe(csv())
        .on('data', (row) => {
            if (!columns) {
                columns = row;
            }
            else {
                let item = {};

                for (let i = 0; i < columns.length; i++) {
                    let colName = columns[i].toLowerCase();

                    if (colName == "nuts_id")       // ID of the NUTS region
                        item.nuts = row[i];
                    else if (colName == "datasets") // empty datasets count
                        item.availableDS = datasets.length - row[i];
                    else if (colName == "quality")
                        item.quality = row[i];
                    else if (colName == "lau2")
                        item.lau2 = row[i];
                    else {
                        let factor = getDataSetFactor(datasets, colName);
                        if (factor) {
                            if (!item[factor])
                                item[factor] = {};

                            //item[factor].push({ dataset: columns[i], value: row[i] });
                            item[factor][columns[i]] = Number(row[i]);
                        }
                    }
                }
                ruralData.push(item);
            }
        })
        .on('end', () => {
            //console.log('Rural data file processing finished.');
            dataLoadedCallback(ruralData);
        });
}

/**
 * Resolves with an array representing rows of CSV file
 * @param {string} inputFileName path to the CSV file with input data for clustering calculation
 */
module.exports.loadClusteringInput = async function (inputFileName) {
    const clusteringData = [];

    /*
     * The parsed CSV array keeps the native csv-parser structure
     * for future easier serialization back to CSV file
     */
    return new Promise((resolve, reject) => {
        fs.createReadStream(inputFileName)
            .pipe(csv())
            .on('data', (row) => {
                clusteringData.push(row);
            })
            .on('end', () => {
                resolve(clusteringData);
            })
            .on('error', reject);
    });
}

/**
 * Resolves once the modified CSV file is written to fs
 */
module.exports.modifyClusteringData = async function ({datasets, data, params, idString, outputFileName}) {
    // regional ID must be copied to the output as well
    let allowedDatasets = [idString];
    for (const factor of params.factors) {
        allowedDatasets = [...allowedDatasets, ...factor.datasets];
    }
    const factorMultipliers = data[0].map((dataset) => {
        if (dataset === idString) return 1;
        const factor = datasets.find(ds => ds.Name === dataset);
        if (!factor) {
            /* If the factor is unknown for this dataset, it will effectivelly turn it off */
            console.log(`Undefined factor for dataset ${dataset}`);
            return 0;
        } else if (!allowedDatasets.includes(dataset)) {
            return 0;
        } else {
            return params.factors.find(f => f.factor === factor.Factor).weight;
        }
    })
    //console.log(factorMultipliers);
    /* The actual modification logic resides here */
    const modifiedData = data.map((row, idx) => {
        return row.map((value, i) => {
            if (idx == 0) {
                /* These are the headers */
                /* Have to check for both allowed datasets and zero multiplications */
                return allowedDatasets.includes(value) && factorMultipliers[i] !== 0 ? value : null;
            } else if (isNaN(value)) {
                /* This is the NUTS ID record at the beginning of each line */
                return value;
            }
            return factorMultipliers[i] === 0 ? null : value*factorMultipliers[i];
        }).filter(val => val !== null);
    });
    //console.log(modifiedData);
    if (modifiedData[0].length <= 1) {
        throw new Error('All datasets turned off. No data to create clusters.');
    }
    return new Promise((resolve, reject) => {
        stringify(modifiedData, (err, output) => {
            if (err) return reject(err);
            fs.writeFile(outputFileName, output, (err) => {
                if (err) reject(err);
                else resolve();
                console.log('Data modification finished.');
            })
        })
    });
}

/** 
 * Reads the out_file.csv created by R script and saves it into an object
*/
module.exports.loadClusters = function (filePath, idString, dataLoadedCallback) {
    //console.log('Reading clustering data file processing started.');
    let clusters = [];

    let columns = undefined;

    fs.createReadStream(filePath)
        .pipe(csv())
        .on('data', (row) => {
            if (!columns) {
                columns = row;
            }
            else {
                let item = {};
                for (let i = 0; i < columns.length; i++) {
                    const colName = columns[i].length > 0 ? columns[i].toLowerCase() : idString;
                    item[colName] = row[i];
                }
                clusters.push(item);
            }
        })
        .on('end', () => {
            console.log('Cluster data file processing finished.');
            dataLoadedCallback(clusters);
        });
}

module.exports.getFactorIndex = function (region, factor) {
    //console.log('getFactorIndex');
    //console.log('region: ' + region.nuts);
    //console.log('factor: ' + JSON.stringify(factor, null, 4));

    let sumValue = 0;
    let count = 0;

    factor.datasets.forEach(ds => {
        //console.log('factor: ' + factor.factor);

        let value = region[factor.factor][ds];
        if (value) {
            sumValue += value;
            count++;
        }
    });

    return { index: sumValue / count, sumValue: sumValue, sumWeight: count * factor.weight };
}

function getDataSetFactor(datasets, colName) {
    for (let i = 0; i < datasets.length; i++) {
        if (datasets[i].Name.toLowerCase() == colName)
            return datasets[i].Factor;
    }

    return undefined;
}