rural-attractivness-service/index.js

const express = require('express');
const helpers = require('./helpers.js');
const nutsData = require('./nuts-data.js');
const cors = require('cors');
const R = require('r-script');
var where = require('lodash.where');

const app = express();

const _datasetsFilePath = 'data/datasets.csv';
const _datasetsCzFilePath = 'data/datasets-cz.csv';
const _dataFilePath = 'data/data.csv';
const _dataCzFilePath = 'data/cz/data-input-CZ_LAU2_37_cols.csv';
//const _clusteringInputFilePath = 'data/clustering/input_all.csv';
const _clusteringCzInputFilePath = 'data/cz/clusters-input-CZ_LAU2_37_cols.csv';
const _clusteringModifiedFilePath = 'data/clustering/input_modified.csv';
const _clusteringCzModifiedFilePath = 'data/cz/input_modified.csv';
const _clustersFilePath = 'data/clustering/out_file.csv';
const _clustersCzFilePath = 'data/cz/out_file_cz.csv';
var _datasets = undefined;
var _datasetsCZ = undefined;
var _ruralData = undefined;
var _ruralDataCZ = undefined;

// parse incoming POST requests body to JSON
app.use(express.json());
// handle CORS
app.use(cors())
// serve static files from the /static directory
app.use(express.static('static'))
// set PUG as default rendering engine
app.set('view engine', 'pug')

/* Dummy web service call without the method specified */
app.get('/', (req, res) => {
  res.send('Rural attractivness web service');
});

/* Makes refresh of the data loaded to the server objects.
   Must be called after the CSV data has changed */
app.get('/refresh', (req, res, next) => {
  nutsData.loadDatasets(_datasetsFilePath, function (ds) {
    //console.log('Datasets loaded succesfully');
    _datasets = ds;
    nutsData.loadRuralData(_dataFilePath, _datasets, function (rd) {
      //console.log('Rural data loaded succesfully');
      _ruralData = rd;
      res.send('Data refreshed');
    });
  });
});

/* Returns JSON array with the list of all the datasets */
app.get('/datasets', (req, res, next) => {
  if (_datasets) {
    helpers.formatResponse(_datasets, req, res);
  }
  else {
    nutsData.loadDatasets(_datasetsFilePath, function (ds) {
      //console.log('Datasets loaded callback');
      _datasets = ds;
      helpers.formatResponse(_datasets, req, res);
    });
  }
});

/* Returns JSON array with the list of all the datasets */
app.get('/datasets/cz', (req, res, next) => {
  console.log('received datasets/cz GET request')
  if (_datasetsCZ) {
    helpers.formatResponse(_datasetsCZ, req, res);
  }
  else {
    nutsData.loadDatasets(_datasetsCzFilePath, function (ds) {
      //console.log('Datasets loaded callback');
      _datasetsCZ = ds;
      helpers.formatResponse(_datasetsCZ, req, res);
    });
  }
});

/* Returns attractivity data for the region with ID equal to the 'nuts' parameter */
app.get('/scores/:nuts', (req, res, next) => {
  if (_ruralData) {
    returnRegionScores(req.params.nuts, req, res);
  }
  else {
    if (_datasets) { // datasets must be loaded prior to data loading
      nutsData.loadRuralData(_dataFilePath, _datasets, function (rd) {
        _ruralData = rd;
        returnRegionScores(req.params.nuts, req, res);
      });
    }
    else {
      nutsData.loadDatasets(_datasetsFilePath, function (ds) {
        _datasets = ds;

        nutsData.loadRuralData(_dataFilePath, _datasets, function (rd) {
          _ruralData = rd;
          returnRegionScores(req.params.nuts, req, res);
        });
      });
    }
  }
});

/* Returns attractivity data for all the regions in source CSV data. */
app.get('/scores', (req, res, next) => {
  if (_ruralData)
    helpers.formatResponse(_ruralData, req, res);
  else {
    if (_datasets) { // datasets must be loaded prior to data loading
      nutsData.loadRuralData(_dataFilePath, _datasets, function (rd) {
        _ruralData = rd;
        helpers.formatResponse(_ruralData, req, res);
      });
    }
    else {
      nutsData.loadDatasets(_datasetsFilePath, function (ds) {
        _datasets = ds;

        nutsData.loadRuralData(_dataFilePath, _datasets, function (rd) {
          _ruralData = rd;
          helpers.formatResponse(_ruralData, req, res);
        });
      });
    }
  }
});

/* Computes and returns attractivity data for all the NUTS regions based on the
   incomming datasets and factor weights. */
app.post('/scores', (req, res, next) => {
  //console.log("query: " + JSON.stringify(req.body.factors, null, 4));

  if (_ruralData) {
    returnAllScores(req, res);
  }
  else {
    if (_datasets) { // datasets must be loaded prior to data loading
      nutsData.loadRuralData(_dataFilePath, _datasets, function (rd) {
        _ruralData = rd;
        returnAllScores(req, res);
      });
    }
    else {
      nutsData.loadDatasets(_datasetsFilePath, function (ds) {
        _datasets = ds;

        nutsData.loadRuralData(_dataFilePath, _datasets, function (rd) {
          _ruralData = rd;
          returnAllScores(req, res);
        });
      });
    }
  }
});

/* Returns attractivity data for all the regions in source CSV data. */
app.get('/scores/cz', (req, res, next) => {
  console.log('received scores/cz GET request')
  if (_ruralDataCZ)
    helpers.formatResponse(_ruralDataCZ, req, res);
  else {
    if (_datasetsCZ) { // datasets must be loaded prior to data loading
      nutsData.loadRuralData(_dataCzFilePath, _datasetsCZ, function (rd) {
        _ruralDataCZ = rd;
        helpers.formatResponse(_ruralDataCZ, req, res);
      });
    }
    else {
      nutsData.loadDatasets(_datasetsCzFilePath, function (ds) {
        _datasetsCZ = ds;

        nutsData.loadRuralData(_dataCzFilePath, _datasetsCZ, function (rd) {
          _ruralDataCZ = rd;
          helpers.formatResponse(_ruralDataCZ, req, res);
        });
      });
    }
  }
});

/* Computes and returns attractivity data for all the NUTS regions based on the
   incomming datasets and factor weights. */
app.post('/scores/cz', (req, res, next) => {
  console.log('received scores/cz POST request')
  //console.log("query: " + JSON.stringify(req.body.factors, null, 4));

  if (_ruralDataCZ) {
    returnAllScoresCz(req, res);
  }
  else {
    if (_datasetsCZ) { // datasets must be loaded prior to data loading
      nutsData.loadRuralData(_dataCzFilePath, _datasetsCZ, function (rd) {
        _ruralDataCZ = rd;
        returnAllScoresCz(req, res);
      });
    }
    else {
      nutsData.loadDatasets(_datasetsCzFilePath, function (ds) {
        _datasetsCZ = ds;

        nutsData.loadRuralData(_dataCzFilePath, _datasetsCZ, function (rd) {
          _ruralDataCZ = rd;
          returnAllScoresCz(req, res);
        });
      });
    }
  }
});

/*
  Only testing purposes
*/
app.get('/runR/:version?', (req, res, next) => {
  //console.log(console);
  let v = req.params.version;
  const filename = v == 'cz' ? './r/CZ_clusters.r' :'./r/selected_data.r';
  console.log('calling R... for ' + v);
  //console.log(req)
  R(filename).data(9).call(
    function (err, data) {
      console.log('R done');
      if (err) {
        console.log(err.toString('utf8'));
        data = { result: err.toString('utf8') };
      }
      else {
        console.log(data);
        data = { result: 'R call succesful' };
      }
      helpers.formatResponse({ response: data }, req, res);
    }
  );
});

/*
    Just informative response. POST with JSON data is required.
*/
app.get(['/clusters', '/clusters/cz'], (req, res, next) => {
  const data = { response: '/clusters methods are only available under POST' }
  helpers.formatResponse(data, req, res);
});

/*
    Modifies input CSV file, calls R script, loads the resulting CSV file and returns it
*/
app.post('/clusters', async (req, res, next) => {
  try {
    if (!_datasets) {
      //TODO: promisify all functions to avoid callback hell and make this work properly
      await nutsData.loadDatasets(_datasetsFilePath, function (ds) {
        //console.log('Datasets loaded succesfully');
        _datasets = ds;
      })
    }
    //console.log(req.body);
    const clusteringData = await nutsData.loadClusteringInput(
      _dataFilePath
    );
    await nutsData.modifyClusteringData({
      datasets: _datasets,
      data: clusteringData,
      params: req.body,
      idString: 'NUTS_ID',
      outputFileName: _clusteringModifiedFilePath
    });
    handleRCall(req, res, 'eu');
  } catch (error) { // Catch errors in async functions
    next(error.toString());
  }
});

/*
    Modifies input CSV file, calls R script, loads the resulting CSV file and returns it
*/
app.post('/clusters/cz', async (req, res, next) => {
  try {
    if (!_datasetsCZ) {
      //TODO: promisify all functions to avoid callback hell and make this work properly
      await nutsData.loadDatasets(_datasetsCzFilePath, function (ds) {
        //console.log('Datasets loaded succesfully');
        _datasetsCZ = ds;
      })
    }
    //console.log(req.body);
    const clusteringData = await nutsData.loadClusteringInput(
      _clusteringCzInputFilePath
    );
    await nutsData.modifyClusteringData({
      region: 'cz',
      datasets: _datasetsCZ,
      data: clusteringData,
      params: req.body,
      idString: 'LAU2',
      outputFileName: _clusteringCzModifiedFilePath
    });
    handleRCall(req, res, 'cz');
  } catch (error) { // Catch errors in async functions
    next(error.toString());
  }
});

app.get('/georeport/:nuts', (req, res, next) => {
  if (_ruralData) {
    renderPugReport(req.params.nuts, req, res);
  }
  else {
    if (_datasets) { // datasets must be loaded prior to data loading
      nutsData.loadRuralData(_dataFilePath, _datasets, function (rd) {
        _ruralData = rd;
        renderPugReport(req.params.nuts, req, res);
      });
    }
    else {
      nutsData.loadDatasets(_datasetsFilePath, function (ds) {
        _datasets = ds;

        nutsData.loadRuralData(_dataFilePath, _datasets, function (rd) {
          _ruralData = rd;
          renderPugReport(req.params.nuts, req, res);
        });
      });
    }
  }
});

// start the service on the port xxxx
app.listen(3000, () => console.log('Rural attractivity WS listening on port 3000...'));


function renderPugReport(nuts, req, res) {
  var found = false;

  _ruralData.forEach(region => {
    if (region.nuts == nuts) {

      let nutsGeoJson = helpers.loadJSON('static/NUTS3_4326.json');
      let filterednutsGeoJson = where(nutsGeoJson.features, { "properties": { "NUTS_ID": nuts } });
      let pilots = helpers.loadJSON('static/pilots.json');

      region.metadata = {
        name: filterednutsGeoJson[0].properties.NUTS_NAME,
        code: nuts,
        centroid: helpers.getCentroid(filterednutsGeoJson[0].geometry),
        pilot: helpers.getPilotRegion(nuts, pilots)
      };

      res.render('nuts', { region: region, datasets: _datasets });
      found = true;
    }
  });

  if (!found)
    res.render('err', {});
}

function returnAllScores(req, res) {
  var resData = [];
  _ruralData.forEach(region => {
    //var region = _ruralData[0];
    var sumWeight = 0;
    var sumValue = 0;
    let regionIndexes = { code: region.nuts };

    req.body.factors.forEach(f => {
      let fi = nutsData.getFactorIndex(region, f);
      //console.log("f: " + JSON.stringify(f));
      //console.log("fi: " + JSON.stringify(fi));

      regionIndexes[f.factor] = fi.index;
      sumValue += fi.sumValue * f.weight;
      sumWeight += fi.sumWeight;
    });

    regionIndexes.aggregate = sumValue / sumWeight;
    resData.push(regionIndexes);
  });

  helpers.formatResponse(resData, req, res);
}

function returnAllScoresCz(req, res) {
  var resData = [];
  _ruralDataCZ.forEach(region => {
    //var region = _ruralData[0];
    var sumWeight = 0;
    var sumValue = 0;
    let regionIndexes = { code: region.lau2 };

    req.body.factors.forEach(f => {
      let fi = nutsData.getFactorIndex(region, f);
      //console.log("f: " + JSON.stringify(f));
      //console.log("fi: " + JSON.stringify(fi));

      regionIndexes[f.factor] = fi.index;
      sumValue += fi.sumValue * f.weight;
      sumWeight += fi.sumWeight;
    });

    regionIndexes.aggregate = sumValue / sumWeight;
    resData.push(regionIndexes);
  });

  helpers.formatResponse(resData, req, res);
}

function returnRegionScores(nuts, req, res) {
  var found = false;
  res.header("Content-Type", 'application/json');

  _ruralData.forEach(region => {
    if (region.nuts == nuts) {
      helpers.formatResponse(region, req, res);
      found = true;
    }
  });

  if (!found)
    // NUTS region not found
    res.status(404).send('NUTS region not found.');
}

function handleRCall(req, res, region = 'eu') {
  const numberOfClusters = req.body.numberOfClusters || 12;
  //console.log('calling R...', numberOfClusters)
  const rScriptInputFileName = region == 'cz' ? './r/CZ_clusters.r' :'./r/selected_data.r';
  const rScriptResultsFileName = region == 'cz' ? _clustersCzFilePath : _clustersFilePath;
  const idString = region == 'cz' ? 'lau2' : 'nuts_id';
  R(rScriptInputFileName).data(numberOfClusters).call(
    function (err, data) {
      //console.log('R done');
      if (err) {
        console.log(err.toString('utf8'));
        data = { result: err.toString('utf8') };
        res.status(204).send({ response: data });
      }
      else {
        console.log(data);
        nutsData.loadClusters(rScriptResultsFileName, idString, function (clusterData) {
          data = clusterData;
          helpers.formatResponse({ response: data }, req, res);
        });
      }
    }
  );
}