rural-attractivness-service/index.js

const express = require('express');
const helpers = require('./helpers.js');
const nutsData = require('./nuts-data.js');
const cors = require('cors');
const R = require('r-script');
var where = require('lodash.where');

const app = express();

const _datasetsFilePath = 'data/datasets.csv';
const _datasetsCzFilePath = 'data/datasets-cz.csv';
const _dataFilePath = 'data/data.csv';
const _dataCzFilePath = 'data/cz/data-input-CZ_LAU2_37_cols.csv';
//const _clusteringInputFilePath = 'data/clustering/input_all.csv';
const _clusteringCzInputFilePath = 'data/cz/clusters-input-CZ_LAU2_37_cols.csv';
const _clusteringModifiedFilePath = 'data/clustering/input_modified.csv';
const _clusteringCzModifiedFilePath = 'data/cz/input_modified.csv';
const _clustersFilePath = 'data/clustering/out_file.csv';
const _clustersCzFilePath = 'data/cz/out_file_cz.csv';
var _datasets = undefined;
var _datasetsCZ = undefined;
var _ruralData = undefined;
var _ruralDataCZ = undefined;

// parse incoming POST requests body to JSON
app.use(express.json());
// handle CORS
app.use(cors())
// serve static files from the /static directory
app.use(express.static('static'))
// set PUG as default rendering engine
app.set('view engine', 'pug')

/* Dummy web service call without the method specified */
app.get('/', (req, res) => {
  res.send('Rural attractivness web service');
});

/* Makes refresh of the data loaded to the server objects.
   Must be called after the CSV data has changed */
app.get('/refresh', async (req, res, next) => {
  try {
    _datasets = await nutsData.loadDatasets(_datasetsFilePath)
    _datasetsCZ = await nutsData.loadDatasets(_datasetsCzFilePath)
    //console.log('Datasets loaded succesfully');
    _ruralData = await nutsData.loadRuralData(_dataFilePath, _datasets)
    _ruralDataCZ = await nutsData.loadRuralData(_datasetsCzFilePath, _datasetsCZ)
    //console.log('Rural data loaded succesfully');
  } catch (e) {
    res.send(e.toString() || e)
  }
  res.send('Data refreshed')
});

/* Returns JSON array with the list of all the datasets */
app.get('/:aoi?/datasets/', async (req, res, next) => {
  console.log('received datasets/ GET request')
  const aoi = req.params.aoi || 'eu'
  let datasets = aoi == 'cz' ? _datasetsCZ : _datasets
  if (!datasets) {
    try {
      if (aoi == 'cz') {
        _datasetsCZ = await nutsData.loadDatasets(_datasetsCzFilePath)
        datasets = _datasetsCZ
      } else {
        _datasets = await nutsData.loadDatasets(_datasetsFilePath)
        datasets = _datasets
      }
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  helpers.formatResponse(datasets, req, res)
});

/* Returns attractivity data for the region with ID equal to the 'nuts' parameter */
app.get('/:aoi?/scores/:nuts', async (req, res, next) => {
  if (!_datasets) { // datasets must be loaded prior to data loading
    try {
      _datasets = await nutsData.loadDatasets(_datasetsFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!_ruralData) {
    try {
      _ruralData = await nutsData.loadRuralData(_dataFilePath, _datasets)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  returnRegionScores(req.params.nuts, req, res)
});

/* Returns attractivity data for all the regions in source CSV data. */
app.get('/:aoi?/scores', async (req, res, next) => {
  console.log('received scores/cz GET request')
  const aoi = req.params.aoi || 'eu'
  let datasets = aoi == 'cz' ? _datasetsCZ : _datasets
  let ruralData = aoi == 'cz' ? _ruralDataCZ : _ruralData
  if (!datasets) { // datasets must be loaded prior to data loading
    try {
      if (aoi == 'cz') {
        _datasetsCZ = await nutsData.loadDatasets(_datasetsCzFilePath)
        datasets = _datasetsCZ
      } else {
        _datasets = await nutsData.loadDatasets(_datasetsFilePath)
        datasets = _datasets
      }
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!ruralData) {
    try {
      if (aoi == 'cz') {
        _ruralDataCZ = await nutsData.loadRuralData(_dataCzFilePath, datasets)
        ruralData = _ruralDataCZ
      } else {
        _ruralData = await nutsData.loadRuralData(_dataFilePath, datasets)
        ruralData = _ruralData
      }
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  helpers.formatResponse(ruralData, req, res)
});

/* Computes and returns attractivity data for all the NUTS regions based on the
   incomming datasets and factor weights. */
app.post('/:aoi?/scores', async (req, res, next) => {
  console.log('received scores/ POST request')
  //console.log("query: " + JSON.stringify(req.body.factors, null, 4));
  const aoi = req.params.aoi || 'eu'
  let datasets = aoi == 'cz' ? _datasetsCZ : _datasets
  let ruralData = aoi == 'cz' ? _ruralDataCZ : _ruralData
  if (!datasets) { // datasets must be loaded prior to data loading
    try {
      if (aoi == 'cz') {
        _datasetsCZ = await nutsData.loadDatasets(_datasetsCzFilePath)
        datasets = _datasetsCZ
      } else {
        _datasets = await nutsData.loadDatasets(_datasetsFilePath)
        datasets = _datasets
      }
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!ruralData) {
    try {
      if (aoi == 'cz') {
        _ruralDataCZ = await nutsData.loadRuralData(_dataCzFilePath, datasets)
        ruralData = _ruralDataCZ
      } else {
        _ruralData = await nutsData.loadRuralData(_dataFilePath, datasets)
        ruralData = _ruralData
      }
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  returnAllScores(req, res, aoi)
});

/*
  Only testing purposes
*/
app.get('/:aoi?/runR/', (req, res, next) => {
  //console.log(console);
  let aoi = req.params.aoi;
  const filename = aoi == 'cz' ? './r/CZ_clusters.r' :'./r/selected_data.r';
  console.log('calling R... for ' + aoi);
  //console.log(req)
  R(filename).data(9).call(
    function (err, data) {
      console.log('R done');
      if (err) {
        console.log(err.toString('utf8'));
        data = { result: err.toString('utf8') };
      }
      else {
        console.log(data);
        data = { result: 'R call succesful' };
      }
      helpers.formatResponse({ response: data }, req, res);
    }
  );
});

/*
    Just informative response. POST with JSON data is required.
*/
app.get('/:aoi?/clusters/', (req, res, next) => {
  const data = { response: '/clusters methods are only available under POST' }
  helpers.formatResponse(data, req, res)
});

/*
    Modifies input CSV file, calls R script, loads the resulting CSV file and returns it
*/
app.post('/:aoi?/clusters/', async (req, res, next) => {
  const aoi = req.params.aoi || 'eu'
  let datasets = aoi == 'cz' ? _datasetsCZ : _datasets
  const dataFilePath = aoi == 'cz' ? _clusteringCzInputFilePath : _dataFilePath
  const clusteringModifiedFilePath = aoi == 'cz' ? _clusteringCzModifiedFilePath : _clusteringModifiedFilePath
  const idString = aoi == 'cz' ? 'LAU2' : 'NUTS_ID'
  if (!datasets) { // datasets must be loaded prior to data loading
    try {
      const datasetsFilePath = aoi == 'cz' ? _datasetsCzFilePath : _datasetsFilePath
      if (aoi == 'cz') {
        _datasetsCZ = await nuts.loadDatasets(_datasetsCzFilePath)
        datasets = _datasetsCZ
      } else {
        _datasets = await nutsData.loadDatasets(_datasetsFilePath)
        datasets = _datasets
      }
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  try {
    //console.log(req.body);
    const clusteringData = await nutsData.loadClusteringInput(
      dataFilePath
    );
    await nutsData.modifyClusteringData({
      datasets: datasets,
      data: clusteringData,
      params: req.body,
      idString: idString,
      outputFileName: clusteringModifiedFilePath
    });
    handleRCall(req, res, aoi)
  } catch (error) { // Catch errors in async functions
    next(error.toString())
  }
});

app.get('/georeport/:nuts', async (req, res, next) => {
  if (!_datasets) { // datasets must be loaded prior to data loading
    try {
      _datasets = await nutsData.loadDatasets(_datasetsFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!_ruralData) {
    try {
      _ruralData = await nutsData.loadRuralData(_dataFilePath, _datasets)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  renderPugReport(req.params.nuts, req, res);
});

// start the service on the port xxxx
app.listen(3000, () => console.log('Rural attractivity WS listening on port 3000...'));


function renderPugReport(nuts, req, res) {
  let found = false;

  _ruralData.forEach(region => {
    if (region.nuts == nuts) {

      let nutsGeoJson = helpers.loadJSON('static/NUTS3_4326.json');
      let filterednutsGeoJson = where(nutsGeoJson.features, { "properties": { "NUTS_ID": nuts } });
      let pilots = helpers.loadJSON('static/pilots.json');

      region.metadata = {
        name: filterednutsGeoJson[0].properties.NUTS_NAME,
        code: nuts,
        centroid: helpers.getCentroid(filterednutsGeoJson[0].geometry),
        pilot: helpers.getPilotRegion(nuts, pilots)
      };

      res.render('nuts', { region: region, datasets: _datasets });
      found = true;
    }
  });

  if (!found)
    res.render('err', {});
}

function returnAllScores(req, res, aoi = 'eu') {
  let resData = []
  const ruralData = aoi == 'cz' ? _ruralDataCZ : _ruralData
  const regionIdString = aoi == 'cz' ? 'lau2' : 'nuts'
  ruralData.forEach(region => {
    //var region = _ruralData[0];
    let sumWeight = 0;
    let sumValue = 0;
    let regionIndexes = { code: region[regionIdString] };

    req.body.factors.forEach(f => {
      let fi = nutsData.getFactorIndex(region, f);
      //console.log("f: " + JSON.stringify(f));
      //console.log("fi: " + JSON.stringify(fi));

      regionIndexes[f.factor] = fi.index;
      sumValue += fi.sumValue * f.weight;
      sumWeight += fi.sumWeight;
    });

    regionIndexes.aggregate = sumValue / sumWeight;
    resData.push(regionIndexes);
  });

  helpers.formatResponse(resData, req, res);
}

function returnRegionScores(nuts, req, res) {
  var found = false;
  res.header("Content-Type", 'application/json');

  _ruralData.forEach(region => {
    if (region.nuts == nuts) {
      helpers.formatResponse(region, req, res);
      found = true;
    }
  });

  if (!found)
    // NUTS region not found
    res.status(404).send('NUTS region not found.');
}

function handleRCall(req, res, aoi = 'eu') {
  const numberOfClusters = req.body.numberOfClusters || 12;
  //console.log('calling R...', numberOfClusters)
  const rScriptInputFileName = aoi == 'cz' ? './r/CZ_clusters.r' :'./r/selected_data.r';
  const rScriptResultsFileName = aoi == 'cz' ? _clustersCzFilePath : _clustersFilePath;
  const idString = aoi == 'cz' ? 'lau2' : 'nuts_id';
  R(rScriptInputFileName).data(numberOfClusters).call(
    function (err, data) {
      //console.log('R done');
      if (err) {
        console.log(err.toString('utf8'));
        data = { result: err.toString('utf8') };
        res.status(204).send({ response: data });
      }
      else {
        console.log(data);
        nutsData.loadClusters(rScriptResultsFileName, idString, function (clusterData) {
          data = clusterData;
          helpers.formatResponse({ response: data }, req, res);
        });
      }
    }
  );
}