rural-attractivness-service/index.js

const express = require('express');
const helpers = require('./helpers.js');
const nutsData = require('./nuts-data.js');
const cors = require('cors');
const R = require('r-script');
var where = require('lodash.where');

const app = express();

const _datasetsFilePath = 'data/datasets.csv';
const _datasetsCzFilePath = 'data/datasets-cz.csv';
const _dataFilePath = 'data/data.csv';
const _dataCzFilePath = 'data/cz/data-input-CZ_LAU2_37_cols.csv';
//const _clusteringInputFilePath = 'data/clustering/input_all.csv';
const _clusteringCzInputFilePath = 'data/cz/clusters-input-CZ_LAU2_37_cols.csv';
const _clusteringModifiedFilePath = 'data/clustering/input_modified.csv';
const _clusteringCzModifiedFilePath = 'data/cz/input_modified.csv';
const _clustersFilePath = 'data/clustering/out_file.csv';
const _clustersCzFilePath = 'data/cz/out_file_cz.csv';
var _datasets = undefined;
var _datasetsCZ = undefined;
var _ruralData = undefined;
var _ruralDataCZ = undefined;

// parse incoming POST requests body to JSON
app.use(express.json());
// handle CORS
app.use(cors())
// serve static files from the /static directory
app.use(express.static('static'))
// set PUG as default rendering engine
app.set('view engine', 'pug')

/* Dummy web service call without the method specified */
app.get('/', (req, res) => {
  res.send('Rural attractivness web service');
});

/* Makes refresh of the data loaded to the server objects.
   Must be called after the CSV data has changed */
app.get('/refresh', async (req, res, next) => {
  try {
    _datasets = await nutsData.loadDatasets(_datasetsFilePath)
    //console.log('Datasets loaded succesfully');
    _ruralData = await nutsData.loadRuralData(_dataFilePath, _datasets)
    //console.log('Rural data loaded succesfully');
  } catch (e) {
    res.send(e.toString() || e)
  }
  res.send('Data refreshed')
});

/* Returns JSON array with the list of all the datasets */
app.get('/datasets', async (req, res, next) => {
  if (!_datasets) {
    try {
      _datasets = await nutsData.loadDatasets(_datasetsFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  helpers.formatResponse(_datasets, req, res)
});

/* Returns JSON array with the list of all the datasets */
app.get('/datasets/cz', async (req, res, next) => {
  console.log('received datasets/cz GET request')
  if (!_datasetsCZ) {
    try {
      _datasetsCZ = await nutsData.loadDatasets(_datasetsCzFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  //console.log('Datasets loaded callback');
  helpers.formatResponse(_datasetsCZ, req, res)
});

/* Returns attractivity data for the region with ID equal to the 'nuts' parameter */
app.get('/scores/:nuts', async (req, res, next) => {
  if (!_datasets) { // datasets must be loaded prior to data loading
    try {
      _datasets = await nutsData.loadDatasets(_datasetsFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!_ruralData) {
    try {
      _ruralData = await nutsData.loadRuralData(_dataFilePath, _datasets)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  returnRegionScores(req.params.nuts, req, res)
});

/* Returns attractivity data for all the regions in source CSV data. */
app.get('/scores', async (req, res, next) => {
  if (!_datasets) { // datasets must be loaded prior to data loading
    try {
      _datasets = await nutsData.loadDatasets(_datasetsFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!_ruralData) {
    try {
      _ruralData = await nutsData.loadRuralData(_dataFilePath, _datasets)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  helpers.formatResponse(_ruralData, req, res)
});

/* Computes and returns attractivity data for all the NUTS regions based on the
   incomming datasets and factor weights. */
app.post('/scores', async (req, res, next) => {
  //console.log("query: " + JSON.stringify(req.body.factors, null, 4));
  if (!_datasets) { // datasets must be loaded prior to data loading
    try {
      _datasets = await nutsData.loadDatasets(_datasetsFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!_ruralData) {
    try {
      _ruralData = await nutsData.loadRuralData(_dataFilePath, _datasets)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  returnAllScores(req, res)
});

/* Returns attractivity data for all the regions in source CSV data. */
app.get('/scores/cz', async (req, res, next) => {
  console.log('received scores/cz GET request')
  if (!_datasetsCZ) { // datasets must be loaded prior to data loading
    try {
      _datasetsCZ = await nutsData.loadDatasets(_datasetsCzFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!_ruralDataCZ) {
    try {
      _ruralDataCZ = await nutsData.loadRuralData(_dataCzFilePath, _datasetsCZ)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  helpers.formatResponse(_ruralDataCZ, req, res)
});

/* Computes and returns attractivity data for all the NUTS regions based on the
   incomming datasets and factor weights. */
app.post('/scores/cz', async (req, res, next) => {
  console.log('received scores/cz POST request')
  //console.log("query: " + JSON.stringify(req.body.factors, null, 4));
  if (!_datasetsCZ) { // datasets must be loaded prior to data loading
    try {
      _datasetsCZ = await nutsData.loadDatasets(_datasetsCzFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!_ruralDataCZ) {
    try {
      _ruralDataCZ = await nutsData.loadRuralData(_dataCzFilePath, _datasetsCZ)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
    returnAllScoresCz(req, res)
});

/*
  Only testing purposes
*/
app.get('/runR/:version?', (req, res, next) => {
  //console.log(console);
  let v = req.params.version;
  const filename = v == 'cz' ? './r/CZ_clusters.r' :'./r/selected_data.r';
  console.log('calling R... for ' + v);
  //console.log(req)
  R(filename).data(9).call(
    function (err, data) {
      console.log('R done');
      if (err) {
        console.log(err.toString('utf8'));
        data = { result: err.toString('utf8') };
      }
      else {
        console.log(data);
        data = { result: 'R call succesful' };
      }
      helpers.formatResponse({ response: data }, req, res);
    }
  );
});

/*
    Just informative response. POST with JSON data is required.
*/
app.get(['/clusters', '/clusters/cz'], (req, res, next) => {
  const data = { response: '/clusters methods are only available under POST' }
  helpers.formatResponse(data, req, res)
});

/*
    Modifies input CSV file, calls R script, loads the resulting CSV file and returns it
*/
app.post('/clusters', async (req, res, next) => {
  if (!_datasets) { // datasets must be loaded prior to data loading
    try {
      _datasets = await nutsData.loadDatasets(_datasetsFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  try {
    //console.log(req.body);
    const clusteringData = await nutsData.loadClusteringInput(
      _dataFilePath
    );
    await nutsData.modifyClusteringData({
      datasets: _datasets,
      data: clusteringData,
      params: req.body,
      idString: 'NUTS_ID',
      outputFileName: _clusteringModifiedFilePath
    });
    handleRCall(req, res, 'eu')
  } catch (error) { // Catch errors in async functions
    next(error.toString())
  }
});

/*
    Modifies input CSV file, calls R script, loads the resulting CSV file and returns it
*/
app.post('/clusters/cz', async (req, res, next) => {
  if (!_datasetsCZ) {
    _datasetsCZ = await nutsData.loadDatasets(_datasetsCzFilePath)
    //console.log('Datasets loaded succesfully');
  }
  try {
    //console.log(req.body);
    const clusteringData = await nutsData.loadClusteringInput(
      _clusteringCzInputFilePath
    );
    await nutsData.modifyClusteringData({
      region: 'cz',
      datasets: _datasetsCZ,
      data: clusteringData,
      params: req.body,
      idString: 'LAU2',
      outputFileName: _clusteringCzModifiedFilePath
    })
    handleRCall(req, res, 'cz');
  } catch (error) { // Catch errors in async functions
    next(error.toString());
  }
});

app.get('/georeport/:nuts', async (req, res, next) => {
  if (!_datasets) { // datasets must be loaded prior to data loading
    try {
      _datasets = await nutsData.loadDatasets(_datasetsFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!_ruralData) {
    try {
      _ruralData = await nutsData.loadRuralData(_dataFilePath, _datasets)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  renderPugReport(req.params.nuts, req, res);
});

// start the service on the port xxxx
app.listen(3000, () => console.log('Rural attractivity WS listening on port 3000...'));


function renderPugReport(nuts, req, res) {
  let found = false;

  _ruralData.forEach(region => {
    if (region.nuts == nuts) {

      let nutsGeoJson = helpers.loadJSON('static/NUTS3_4326.json');
      let filterednutsGeoJson = where(nutsGeoJson.features, { "properties": { "NUTS_ID": nuts } });
      let pilots = helpers.loadJSON('static/pilots.json');

      region.metadata = {
        name: filterednutsGeoJson[0].properties.NUTS_NAME,
        code: nuts,
        centroid: helpers.getCentroid(filterednutsGeoJson[0].geometry),
        pilot: helpers.getPilotRegion(nuts, pilots)
      };

      res.render('nuts', { region: region, datasets: _datasets });
      found = true;
    }
  });

  if (!found)
    res.render('err', {});
}

function returnAllScores(req, res) {
  var resData = [];
  _ruralData.forEach(region => {
    //var region = _ruralData[0];
    var sumWeight = 0;
    var sumValue = 0;
    let regionIndexes = { code: region.nuts };

    req.body.factors.forEach(f => {
      let fi = nutsData.getFactorIndex(region, f);
      //console.log("f: " + JSON.stringify(f));
      //console.log("fi: " + JSON.stringify(fi));

      regionIndexes[f.factor] = fi.index;
      sumValue += fi.sumValue * f.weight;
      sumWeight += fi.sumWeight;
    });

    regionIndexes.aggregate = sumValue / sumWeight;
    resData.push(regionIndexes);
  });

  helpers.formatResponse(resData, req, res);
}

function returnAllScoresCz(req, res) {
  var resData = [];
  _ruralDataCZ.forEach(region => {
    //var region = _ruralData[0];
    var sumWeight = 0;
    var sumValue = 0;
    let regionIndexes = { code: region.lau2 };

    req.body.factors.forEach(f => {
      let fi = nutsData.getFactorIndex(region, f);
      //console.log("f: " + JSON.stringify(f));
      //console.log("fi: " + JSON.stringify(fi));

      regionIndexes[f.factor] = fi.index;
      sumValue += fi.sumValue * f.weight;
      sumWeight += fi.sumWeight;
    });

    regionIndexes.aggregate = sumValue / sumWeight;
    resData.push(regionIndexes);
  });

  helpers.formatResponse(resData, req, res);
}

function returnRegionScores(nuts, req, res) {
  var found = false;
  res.header("Content-Type", 'application/json');

  _ruralData.forEach(region => {
    if (region.nuts == nuts) {
      helpers.formatResponse(region, req, res);
      found = true;
    }
  });

  if (!found)
    // NUTS region not found
    res.status(404).send('NUTS region not found.');
}

function handleRCall(req, res, region = 'eu') {
  const numberOfClusters = req.body.numberOfClusters || 12;
  //console.log('calling R...', numberOfClusters)
  const rScriptInputFileName = region == 'cz' ? './r/CZ_clusters.r' :'./r/selected_data.r';
  const rScriptResultsFileName = region == 'cz' ? _clustersCzFilePath : _clustersFilePath;
  const idString = region == 'cz' ? 'lau2' : 'nuts_id';
  R(rScriptInputFileName).data(numberOfClusters).call(
    function (err, data) {
      //console.log('R done');
      if (err) {
        console.log(err.toString('utf8'));
        data = { result: err.toString('utf8') };
        res.status(204).send({ response: data });
      }
      else {
        console.log(data);
        nutsData.loadClusters(rScriptResultsFileName, idString, function (clusterData) {
          data = clusterData;
          helpers.formatResponse({ response: data }, req, res);
        });
      }
    }
  );
}