rural-attractivness-service/index.js

const express = require('express');
const helpers = require('./helpers.js');
const nutsData = require('./nuts-data.js');
const cors = require('cors');
const R = require('r-script');
var where = require('lodash.where');

const app = express();

const _datasetsEuFilePath = 'data/datasets.csv';
//const _datasetsCzFilePath = 'data/datasets-cz.csv';
const _dataEuFilePath = 'data/data.csv';
const _dataCzFilePath = 'data/cz/data-input-CZ_LAU2_37_cols.csv';
const _dataAfricaFilePath = 'data/africa/ke+ug_normalized_input.csv';
//const _clusteringEuInputFilePath = 'data/clustering/input_all.csv';
//const _clusteringCzInputFilePath = 'data/cz/clusters-input-CZ_LAU2_37_cols.csv';
const _clusteringEuModifiedFilePath = 'data/clustering/input_modified.csv';
const _clusteringCzModifiedFilePath = 'data/cz/input_modified.csv';
const _clusteringAfricaModifiedFilePath = 'data/africa/input_modified.csv';
const _clustersEuFilePath = 'data/clustering/out_file.csv';
const _clustersCzFilePath = 'data/cz/out_file_cz.csv';
const _clustersAfricaFilePath = 'data/africa/out_file_africa.csv';
const _ontologyFilePath = 'data/rural_attractiveness.owl.json';
var _datasetsEU = undefined;
var _datasetsCZ = undefined;
var _datasetsAfrica = undefined;
var _ruralDataEU = undefined;
var _ruralDataCZ = undefined;
var _ruralDataAfrica = undefined;

// parse incoming POST requests body to JSON
app.use(express.json());
// handle CORS
app.use(cors())
// serve static files from the /static directory
app.use(express.static('static'))
// set PUG as default rendering engine
app.set('view engine', 'pug')

/* Dummy web service call without the method specified */
app.get('/', (req, res) => {
  helpers.describeSelf(req, res)
});

/* Makes refresh of the data loaded to the server objects.
   Must be called after the CSV data has changed */
app.get('/refresh', async (req, res, next) => {
  try {
    _datasetsEU = await nutsData.loadDatasets(_dataEuFilePath)
    _datasetsCZ = await nutsData.loadDatasets(_dataCzFilePath)
    //console.log('Datasets loaded succesfully');
    _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath)
    _ruralDataCZ = await nutsData.loadRuralData(_dataCzFilePath)
    //console.log('Rural data loaded succesfully');
  } catch (e) {
    res.send(e.toString() || e)
  }
  res.send('Data refreshed')
});

/* Returns JSON array with the list of all the datasets */
app.get('/:aoi?/datasets/', async (req, res, next) => {
  const aoi = req.params.aoi || 'eu'
  const datasets = await loadDatasetsIfNeeded(req, res, aoi)
  helpers.formatResponse(datasets, req, res)
});

/* Returns attractivity data for the region with ID equal to the 'nuts' parameter */
app.get('/:aoi?/scores/:nuts', async (req, res, next) => {
  if (!_datasetsEU) { // datasets must be loaded prior to data loading
    try {
      _datasetsEU = await nutsData.loadDatasets(_dataEuFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!_ruralDataEU) {
    try {
      _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  returnRegionScores(req.params.nuts, req, res)
});

/* Returns attractivity data for all the regions in source CSV data. */
app.get('/:aoi?/scores', async (req, res, next) => {
  const aoi = req.params.aoi || 'eu'
  await loadDatasetsIfNeeded(req, res, aoi)
  const ruralData = await loadRuralDataIfNeeded(req, res, aoi)
  helpers.formatResponse(ruralData, req, res)
});

/* Computes and returns attractivity data for all the NUTS regions based on the
   incomming datasets and factor weights. */
app.post('/:aoi?/scores', async (req, res, next) => {
  //console.log("query: " + JSON.stringify(req.body.factors, null, 4));
  const aoi = req.params.aoi || 'eu'
  await loadDatasetsIfNeeded(req, res, aoi)
  await loadRuralDataIfNeeded(req, res, aoi)
  const resData = returnAllScores(req, res, aoi)
  helpers.formatResponse(resData, req, res);
});

/*
  Only testing purposes
*/
app.get('/:aoi?/runR/', (req, res, next) => {
  let aoi = req.params.aoi
  let filename
  switch(aoi) {
    case 'cz':
      filename = './r/CZ_clusters.r'
      break
    case 'africa':
      filename = './r/african_clusters.r'
      break
    case 'eu':
      filename = './r/selected_data.r'
      break
  }
  console.log('calling R... for ' + aoi);
  //console.log(req)
  R(filename).data(9).call(
    function (err, data) {
      console.log('R done');
      if (err) {
        console.log(err.toString('utf8'));
        data = { result: err.toString('utf8') };
      }
      else {
        console.log(data);
        data = { result: 'R call succesful' };
      }
      helpers.formatResponse({ response: data }, req, res);
    }
  );
});

/*
    Just informative response. POST with JSON data is required.
*/
app.get('/:aoi?/clusters/', (req, res, next) => {
  const data = { response: '/clusters methods are only available under POST' }
  helpers.formatResponse(data, req, res)
});

/*
    Modifies input CSV file, calls R script, loads the resulting CSV file and returns it
*/
app.post('/:aoi?/clusters/', async (req, res, next) => {
  const aoi = req.params.aoi || 'eu'
  const datasets = await loadDatasetsIfNeeded(req, res, aoi)
  let dataFilePath, clusteringModifiedFilePath, idString
  switch(aoi) {
    case 'cz':
      dataFilePath = _dataCzFilePath
      clusteringModifiedFilePath = _clusteringCzModifiedFilePath
      idString = 'LAU2'
      break
    case 'africa':
      dataFilePath = _dataAfricaFilePath
      clusteringModifiedFilePath = _clusteringAfricaModifiedFilePath
      idString = 'district_code'
      break
    case 'eu':
      dataFilePath = _dataEuFilePath
      clusteringModifiedFilePath = _clusteringEuModifiedFilePath
      idString = 'NUTS_ID'
      break
  }
  try {
    //console.log(req.body);
    const clusteringData = await nutsData.loadClusteringInput(
      dataFilePath
    );
    await nutsData.modifyClusteringData({
      datasets: datasets,
      data: clusteringData,
      params: req.body,
      idString: idString,
      outputFileName: clusteringModifiedFilePath
    });
    handleRCall(req, res, aoi)
  } catch (error) { // Catch errors in async functions
    next(error.toString())
  }
});

/*
 * Reads the ontology file and sends its content
 * This ensures that both backend and frontend are using the same version of the ontology
*/
app.get('/ontology/', async (req, res, next) => {
  const onto = await nutsData.loadOntology(_ontologyFilePath)
  helpers.formatResponse(onto, req, res)
});

//FIXME: use ontology to read datasets metadata // CURRENTLY BROKEN
app.get('/georeport/:nuts', async (req, res, next) => {
  if (!_datasetsEU) { // datasets must be loaded prior to data loading
    try {
      _datasetsEU = await nutsData.loadDatasets(_dataEuFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  if (!_ruralDataEU) {
    try {
      _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath)
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  renderPugReport(req.params.nuts, req, res);
});

// start the service on the port xxxx
app.listen(3000, () => console.log('Rural attractivity WS listening on port 3000...'));


function renderPugReport(nuts, req, res) {
  let found = false;

  _ruralDataEU.forEach(region => {
    if (region.nuts == nuts) {

      let nutsGeoJson = helpers.loadJSON('static/NUTS3_4326.json');
      let filterednutsGeoJson = where(nutsGeoJson.features, { "properties": { "NUTS_ID": nuts } });
      let pilots = helpers.loadJSON('static/pilots.json');

      region.metadata = {
        name: filterednutsGeoJson[0].properties.NUTS_NAME,
        code: nuts,
        centroid: helpers.getCentroid(filterednutsGeoJson[0].geometry),
        pilot: helpers.getPilotRegion(nuts, pilots)
      };

      //FIXME: don't imput _datasets here but already pre-processed metadata object
      res.render('nuts', { region: region, datasets: _datasetsEU });
      found = true;
    }
  });

  if (!found)
    res.render('err', {});
}

function returnAllScores(req, res, aoi = 'eu') {
  let resData = []
  let ruralData, regionIdString
  switch(aoi) {
    case 'cz':
      ruralData = _ruralDataCZ
      regionIdString = 'lau2'
      break
    case 'africa':
      ruralData = _ruralDataAfrica
      regionIdString = 'district'
      break
    case 'eu':
      ruralData = _ruralDataEU
      regionIdString = 'nuts'
      break
  }
  ruralData.forEach(region => {
    //var region = _ruralData[0];
    let sumWeight = 0;
    let sumValue = 0;
    let regionIndexes = { code: region[regionIdString] };

    req.body.factors.forEach(factor => {
      let factIndex = nutsData.getFactorIndex(region, factor);
      //console.log("f: " + JSON.stringify(factor));
      //console.log("fi: " + JSON.stringify(factIndex));

      regionIndexes[factor.factor] = factIndex.index;
      sumValue += factIndex.sumValue * factor.weight;
      sumWeight += factIndex.sumWeight;
    });

    regionIndexes.aggregate = sumValue / sumWeight;
    resData.push(regionIndexes);
  });
  //console.log(resData)
  return resData;
}

function returnRegionScores(nuts, req, res) {
  var found = false;
  res.header("Content-Type", 'application/json');

  _ruralDataEU.forEach(region => {
    if (region.nuts == nuts) {
      helpers.formatResponse(region, req, res);
      found = true;
    }
  });

  if (!found)
    // NUTS region not found
    res.status(404).send('NUTS region not found.');
}

function handleRCall(req, res, aoi = 'eu') {
  const numberOfClusters = req.body.numberOfClusters || 12;
  //console.log('calling R...', numberOfClusters)
  let rScriptInputFileName, rScriptResultsFileName, idString
  switch(aoi) {
    case 'cz':
      rScriptInputFileName = './r/CZ_clusters.r'
      rScriptResultsFileName = _clustersCzFilePath
      idString = 'lau2'
      break
    case 'africa':
      rScriptInputFileName = './r/african_clusters.r'
      rScriptResultsFileName = _clustersAfricaFilePath
      idString = 'district_code'
      break
    case 'eu':
      rScriptInputFileName = './r/selected_data.r'
      rScriptResultsFileName = _clustersEuFilePath
      idString = 'nuts_id'
      break
  }
  R(rScriptInputFileName).data(numberOfClusters).call(
    function (err, data) {
      //console.log('R done');
      if (err) {
        console.log(err.toString('utf8'));
        data = { result: err.toString('utf8') };
        res.status(204).send({ response: data });
      }
      else {
        //console.log(data);
        nutsData.loadClusters(rScriptResultsFileName, idString, function (clusterData) {
          data = clusterData;
          helpers.formatResponse({ response: data }, req, res);
        });
      }
    }
  );
}

async function loadDatasetsIfNeeded(req, res, aoi) {
  let datasets
  switch(aoi) {
    case 'cz': datasets = _datasetsCZ; break;
    case 'africa': datasets = _datasetsAfrica; break;
    case 'eu': datasets = _datasetsEU; break;
  }
  if (!datasets) {
    try {
      if (aoi == 'cz') {
        _datasetsCZ = await nutsData.loadDatasets(_dataCzFilePath)
        datasets = _datasetsCZ
      } else if (aoi == 'africa') {
        _datasetsAfrica = await nutsData.loadDatasets(_dataAfricaFilePath)
        datasets = _datasetsAfrica
      } else {
        _datasetsEU = await nutsData.loadDatasets(_dataEuFilePath)
        datasets = _datasetsEU
      }
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  return datasets
}

async function loadRuralDataIfNeeded(req, res, aoi) {
  let ruralData
  switch(aoi) {
    case 'cz': ruralData = _ruralDataCZ; break;
    case 'africa': ruralData = _ruralDataAfrica; break;
    case 'eu': ruralData = _ruralDataEU; break;
  }
  if (!ruralData) {
    try {
      if (aoi == 'cz') {
        _ruralDataCZ = await nutsData.loadRuralData(_dataCzFilePath)
        ruralData = _ruralDataCZ
      } else if (aoi == 'africa') {
        _ruralDataAfrica = await nutsData.loadRuralData(_dataAfricaFilePath)
        ruralData = _ruralDataAfrica
      } else {
        _ruralDataEU = await nutsData.loadRuralData(_dataEuFilePath)
        ruralData = _ruralDataEU
      }
    } catch (e) {
      res.send(e.toString() || e)
    }
  }
  return ruralData
}