Cleaning of the project: deleted useless files

README.md

@@ -26,18 +26,18 @@ is disable.
 ##### docker-compose.yaml
 ```yaml
 services:
-  fcs2:
-    container_name: fieldclimateSenslog2
+  fcs1:
+    container_name: fieldclimateSenslog1
     build:
       dockerfile: Dockerfile
       context: .
       args:
-         MAVEN_PROFILE: FieldClimateSenslog2
+         MAVEN_PROFILE: FieldClimateSenslog1
     ports:
       - "5005:5005"
     restart: always
     environment:
-        APP_PARAMS: -cf config/test.yaml
+        APP_PARAMS: -cf config/fieldclimateSenslog1.yaml
         DEBUG: "true"
 
 ```

config/default.yaml

@@ -1,44 +0,0 @@
-settings:
-    - AzureLoraWan:
-        name: "IoT LoraWan"
-        provider: "cz.senslog.connector.fetch.azure.ConnectorFetchAzureProvider"
-        startDate: 2019-07-13T11:37:00.000
-        endDate: 2019-07-13T11:37:14.900
-        limitPerSensor: 60
-        
-        sensorInfoHost:
-            domain: "https://iotlorawan.azurewebsites.net"
-            path: "api/sensors"
-        
-        sensorDataHost:
-            domain: "https://iotlorawan.azurewebsites.net"
-            path: "api/sensordata"
-        
-        authentication:
-            host:
-                domain: "https://iotlorawan.azurewebsites.net"
-                path: "api/accounts/login"
-            username: "<username>"
-            password: "<password>"
-            refreshPeriodIfFail: 10000
-        
-    - SenslogV1:
-        name: "Senslog V1"
-        provider: "cz.senslog.connector.push.rest.senslog.v1.SenslogV1ConnectorPushProvider"
-        host:
-            domain: "http://localhost:8080"
-            path: "DBService-1.4-SNAPSHOT/FeederServlet"
-
-    - SenslogV2:
-          name: "Senslog V2"
-          provider: "cz.senslog.connector.push.rest.senslog.v2.SenslogV2ConnectorPushProvider"
-
-    - Fieldclimate:
-          name: "FieldClimate: Pessl Instruments "
-          provider: "cz.senslog.connector.fetch.fieldclimate.ConnectorFetchFieldClimateProvider"
-
-connectors:
-    - AzureSenslogV1:
-        fetcher: "AzureLoraWan"
-        pusher: "SenslogV1"
-        period: 60

config/fieldclimateSenslog1.yaml

@@ -9,7 +9,7 @@ settings:
   - Fieldclimate:
       name: "FieldClimate: Pessl Instruments"
       provider: "cz.senslog.connector.fetch.fieldclimate.ConnectorFetchFieldClimateProvider"
-      startDate: 2019-03-08T00:00:00.000
+      startDate: 2019-11-07T23:00:00.000
 
       <<: &apiDomain
         domain: "https://api.fieldclimate.com/v1"
@@ -35,5 +35,5 @@ connectors:
     - FieldclimateToV1:
         fetcher: "Fieldclimate"
         pusher: "SenslogV1"
-        period: 3000
+        period: 300
         initDelay: 5

config/test.yaml

@@ -1,21 +0,0 @@
-settings:
-    - SenslogV2:
-          name: "Senslog V2"
-          provider: "cz.senslog.connector.push.rest.senslog.v2.SenslogV2ConnectorPushProvider"
-
-    - Fieldclimate:
-          name: "FieldClimate: Pessl Instruments"
-          provider: "cz.senslog.connector.fetch.fieldclimate.ConnectorFetchFieldClimateProvider"
-
-connectors:
-    - FieldclimateToV2_1:
-          fetcher: "Fieldclimate"
-          pusher: "SenslogV2"
-          period: 2
-          initDelay: 2
-
-    - FieldclimateToV2_2:
-        fetcher: "Fieldclimate"
-        pusher: "SenslogV2"
-        period: 2
-        initDelay: 2

connector-app/src/main/java/cz/senslog/connector/app/Application.java

@@ -106,11 +106,10 @@ class Application extends Thread {
 
 
         ServiceProvider serviceProvider = new ServiceProvider(ConnectorFetch::getProvider, ConnectorPush::getProvider);
-        ModelConverterProvider connectorProvider = new ModelConverterProvider();
-        ConnectorBuilder connectorBuilder = ConnectorBuilder.init(serviceProvider, connectorProvider, configService);
+        ModelConverterProvider converterProvider = new ModelConverterProvider();
+        ConnectorBuilder connectorBuilder = ConnectorBuilder.init(serviceProvider, converterProvider, configService);
         Set<Connector> connectors = connectorBuilder.createConnectors();
 
-
         if (!connectors.isEmpty()) {
             scheduler = Executors.newScheduledThreadPool(connectors.size());
             latch = new CountDownLatch(connectors.size());

connector-common/src/main/java/cz/senslog/connector/json/Formatter.java

@@ -1,5 +0,0 @@
-package cz.senslog.connector.json;
-
-public interface Formatter<T> {
-    String format(T object);
-}

connector-fetch-fieldclimate/src/main/java/cz/senslog/connector/fetch/fieldclimate/FieldClimateFetcher.java

@@ -121,7 +121,7 @@ public class FieldClimateFetcher implements ConnectorFetcher<FieldClimateModel>
 
             logger.debug("Getting a start date time for station {}.", stationId);
             LocalDateTime localFrom = stationTimeRanges.get(stationId);
-            LocalDateTime localTo = localFrom.plusDays(1); // default interval (5h around 600 observations)
+            LocalDateTime localTo = localFrom.plusHours(5); // default interval (5h around 600 observations)
 
             StationTimeRange timeRange = getTimeRangeForStation(stationId);
             if (timeRange == null) break;
@@ -156,35 +156,49 @@ public class FieldClimateFetcher implements ConnectorFetcher<FieldClimateModel>
             HttpResponse response = httpClient.send(request);
             logger.info("Received a response with a status: {}.", response.getStatus());
 
-            if (response.isError()) {
-                logger.error("Can not get data from the station {}. Error {} {}",
-                        stationId, response.getStatus(), response.getBody());
-                continue;
-            }
 
-            StationData stationData;
-            try {
-                logger.debug("Parsing body of the response to the class {}.", StationData.class);
-                stationData = jsonToObject(response.getBody(), StationData.class);
-                logger.info("Received {} records for the sensor {}.", stationData.getData().size(), stationId);
-            } catch (SyntaxException e) {
-                logger.catching(e); continue;
-            }
+            if (response.isOk()) {
+
+                StationData stationData;
+                try {
+                    logger.debug("Parsing body of the response to the class {}.", StationData.class);
+                    stationData = jsonToObject(response.getBody(), StationData.class);
+                    logger.info("Received {} records for the sensor {}.", stationData.getData().size(), stationId);
+                } catch (SyntaxException e) {
+                    logger.catching(e);
+                    continue;
+                }
 
-            stationData.setId(stationId);
+                stationData.setId(stationId);
 
-            LocalDateTime latestTimestamp = LocalDateTime.MIN;
-            for (Map<String, String> sensorDataMap : stationData.getData()) {
-                String strDate = sensorDataMap.getOrDefault("date", "");
-                LocalDateTime timestamp = LocalDateTime.parse(strDate, ofPattern("yyyy-MM-dd HH:mm:ss"));
-                sensorDataMap.put("date", timestamp.format(ISO_DATE_TIME));
+                LocalDateTime latestTimestamp = LocalDateTime.MIN;
+                for (Map<String, String> sensorDataMap : stationData.getData()) {
+                    String strDate = sensorDataMap.getOrDefault("date", "");
+                    LocalDateTime timestamp = LocalDateTime.parse(strDate, ofPattern("yyyy-MM-dd HH:mm:ss"));
+                    sensorDataMap.put("date", timestamp.format(ISO_DATE_TIME));
 
-                if (latestTimestamp.isBefore(timestamp)) {
-                    latestTimestamp = timestamp;
+                    if (latestTimestamp.isBefore(timestamp)) {
+                        latestTimestamp = timestamp;
+                    }
                 }
-            }
 
-            localTo = latestTimestamp.isAfter(localTo) ? latestTimestamp : localTo;
+                localTo = latestTimestamp.isAfter(localTo) ? latestTimestamp : localTo;
+
+                stationDataList.add(stationData);
+
+                int records = stationData.getSensors().size();
+                totalFetched += records;
+
+                logger.info("Fetched {} records for the station {}.", records, stationId);
+
+            } else {
+
+                if (response.getStatus() >= 300) {
+                    logger.error("Can not get data from the station {}. Error {} {}",
+                            stationId, response.getStatus(), response.getBody());
+                    continue;
+                }
+            }
 
             stationTimeRanges.put(stationId, localTo);
 
@@ -195,13 +209,6 @@ public class FieldClimateFetcher implements ConnectorFetcher<FieldClimateModel>
             if (globalTo.isBefore(localTo)) {
                 globalTo = localTo;
             }
-
-            stationDataList.add(stationData);
-
-            int records = stationData.getSensors().size();
-            totalFetched += records;
-
-            logger.info("Fetched {} records for the station {}.", records, stationId);
         }
 
         FieldClimateModel model;

connector-model/src/main/java/cz/senslog/connector/model/converter/FieldClimateModelSenslogV1ModelConverter.java

@@ -93,7 +93,11 @@ public class FieldClimateModelSenslogV1ModelConverter implements Converter<Field
                             continue;
                         }
 
-                        Float value = Float.valueOf(dataMap.get(sensorDataHash));
+                        String strValue = dataMap.get(sensorDataHash);
+
+                        if (strValue == null) continue;
+
+                        Float value = Float.valueOf(strValue);
                         String strDate = dataMap.getOrDefault("date", "");
                         ZonedDateTime time = LocalDateTime.parse(strDate, ISO_DATE_TIME).atZone(UTC);
 

example-json/sensor_data.json

@@ -1,96 +0,0 @@
-{
-    "eui": "0E7E3464333100B6",
-    "name": "Ascoel CO2 Establish",
-    "data": [
-        {
-            "time": "2019-06-13T11:47:27+02:00",
-            "temperature": 25.2,
-            "humidity": 45.87,
-            "co2": 578.0,
-            "rssi": -122,
-            "snr": 2.0,
-            "batteryLevel": 3.6
-        },
-        {
-            "time": "2019-06-13T11:52:28+02:00",
-            "temperature": 25.3,
-            "humidity": 45.87,
-            "co2": 578.0,
-            "rssi": -111,
-            "snr": -10.0,
-            "batteryLevel": 3.6
-        },
-        {
-            "time": "2019-06-13T12:02:44+02:00",
-            "temperature": 25.4,
-            "humidity": 45.87,
-            "co2": 681.0,
-            "rssi": -116,
-            "snr": 4.8,
-            "batteryLevel": 3.6
-        },
-        {
-            "time": "2019-06-13T12:07:45+02:00",
-            "temperature": 25.5,
-            "humidity": 46.1,
-            "co2": 681.0,
-            "rssi": -121,
-            "snr": 0.2,
-            "batteryLevel": 3.6
-        },
-        {
-            "time": "2019-06-13T12:13:00+02:00",
-            "temperature": 25.5,
-            "humidity": 46.1,
-            "co2": 680.0,
-            "rssi": -120,
-            "snr": -8.2,
-            "batteryLevel": 3.6
-        },
-        {
-            "time": "2019-06-13T12:18:01+02:00",
-            "temperature": 25.5,
-            "humidity": 46.24,
-            "co2": 680.0,
-            "rssi": -117,
-            "snr": -6.0,
-            "batteryLevel": 3.6
-        },
-        {
-            "time": "2019-06-13T12:28:17+02:00",
-            "temperature": 25.5,
-            "humidity": 46.37,
-            "co2": 679.0,
-            "rssi": -117,
-            "snr": -2.8,
-            "batteryLevel": 3.6
-        },
-        {
-            "time": "2019-06-13T12:38:19+02:00",
-            "temperature": 25.6,
-            "humidity": 46.51,
-            "co2": 679.0,
-            "rssi": -118,
-            "snr": -4.5,
-            "batteryLevel": 3.6
-        },
-        {
-            "time": "2019-06-13T12:43:34+02:00",
-            "temperature": 25.6,
-            "humidity": 46.1,
-            "co2": 674.0,
-            "rssi": -116,
-            "snr": -4.8,
-            "batteryLevel": 3.6
-        },
-        {
-            "time": "2019-06-13T12:53:36+02:00",
-            "temperature": 25.7,
-            "humidity": 46.1,
-            "co2": 674.0,
-            "rssi": -121,
-            "snr": -5.2,
-            "batteryLevel": 3.6
-        }
-    ]
-}

example-json/sensors_info.json

@@ -1,30 +0,0 @@
-[
-    {
-        "eui": "8CF9574000000948",
-        "acronym": "RHF-948",
-        "name": "AFarCloud - AS07 - IAS - Outdoor",
-        "period": 600,
-        "tolerance": 3,
-        "location": "VÚŽV Uhříněvěs",
-        "description": "Monitoruje teplotu a vlhkost na výzkumné farmě v Netlukách.",
-        "sensorDataId": 2078282,
-        "sensorDataTime": "2019-08-14T17:02:51",
-        "created": "2019-06-03T12:07:32.807",
-        "deleted": null,
-        "status": 1
-    },
-    {
-        "eui": "8CF95740000008AE",
-        "acronym": "RHF-8AE",
-        "name": "AFarCloud - AS07 - IAS - Indoor",
-        "period": 600,
-        "tolerance": 3,
-        "location": "VÚŽV Uhříněvěs",
-        "description": "Monitoruje teplotu a vlhkost v kravíně na výzkumné farmě v Netlukách.",
-        "sensorDataId": 2077979,
-        "sensorDataTime": "2019-08-14T16:30:32",
-        "created": "2019-06-07T16:05:38.42",
-        "deleted": null,
-        "status": 1
-    }
-]

run.sh

@@ -1,3 +0,0 @@
-mvn clean
-mvn package
-java -cp "bin/*" cz.senslog.connector.app.Main -cf config/default.yaml