Fieldclimate: edited sensor types, added period time to config file, edited edge of time window

config/fieldclimateSenslog1.yaml

@@ -9,7 +9,9 @@ settings:
   - Fieldclimate:
       name: "FieldClimate: Pessl Instruments"
       provider: "cz.senslog.connector.fetch.fieldclimate.ConnectorFetchFieldClimateProvider"
-      startDate: 2019-11-07T23:00:00.000
+      startDate: 2019-01-01T00:00:00.000
+#      startDate: 2019-11-07T23:00:00.000
+      period: 12 # in hours
 
       <<: &apiDomain
         domain: "https://api.fieldclimate.com/v1"
@@ -35,5 +37,5 @@ connectors:
     - FieldclimateToV1:
         fetcher: "Fieldclimate"
         pusher: "SenslogV1"
-        period: 300
-        initDelay: 5
+        period: 10
+        initDelay: 1

connector-fetch-fieldclimate/src/main/java/cz/senslog/connector/fetch/fieldclimate/FieldClimateConfig.java

@@ -13,6 +13,7 @@ public class FieldClimateConfig {
     private final HostConfig stationDataHost;
     private final HostConfig stationTimeRangeHost;
     private final AuthConfig authentication;
+    private final Integer period;
 
     public FieldClimateConfig(DefaultConfig config) {
         this.startDate = config.getLocalDateTimeProperty("startDate");
@@ -20,6 +21,7 @@ public class FieldClimateConfig {
         this.stationDataHost = new HostConfig(config.getPropertyConfig("stationDataHost"));
         this.stationTimeRangeHost = new HostConfig(config.getPropertyConfig("stationTimeRangeHost"));
         this.authentication = new AuthConfig(config.getPropertyConfig("authentication"));
+        this.period = config.getIntegerProperty("period");
     }
 
     public LocalDateTime getStartDate() {
@@ -41,4 +43,8 @@ public class FieldClimateConfig {
     public AuthConfig getAuthentication() {
         return authentication;
     }
+
+    public Integer getPeriod() {
+        return period;
+    }
 }

connector-fetch-fieldclimate/src/main/java/cz/senslog/connector/fetch/fieldclimate/FieldClimateFetcher.java

@@ -121,18 +121,24 @@ public class FieldClimateFetcher implements ConnectorFetcher<FieldClimateModel>
 
             logger.debug("Getting a start date time for station {}.", stationId);
             LocalDateTime localFrom = stationTimeRanges.get(stationId);
-            LocalDateTime localTo = localFrom.plusHours(5); // default interval (5h around 600 observations)
 
             StationTimeRange timeRange = getTimeRangeForStation(stationId);
-            if (timeRange == null) break;
+            if (timeRange == null) continue;
 
             if (localFrom.isBefore(timeRange.getMin_date())) {
                 localFrom = timeRange.getMin_date();
             }
 
+            LocalDateTime localTo = localFrom.plusHours(config.getPeriod());
+
             if (localTo.isAfter(timeRange.getMax_date())) {
                 localTo = timeRange.getMax_date();
             }
+
+            if (localFrom.equals(localTo)) {
+                logger.info("No new data for the station {} is available at this moment.", stationId); continue;
+            }
+
             logger.info("New data for the station {} are going to be fetched from {} to {}.",
                     stationId, localFrom, localTo);
 
@@ -221,9 +227,8 @@ public class FieldClimateFetcher implements ConnectorFetcher<FieldClimateModel>
 
         } else {
             logger.warn("No data was fetched.");
-
-            LocalDateTime date = config.getStartDate();
-            model = new FieldClimateModel(emptyList(), date, date);
+            LocalDateTime now = LocalDateTime.now();
+            model = new FieldClimateModel(emptyList(), now, now);
         }
 
         return model;

connector-fetch-fieldclimate/src/test/java/cz/senslog/connector/fetch/fieldclimate/FieldClimateFetcherTest.java

@@ -35,7 +35,7 @@ class FieldClimateFetcherTest {
 
     private static FieldClimateConfig config;
     private static AuthenticationService authService;
-    private static LocalDateTime startDate = LocalDateTime.of(2019, 10, 1, 0, 0,0);
+    private static LocalDateTime startDate = LocalDateTime.of(2019, 3, 6, 00, 00,0);
 
     @BeforeAll
     static void init() {
@@ -61,6 +61,7 @@ class FieldClimateFetcherTest {
         }});
 
         fcDConfig.setProperty("startDate", startDate);
+        fcDConfig.setProperty("period", 12);
 
         config = new FieldClimateConfig(fcDConfig);
         authService = new AuthenticationService(config.getAuthentication());
@@ -88,7 +89,6 @@ class FieldClimateFetcherTest {
         FieldClimateFetcher fetcher = new FieldClimateFetcher(config, authService, httpClient);
 
         fetcher.init();
-
     }
 
     @Test
@@ -370,6 +370,60 @@ class FieldClimateFetcherTest {
         assertEquals(100, Integer.valueOf(data.get(sensorDataHash)));
     }
 
+    @Test
+    void fetch_TimeRange_EmptyModel() throws Exception {
+        String stationName = "original_name";
+        HttpClient httpClient = mock(HttpClient.class);
+        when(httpClient.send(any(HttpRequest.class))).thenAnswer((Answer<HttpResponse>) invocationOnMock -> {
+            HttpRequest request = invocationOnMock.getArgument(0, HttpRequest.class);
+            String path = request.getUrl().getPath();
+            if (path.contains("/user/stations")) {
+                StationInfo stationInfo = new StationInfo();
+                StationInfo.Name name = new StationInfo.Name();
+                name.setOriginal(stationName);
+                stationInfo.setName(name);
+                List<StationInfo> stations = singletonList(stationInfo);
+                return HttpResponse.newBuilder()
+                        .status(OK).body(stations.toString()).build();
+            }
+            if (path.contains("/data/"+stationName)) {
+                String json = String.format("{\"min_date\":\"%s\",\"max_date\":\"%s\"}",
+                        startDate.minusDays(1).format(ofPattern("yyyy-MM-dd HH:mm:ss")),
+                        startDate.plusDays(1).format(ofPattern("yyyy-MM-dd HH:mm:ss"))
+                );
+                return HttpResponse.newBuilder().status(OK).body(json).build();
+            }
+            if (path.contains("/data/normal/"+stationName)) {
+                StationData data = new StationData();
+                data.setId(stationName);
+                SensorDataInfo dataInfo = new SensorDataInfo();
+                Map<String, String> dataMap = new HashMap<>();
+                dataMap.put("date", startDate.plusHours(1).format(ofPattern("yyyy-MM-dd HH:mm:ss")));
+                data.setSensors(singletonList(dataInfo));
+                data.setData(singletonList(dataMap));
+                String json = data.toString();
+                return HttpResponse.newBuilder()
+                        .status(OK).body(json).build();
+            }
+            return HttpResponse.newBuilder()
+                    .status(SERVER_ERROR).build();
+        });
+        FieldClimateFetcher fetcher = new FieldClimateFetcher(config, authService, httpClient);
+        fetcher.init();
+
+        FieldClimateModel model1 = fetcher.fetch();
+        assertEquals(startDate, model1.getFrom());
+        assertEquals(startDate.plusHours(config.getPeriod()), model1.getTo());
+
+        FieldClimateModel model2 = fetcher.fetch();
+        assertEquals(startDate.plusHours(config.getPeriod()), model2.getFrom());
+        assertEquals(startDate.plusHours(2*config.getPeriod()), model2.getTo());
+
+        FieldClimateModel model3 = fetcher.fetch();
+        assertEquals(startDate.plusHours(2*config.getPeriod()), model3.getFrom());
+        assertEquals(startDate.plusHours(2*config.getPeriod()), model3.getTo());
+
+    }
 
 
     /*
@@ -380,15 +434,30 @@ class FieldClimateFetcherTest {
 
         fetcher.init();
 
-//        fetcher.fetch();
-//        fetcher.fetch();
+        for (int i = 0; i < 10; i++) {
+            FieldClimateModel model = fetcher.fetch();
+
+            model.getStations().removeIf(s -> !s.getId().equals("0120821E"));
+            model.getStations().forEach(s -> s.getSensors().removeIf(se -> se.getCode() != 19969));
+
+            model.getStations().stream().peek(st -> System.out.println(format("%s %s-%s", st.getId(), model.getFrom(), model.getTo()))).forEach(st -> st.getSensors()
+                    .forEach(se -> System.out.println(format("\t%s\t%s\t%s", se.getName(), se.getCode(), se.getCh()))));
+
+            SenslogV1Model sModel = new FieldClimateModelSenslogV1ModelConverter().convert(model);
+
+            System.out.println("\nObservation");
+            sModel.getObservations().stream().map(s -> (Observation)s).forEach(s -> System.out.println(format("\t%s\t%s", s.getUnitId(), s.getSensorId())));
+        }
+
 
         SenslogV1Model model = new FieldClimateModelSenslogV1ModelConverter().convert(fetcher.fetch());
         Observation obs = (Observation)model.getObservations().get(0);
 
         System.out.println(obs);
+
     }
-*/
+    */
+
     /*
     private void saveToCSV(FieldClimateModel model) throws IOException {
 

connector-model/src/main/java/cz/senslog/connector/model/fieldclimate/SensorType.java

@@ -15,7 +15,7 @@ public enum SensorType {
 
     EAG_SOIL_MOISTURE       (19969, 410160000, Group.MULTICHANNEL), // [%]
     EAG_SOIL_SALINITY       (20228, 800010000, Group.MULTICHANNEL), // [VIC]
-    SOIL_TEMPERATURE        (17153, 340400000, Group.MULTICHANNEL), // [°C]
+    SOIL_TEMPERATURE        (17153, 340410000, Group.MULTICHANNEL), // [°C]
     LEAF_WETNESS            (4, 790010000),     // [min]
     WIND_SPEED              (5, 470110000),     // [m/s]
     PRECIPITATION           (6, 480070000),     // [mm]
@@ -25,7 +25,7 @@ public enum SensorType {
     SOLAR_RADIATION         (600, 620020000),   // [W/m2]
     BATTERY                 (7, 360220000),     // [mV]
     SOLAR_PANEL             (30, 360230000),    // [mV]
-    VPD                     (25, 460070000),    // [mbar]
+    VPD                     (25, 460070000),    // [mBar]
     DELTA_T                 (27, 810010000),    // [°C]
 
     HC_AIR_TEMPERATURE      (506, 340400000),   // [°C]

connector-push-rest-senslog-v1/src/main/java/cz/senslog/connector/push/rest/senslog/v1/SenslogV1Pusher.java

@@ -73,18 +73,17 @@ class  SenslogV1Pusher implements ConnectorPusher<SenslogV1Model> {
     @Override
     public void push(SenslogV1Model model) {
 
-        if (model == null || model.getObservations() == null) {
-            logger.error("Nothing is to push. Received model is empty.");
-            return;
-        }
+        if (model != null && model.getObservations() != null) {
 
-        List<Record> observations = model.getObservations();
-        logger.info("Received {} observation to send.", observations.size());
+            List<Record> observations = model.getObservations();
+            logger.info("Received {} new observations from {} to {} to push.",
+                    observations.size(), model.getFrom(), model.getTo());
 
-        logger.debug("Adding all new observations to the queue.");
-        observationQueue.addAll(observations);
+            logger.debug("Adding all new observations to the queue.");
+            observationQueue.addAll(observations);
+        }
 
-        logger.debug("Adding all failed observations to the queue.");
+        logger.info("Adding {} failed observations to the queue.", failedObservations.size());
         observationQueue.addAll(failedObservations);
         failedObservations.clear();