wfs rewritten to use more OWSLib

OWS.py

@@ -59,27 +59,48 @@ class OWS:
         if url:
             self.url = url
             logging.debug("OWS.py::__init__()::url: '%s'" % url)
-            self.__getCapabilities()
         if qstring:
             self.qstring = qstring
 
+        self.__getCapabilities()
+
+    def __getCapabilitiesFromFile(self,service,version):
+
+        if os.path.exists(os.path.join(self.cachedir,"capabilities.xml")):
+            logging.debug("OWS.py:: Using cached version of capabilities document")
+            capafile = open(os.path.join(self.cachedir,"capabilities.xml"))
+            return service(self.url,xml=capafile.read(),version=version)
+        else:
+            return None
 
     def __getCapabilities(self):
+
+        if self.url:
+            logging.debug("OWS.py::__getCapabilities::self.url: '%s'" % self.url)
+            self.parsedUrl = urlparse.urlparse(self.url)
+
         self.__getCacheDir()
-        logging.debug("OWS.py::__getCapabilities::self.url: '%s'" % self.url)
-        self.parsedUrl = urlparse.urlparse(self.url)
 
         if self.service == "WFS":
             from owslib.wfs import WebFeatureService
             # FIXME Make default version configurable
             try:
-                self.capabilities = WebFeatureService(url=self.url,version="1.1.0")
+                self.capabilities = self.__getCapabilitiesFromFile(WebFeatureService,"1.1.0")
+                if self.capabilities == None and self.url:
+                    logging.debug("OWS.py:: Downloading new capabilities file")
+                    self.capabilities = WebFeatureService(url=self.url,version="1.1.0")
             except:
-                self.capabilities = WebFeatureService(url=self.url,version="1.0.0")
+                self.capabilities = self.__getCapabilitiesFromFile(WebFeatureService,"1.0.0")
+                if self.capabilities == None and self.url:
+                    logging.debug("OWS.py:: Downloading new capabilities file")
+                    self.capabilities = WebFeatureService(url=self.url,version="1.0.0")
 
         elif self.service == "WCS":
             from owslib.wcs import WebCoverageService
-            self.capabilities = WebCoverageService(url=self.url,version="1.0.0")
+            self.capabilities = self.__getCapabilitiesFromFile(WebCoverageService,"1.0.0")
+            if self.capabilities == None and self.url:
+                logging.debug("OWS.py:: Downloading new capabilities file")
+                self.capabilities = WebCoverageService(url=self.url,version="1.0.0")
 
     def getParams(self):
         params =  urlparse.parse_qs(self.qstring)
@@ -170,9 +191,6 @@ class OWS:
             else:
                 mapobj = self.makeMap(request.getValueByName("map"))
 
-        # WFS Filter encoding, if available
-        if request.getValueByName("fes"):
-            self.setFilter(mapobj,request)
 
         # mapobj.getLayerByName(request.getValueByName("layers")).metadata.get("wfs_filter")
         res = mapobj.OWSDispatch(request)
@@ -320,6 +338,7 @@ class OWS:
         """
 
         bbox = None
+
         if layer.boundingBoxWGS84:
 
             dest = osr.SpatialReference()
@@ -347,30 +366,6 @@ class OWS:
                     (layer.id,bbox))
         return bbox
 
-    def setFilter(self,mapobj,request):
-        """ Set WFS filter encoding 
-        """
-        # get the layer
-        layerobj = mapobj.getLayerByName(request.getValueByName("layers"))
-
-        # get the filter
-        fes = request.getValueByName("fes")
-        logging.debug("FES received from HSLayers: %s" % fes)
-
-        # cut off the opening and closing <Filter> tag
-        # - this is needed for mapserver
-        #root = etree.XML(fes)
-        #msFilter = ""
-        #for child in root:
-        #    msFilter += etree.tostring(child)
-        #logging.debug("Setting the filter %s" % msFilter)
-
-        # set the filter
-        layerobj.setMetaData("wfs_filter",fes)
-        #layerobj.setMetaData("wfs_filter",msFilter)
-
-        # save the mapfile - debugging
-        mapobj.save("mapfile.fes")
 
 
 def getService(configFile=None):

wfs/__init__.py

@@ -8,10 +8,12 @@ from lxml import objectify
 import urllib
 import urlparse
 import logging
-from osgeo import ogr
-import pyproj
-import os
+from osgeo import ogr,gdal,osr
+from owslib.crs import Crs
+import os,sys
 import re
+import shutil
+import tempfile
 
 class WFS(OWS):
 
@@ -19,10 +21,219 @@ class WFS(OWS):
     wfsns = "http://www.opengis.net/wfs"
     layerDefFile = None
     lyrobj = None
+    wfs = None
 
     def __init__(self,url=None,qstring=None,configFile=None):
         OWS.__init__(self,url,qstring,configFile)
 
+    def dispatch(self):
+        """Dispatch given request
+        """
+        request = mapscript.OWSRequest()
+        request.loadParams()
+
+        typename = request.getValueByName("layers")
+        if typename:
+            layer = self.capabilities.contents[typename]
+
+        self.request=request.getValueByName("REQUEST")
+
+        # if no 'map' parameter in URL, create new mapfile
+        if not request.getValueByName("map"):
+            logging.debug("Creating new mapfile")
+            self.mapobj = self.makeMap()
+        else:
+            # there is 'map' parameter in URL and the file exists, load it
+            logging.debug("Using existing mapfile %s" % request.getValueByName("map"))
+            if os.path.isfile(request.getValueByName("map")):
+                self.mapobj = mapscript.mapObj(request.getValueByName("map"))
+            # there is 'map' parameter in URL BUT the file does not exist:
+            # create
+            else:
+                self.mapobj = self.makeMap(request.getValueByName("map"))
+
+            if self.mapobj:
+                self.mapfilename = request.getValueByName("map")
+
+        # download data
+        if self.request.upper() == "GETMAP":
+            dataFile = self.getData(request,typename,layer)
+            if dataFile:
+                #layer.data = dataFile.name
+                pass
+
+
+        res = self.mapobj.OWSDispatch(request)
+
+        layerobj = self.mapobj.getLayerByName(typename)
+
+        if mapscript.MS_DONE == res:
+            raise OWSExceptions.NoValidRequest("No valid OWS Request")
+        elif mapscript.MS_FAILURE == res:
+            pass
+            #raise OWSExceptions.RequestFailed("Request failed")
+
+    def getData(self,request,typename,layer):
+        """Download data from WFS server and store them to local harddrive
+        """
+
+        fes = request.getValueByName("fes")
+        bbox = request.getValueByName("bbox").split(",")
+        import sys
+        print >>sys.stderr, "###########",bbox
+        featureid = request.getValueByName("featureid")
+        featureversion = request.getValueByName("featureversion")
+        datadir = os.path.join(self.cachedir,"cache")
+
+        layer = self.capabilities.contents[typename]
+        crs = self.__getLayerCrs(layer.crsOptions)
+        version = request.getValueByName("version")
+
+        # get propper bbox for the WFS request
+        if version == "1.3.0":
+            requestCrs = request.getValueByName("crs")
+        else:
+            requestCrs = request.getValueByName("srs")
+        requestCrs = Crs(requestCrs)
+
+        bbox = self.__adjustBBox(bbox,requestCrs, crs,version)
+
+        # clear dir
+        outfn = None 
+
+        if os.path.isdir(datadir):
+            if os.path.isfile(os.path.join(datadir,"%s.gml"%typename)):
+                # find out bbox of the data
+                bboxfile = os.path.join(datadir,"%s.bbox"%typename)
+                filterfile = os.path.join(datadir,"%s.filter"%typename)
+                if os.path.isfile(bboxfile) and not os.path.isfile(filterfile)\
+                        and not fes:
+                    # convert "string" to [floats] as [minx,miny,maxx,maxy]
+                    storedbbox = map(lambda x: float(x), open(bboxfile).read().split(","))
+                    if storedbbox[0] <= bbox[0] and\
+                    storedbbox[1] <= bbox[1] and \
+                    storedbbox[2] >= bbox[2] and \
+                    storedbbox[3] >= bbox[3]:
+                        logging.info(
+                                "Using cached file for type [%s] with bbox [%f,%f,%f,%f], not downloading new data"%\
+                                        (typename, bbox[0],bbox[1],bbox[2],bbox[3]))
+                        # setting output file name
+                        outfn = os.path.join(datadir,"%s.gml"%typename)
+                    else:
+                        # remove pre-cached file with only little area
+                        logging.info("Removing pre-cached files %s.*"% typename)
+                        self.__clear(datadir,typename)
+        else:
+            os.mkdir(os.path.join(self.cachedir,"cache"))
+
+        # create new cache file, if it does not exist yet
+        # download the data from the server
+        if outfn == None:
+            # clear, just to be sure
+            self.__clear(datadir,typename)
+                
+            # create cached bbox
+            outbbox = open(os.path.join(datadir,"%s.bbox"%typename),"w")
+            outbbox.write("%f,%f,%f,%f"%(bbox[0],bbox[1],bbox[2],bbox[3]))
+            outbbox.close()
+
+            # create cached filter
+            if fes:
+                outfes = open(os.path.join(datadir,"%s.filter"%typename),"w")
+                outfes.write(fes)
+                outfes.close()
+
+            # create chace file
+            outfn= os.path.join(datadir,"%s.gml"%typename)
+            cacheFile = open(outfn,"w")
+
+            # download feature from WFS
+            logging.debug("Downloading data [%s] from bbox [%f,%f,%f,%f]"%\
+                            (typename, bbox[0],bbox[1],bbox[2],bbox[3]))
+            bbox.append(crs.getcode())
+            import sys
+            print >>sys.stderr, typename, bbox, fes, crs.getcode()
+            feature = self.capabilities.getfeature(
+                        typename=typename, 
+                        bbox=bbox,
+                        filter=fes,
+                        srsname=crs.getcode())
+
+            self.capabilities
+
+            cacheFile.write(feature.read())
+            cacheFile.close()
+
+        # set layer connection
+        layerobj = self.mapobj.getLayerByName(typename)
+        layerobj.connection = os.path.abspath(outfn)
+
+        # data are downloaded
+        # make sure, they have propper axis order - x,y
+        if self.capabilities.version == "1.1.0" and crs.axisorder == "yx":
+            logging.debug("Setting GML_INVERT_AXIS_ORDER_IF_LAT_LONG variable to YES")
+            gdal.SetConfigOption("GML_INVERT_AXIS_ORDER_IF_LAT_LONG","YES")
+            gdal.SetConfigOption("GML_CONSIDER_EPSG_AS_URN","YES")
+
+            # >>> from osgeo import ogr
+            # >>> from osgeo import gdal
+            # >>> gdal.SetConfigOption("GML_INVERT_AXIS_ORDER_IF_LAT_LONG","YES")
+            # >>> gdal.SetConfigOption("GML_CONSIDER_EPSG_AS_URN","YES")
+            # >>> ind = ogr.Open("data.xml")
+            # >>> outd = ogr.GetDriverByName("GML")
+            # >>> outf = outd.CopyDataSource(ind,"out4.xml")
+            # >>> outf.Destroy()
+
+        logging.info("Connection of [%s] set to %s" % (typename,layerobj.connection))
+        return outfn
+
+    def __adjustBBox(self,bbox,src,target,version):
+        """ adjust bounding coordinates and axis order
+        """
+
+        # swap axis, if needed
+        if version == "1.3.0" and src.axisorder == "yx":
+            bbox[0],bbox[1] = bbox[1],bbox[0]
+            bbox[2],bbox[3] = bbox[3],bbox[2]
+
+        # coordinate transformation
+        projsrc = osr.SpatialReference()
+        projsrc.ImportFromEPSG(src.code)
+
+        projtarget = osr.SpatialReference()
+        projtarget.ImportFromEPSG(target.code)
+
+        trans = osr.CoordinateTransformation(projsrc,projtarget)
+        bbox[0],bbox[1],z = trans.TransformPoint(float(bbox[0]),float(bbox[1]))
+        bbox[2],bbox[3],z = trans.TransformPoint(float(bbox[2]),float(bbox[3]))
+
+        return bbox
+
+    def setFilter(self,mapobj,request):
+        """ Set WFS filter encoding 
+        """
+        # get the layer
+        layerobj = mapobj.getLayerByName(request.getValueByName("layers"))
+
+
+        # get the filter
+        logging.debug("FES received from HSLayers: %s" % fes)
+
+        # cut off the opening and closing <Filter> tag
+        # - this is needed for mapserver
+        #root = etree.XML(fes)
+        #msFilter = ""
+        #for child in root:
+        #    msFilter += etree.tostring(child)
+        #logging.debug("Setting the filter %s" % msFilter)
+
+        # set the filter
+        layerobj.setMetaData("wfs_filter",fes)
+        #layerobj.setMetaData("wfs_filter",msFilter)
+
+        # save the mapfile - debugging
+        mapobj.save("mapfile.fes")
+
     def makeMap(self,mapfilename=None):
 
         mapobj = self.getMapObj(mapfilename)
@@ -70,13 +281,9 @@ class WFS(OWS):
                 lyrobj.setMetaData("wfs_title",layer.title)
             #if layer.abstract:
             #    lyrobj.setMetaData("ows_abstract",  layer.abstract)
-            lyrobj.setMetaData("wfs_typename", name)
             logging.debug("WFS version %s",self.capabilities.version)
-            lyrobj.setMetaData("wfs_version",self.capabilities.version)
             lyrobj.setMetaData("gml_include_items","all")
-            lyrobj.setMetaData("wfs_request_method","GET")
-            lyrobj.setConnectionType(mapscript.MS_WFS,'')
-            lyrobj.connection = self.getLayerUrl()
+            lyrobj.setConnectionType(mapscript.MS_OGR,'')
             lyrobj.data = re.sub(r".*:","",name)
             crs = self.__getLayerCrs(layer.crsOptions)
             if ds:
@@ -191,4 +398,11 @@ class WFS(OWS):
                 return crs
         return crss[0]
 
+    def __clear(self,datadir,typename):
+        """Remove all cached files with following typename
+        """
 
+        for i in os.listdir(datadir):
+            if i.find(typename) == 0:
+                logging.debug("Removing pre-cached file %s"% i)
+                os.remove(os.path.join(datadir,i))