Feat/buildings preprocessing

cbsurge/admin/__init__.py

@@ -1,13 +1,3 @@
-# from cbsurge.admin.ocha import ARCGIS_SERVER_ROOT
-# from cbsurge.admin.ocha import OCHA_COD_ARCGIS_SERVER_ROOT
-# from cbsurge.admin.ocha import http_get_json
-# from cbsurge.admin.osm import OVERPASS_API_URL
-# import asyncio
-#
-# asyncio.run(http_get_json(ARCGIS_SERVER_ROOT, timeout=10))
-# asyncio.run(http_get_json(OCHA_COD_ARCGIS_SERVER_ROOT, timeout=10))
-# asyncio.run(http_get_json(OVERPASS_API_URL, timeout=10))
-#
 
 import logging
 from cbsurge.admin.osm import fetch_admin as fetch_osm_admin, ADMIN_LEVELS
@@ -16,28 +6,7 @@
 import click
 import json
 
-def silence_httpx_az():
-    azlogger = logging.getLogger('azure.core.pipeline.policies.http_logging_policy')
-    azlogger.setLevel(logging.WARNING)
-    httpx_logger = logging.getLogger('httpx')
-    httpx_logger.setLevel(logging.WARNING)
-
-class BboxParamType(click.ParamType):
-    name = "bbox"
-
-    def convert(self, value, param, ctx):
-        try:
-            bbox = [float(x.strip()) for x in value.split(",")]
-            fail = False
-        except ValueError:  # ValueError raised when passing non-numbers to float()
-            fail = True
-
-        if fail or len(bbox) != 4:
-            self.fail(
-                f"bbox must be 4 floating point numbers separated by commas. Got '{value}'"
-            )
-
-        return bbox
+logger = logging.getLogger(__name__)
 
 @click.group()
 def admin():
@@ -102,7 +71,6 @@ def osm(bbox=None,admin_level=None, osm_level=None, clip=False, h3id_precision=7
 
     rapida admin osm -b "27.767944,-5.063586,31.734009,-0.417477" -l 0 > osm.geojson
     """
-    silence_httpx_az()
     logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)
     geojson = fetch_osm_admin(bbox=bbox, admin_level=admin_level,osm_level=osm_level, clip=clip, h3id_precision=h3id_precision)
     if geojson:
@@ -158,8 +126,8 @@ def ocha(bbox=None,admin_level=None,  clip=False, h3id_precision=7, debug=False)
 
     rapida admin ocha -b "27.767944,-5.063586,31.734009,-0.417477" -l 0 > ocha.geojson
     """
-    silence_httpx_az()
-    logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)
+    #logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)
+    logger.info('HERE')
     geojson = fetch_ocha_admin(bbox=bbox, admin_level=admin_level, clip=clip, h3id_precision=h3id_precision)
     if geojson:
         click.echo(json.dumps(geojson))

cbsurge/admin/ocha.py

@@ -81,7 +81,7 @@ def fetch_ocha_countries(bounding_box = None, ):
                         countries.append(service_country)
                 else:
                     countries.append(service_country)
-        return tuple(countries)
+        return tuple(set(countries))
     except Exception as e:
         logger.error(f'Failed to fetch available countries. {e}')
         raise
@@ -152,6 +152,7 @@ def fetch_admin(bbox=None, admin_level=None, clip=False,h3id_precision=7, ):
     bounding box covers several countries.
 
     """
+
     west, south, east, north = bbox
     try:
         int(admin_level)
@@ -171,6 +172,8 @@ def fetch_admin(bbox=None, admin_level=None, clip=False,h3id_precision=7, ):
 
 
     ocha_countries = fetch_ocha_countries()
+    print(len(ocha_countries))
+
 
     timeout = httpx.Timeout(connect=10, read=1800, write=1800, pool=1000)
     bbox_polygon = box(west, south, east, north)

cbsurge/cli.py

@@ -1,10 +1,20 @@
-import click as click
-from cbsurge.initialize import init
+
+from cbsurge.util import setup_logger
 from cbsurge.admin import admin
+from cbsurge.initialize import init
 from cbsurge.exposure.builtenv import builtenv
+import click
+
+@click.group
+def cli(ctx):
+    """Main CLI for the application."""
+    pass
+cli.add_command(admin)
+cli.add_command(builtenv)
+
 from cbsurge.exposure.population import population
 from cbsurge.stats import stats
-import click
+
 
 @click.group
 
@@ -21,4 +31,6 @@ def cli():
 
 
 if __name__ == '__main__':
+
+    logger = setup_logger('rapida', make_root=True)
     cli()

cbsurge/exposure/builtenv/__init__.py

@@ -1,8 +1,8 @@
 import click
-
+from cbsurge.exposure.builtenv.buildings import buildings
 @click.group()
 def builtenv():
     f"""Command line interface for {__package__} package"""
     pass
 
-
+builtenv.add_command(buildings)

cbsurge/exposure/builtenv/buildings/__init__.py

@@ -1,8 +1,95 @@
-import click
-
+from email.policy import default
 
+import click
+from cbsurge.exposure.builtenv.buildings.fgb import download_bbox, download_admin
+from cbsurge.exposure.builtenv.buildings.pmt import download as download_pmt, GMOSM_BUILDINGS
+from cbsurge.util import BboxParamType
+import asyncio
 
 @click.group()
 def buildings():
     f"""Command line interface for {__package__} package"""
     pass
+
+@click.group()
+def download():
+    f"""Command line interface for {__package__} package"""
+    pass
+
+
+@download.command(no_args_is_help=True)
+
+@click.option('-b', '--bbox', required=True, type=BboxParamType(),
+              help='Bounding box xmin/west, ymin/south, xmax/east, ymax/north' )
+@click.option('-o', '--out-path', required=True, type=click.Path(),
+              help='Full path to the buildings dataset' )
+
+@click.option('-bs', '--batch-size', type=int, default=65535,
+              help='The max number of buildings to be dowloaded in one chunk or batch. '
+                   )
+def fgbbbox(bbox=None, out_path=None, batch_size:[int,None]=1000):
+    """
+        Download/stream buildings from VIDA buildings using pyogrio/pyarrow API
+
+        :param bbox: iterable of floats, xmin, ymin, xmax,ymax
+        :param out_path: str, full path where the buildings layer will be written
+        :param batch_size: int, default=1000, the max number of buildings to download in one batch
+        If supplied, the buildings are downloaded in batches otherwise they are streamd through pyarrow library
+
+    """
+    download_bbox(bbox=bbox, out_path=out_path, batch_size=batch_size)
+
+
+
+@download.command(no_args_is_help=True)
+@click.option('-a', '--admin-path', required=True, type=click.Path(),
+              help='Full path to the admin dataset' )
+@click.option('-o', '--out-path', required=True, type=click.Path(),
+              help='Full path to the buildings dataset' )
+@click.option('--country-col-name', required=True, type=str,
+              help='The name of the column from the admin layer attributes that contains the ISO3 country code' )
+@click.option('--admin-col-name', required=True, type=str,
+              help='The name of the column from the admin layer attributes that contains the admin unit name' )
+@click.option('-bs', '--batch-size', type=int, default=65535,
+              help='The max number of buildings to be dowloaded in one chunk or batch. ')
+
+def fgbadmin(admin_path=None, out_path=None, country_col_name=None, admin_col_name=None, batch_size=None):
+
+    """Fetch buildings from VIDA FGB based on the boundaries of the admin units"""
+
+
+    download_admin(admin_path=admin_path,out_path=out_path,
+                   country_col_name=country_col_name, admin_col_name=admin_col_name, batch_size=batch_size)
+
+
+
+@download.command(no_args_is_help=True)
+
+@click.option('-b', '--bbox', required=True, type=BboxParamType(),
+              help='Bounding box xmin/west, ymin/south, xmax/east, ymax/north' )
+@click.option('-o', '--out-path', required=True, type=click.Path(),
+              help='Full path to the buildings dataset' )
+
+@click.option('-z', '--zoom-level', type=int,
+              help='The zoom level from which to fetch the buildings.Defaults to dataset max zoom-1')
+@click.option('-x', type=int,
+              help='The x coordinate of the til;e for a specific zoom level')
+@click.option('-y', type=int,
+              help='The y coordinate of the til;e for a specific zoom level')
+
+def pmt(bbox=None, out_path=None, zoom_level=None,  x=None,y=None):
+    """
+
+        Fetch buildings from url remote source in PMTiles format from VIDA
+        https://data.source.coop/vida/google-microsoft-osm-open-buildings/pmtiles/goog_msft_osm.pmtiles
+
+        :param bbox: iterable of floats, xmin, ymin, xmax,ymax
+        :param out_path: str, full path where the buildings layer will be written
+        :param zoom_level: int the zoom level, defaults to max zoom level -1
+        :param x:int,  the tile x coordinate at a specific zoom level, can be used as a filter
+        :param y: int, the tile y coordinate at a specific zoom level, can be sued as a filter
+
+    """
+    asyncio.run(download_pmt(bbox=bbox,out_path=out_path, zoom_level=zoom_level, x=x, y=y))
+
+buildings.add_command(download)