From e6972e3327e69dbd71a7f491a837ac3acfdb8273 Mon Sep 17 00:00:00 2001 From: ischlo Date: Mon, 11 Nov 2024 11:37:55 +0000 Subject: [PATCH] rast converter working --- src/scalenav/rast_converter.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/scalenav/rast_converter.py b/src/scalenav/rast_converter.py index 4284073..314ccca 100644 --- a/src/scalenav/rast_converter.py +++ b/src/scalenav/rast_converter.py @@ -18,6 +18,7 @@ from pyarrow.parquet import ParquetWriter from glob import glob from pyproj import Transformer +from tqdm import tqdm def rast_converter(in_path, out_path="rast_convert.parquet"): @@ -112,9 +113,10 @@ def rast_converter(in_path, out_path="rast_convert.parquet"): description='Convert rasters to parquet files', epilog='') - parser.add_argument('in_path',help="A path to a folder containing '.tif' files.") - parser.add_argument('out_path',nargs="?",default='rast_convert.parquet',help="A '.parquet' file to save into. Will be created or overwriten on execution. Default: %(default)s") - parser.add_argument('dst_crs',nargs="?",default='EPSG:4326',help="A crs value for the output.", type=str) + parser.add_argument('in_path',help="A path to a file or folder containing '.tif' files.") + parser.add_argument('out_path',nargs=1,default='rast_convert.parquet',help="A '.parquet' file to save into. Will be created or overwriten on execution. Default: %(default)s") + parser.add_argument('dst_crs',nargs=1,default='EPSG:4326',help="A crs value for the output.", type=str) + parser.add_argument('include_negative',nargs=1,default=False,help="Whether the data to process includes relevant negative or 0 values. Can have a significant impact on running time and output size.", type=str) # parser.add_argument('-v', '--verbose', # action='store_true') @@ -124,20 +126,18 @@ def rast_converter(in_path, out_path="rast_convert.parquet"): in_path = args["in_path"] out_path = args["out_path"] dst_crs = args["dst_crs"] # epsg:4326 by default. + include = args["include_negative"] # epsg:4326 by default. + if not search(pattern=r".parquet$",string=out_path): raise ValueError("Provide a 'parquet' filename to write the outputs.") vrt_options = { - # 'resampling': Resampling.cubic, 'crs': dst_crs, - # 'transform': dst_transform, - # 'height': dst_height, - # 'width': dst_width, } - rast_schema = schema([('lon',float32()) - ,('lat',float32()) + rast_schema = schema([('lon',float16()) + ,('lat',float16()) ,('band_var',float32()) ]) rast_schema.with_metadata({ @@ -176,7 +176,7 @@ def rast_converter(in_path, out_path="rast_convert.parquet"): win_transfrom = vrt.window_transform - for _, window in vrt.block_windows(): + for _, window in tqdm(vrt.block_windows()): band1 = vrt.read(window=window) @@ -197,7 +197,8 @@ def rast_converter(in_path, out_path="rast_convert.parquet"): ,'lat': lats.flatten()}) out.drop(index=out.loc[out.band_var==nodata].index,inplace=True) - out.drop(index=out.loc[out.band_var<=0].index,inplace=True) + if not include: + out.drop(index=out.loc[out.band_var<=0].index,inplace=True) if out.shape[0]!=0: writer.write_table(Table.from_pandas(df=out,schema = rast_schema,preserve_index=False,safe=True))