Synthetic test

.gitignore

@@ -9,3 +9,4 @@
 */*/*.log
 */*/geom
 */*/weather_files
+*/*/*/tmp.yaml

notebooks/RAiDER_ARIA/ariaTesting.ipynb

@@ -0,0 +1,387 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "eb49726a",
+   "metadata": {
+    "init_cell": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Tutorial directory:  /Users/buzzanga/Software_InSAR/RAiDER-docs_git/notebooks/RAiDER_ARIA\n",
+      "Work directory:  /Users/buzzanga/Software_InSAR/RAiDER-docs_git/notebooks/RAiDER_ARIA\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os, os.path as op\n",
+    "import shutil\n",
+    "import yaml\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
+    "\n",
+    "import pandas as pd\n",
+    "import xarray as xr\n",
+    "from wand.image import Image as WImage\n",
+    "\n",
+    "## Defining the home and data directories\n",
+    "tutorial_home_dir = os.path.abspath(os.getcwd())\n",
+    "work_dir = os.path.abspath(os.getcwd())\n",
+    "print(\"Tutorial directory: \", tutorial_home_dir)\n",
+    "print(\"Work directory: \", work_dir)\n",
+    "\n",
+    "# Verifying if RAiDER is installed correctly\n",
+    "try:\n",
+    "    import RAiDER\n",
+    "except:\n",
+    "    raise Exception('RAiDER is missing from your PYTHONPATH')\n",
+    "\n",
+    "os.chdir(work_dir)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6f52403a",
+   "metadata": {},
+   "source": [
+    "### helpers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "78fe0d71",
+   "metadata": {
+    "init_cell": true
+   },
+   "outputs": [],
+   "source": [
+    "def new_yaml_group(example_yaml, dct_group, dst='tmp.yaml'):\n",
+    "    \"\"\" Write a temporary yaml file with the new 'value' for 'key', preserving parms in example_yaml\"\"\"\n",
+    "    with open(example_yaml, 'r') as f:\n",
+    "        try:\n",
+    "            params = yaml.safe_load(f)\n",
+    "        except yaml.YAMLError as exc:\n",
+    "            print(exc)\n",
+    "            raise ValueError(f'Something is wrong with the yaml file {example_yaml}')\n",
+    "    \n",
+    "    params = {**params, **dct_group}\n",
+    "    dst = op.join(os.path.dirname(example_yaml), dst)\n",
+    "#     print (params)\n",
+    "    \n",
+    "    with open(dst, 'w') as fh:\n",
+    "        yaml.dump(params, fh, default_flow_style=False)\n",
+    "    \n",
+    "    print ('Wrote new cfg file:', dst)\n",
+    "    return dst"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4a1bd772",
+   "metadata": {},
+   "source": [
+    "#### Pieces of prepARIA"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "233bae03",
+   "metadata": {},
+   "source": [
+    "These are just for temporarily making the config file"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "4e488967",
+   "metadata": {
+    "init_cell": true
+   },
+   "outputs": [],
+   "source": [
+    "def makeLatLonGrid1(f:str, reg, out_dir):\n",
+    "    ds0  = xr.open_dataset(f, group='science/grids/data')\n",
+    "\n",
+    "    Lat, Lon  = np.meshgrid(ds0.latitude.data, ds0.longitude.data)\n",
+    "    print (Lat.shape, Lon.shape)\n",
+    "\n",
+    "    da_lat = xr.DataArray(Lat.T, coords=[Lon[0, :], Lat[:, 0]], dims='lon lat'.split())\n",
+    "    da_lon = xr.DataArray(Lon.T, coords=[Lon[0, :], Lat[:, 0]], dims='lon lat'.split())\n",
+    "    dst_lat = op.join(out_dir, 'lat_HR.geo')\n",
+    "    dst_lon = op.join(out_dir, 'lon_HR.geo')\n",
+    "    da_lat.to_netcdf(dst_lat)\n",
+    "    da_lon.to_netcdf(dst_lon)\n",
+    "    logger.debug('Wrote: %s', dst_lat)\n",
+    "    logger.debug('Wrote: %s', dst_lon)\n",
+    "    return"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "bd10c986",
+   "metadata": {
+    "init_cell": true
+   },
+   "outputs": [],
+   "source": [
+    "def parse_time_GUNW(f:str):\n",
+    "    \"\"\" Get the center time of the secondary date from the filename \"\"\"\n",
+    "    tt = op.basename(f).split('-')[7]\n",
+    "    return f'{tt[:2]}:{tt[2:4]}:{tt[4:]}'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "7a565639",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def makeLatLonGrid_dep(f:str, reg, out_dir):\n",
+    "    '''\n",
+    "    Convert the geocoded grids to lat/lon files for input to RAiDER\n",
+    "    '''\n",
+    "    from RAiDER.utilFcns import writeArrayToRaster\n",
+    "    group   = 'science/grids/data'\n",
+    "    lat_f   = os.path.join(f'NETCDF:\"{f}\":{group}/latitude')\n",
+    "    lon_f   = os.path.join(f'NETCDF:\"{f}\":{group}/longitude')\n",
+    "\n",
+    "    ds0   = xr.open_dataset(f, group='science/grids/data')\n",
+    "\n",
+    "    \n",
+    "    gt   = (0, 1, 0, 0, 0, 1)\n",
+    "    proj = ds['crs'].crs_wkt\n",
+    "\n",
+    "    lats = ds.latitude.data\n",
+    "    lons = ds.longitude.data\n",
+    "\n",
+    "    ySize = len(lats)\n",
+    "    xSize = len(lons)\n",
+    "\n",
+    "    # ISCE lats are ordered from smallest to biggest\n",
+    "    LATS  = np.flipud(np.tile(lats, (xSize, 1)).T)\n",
+    "    LONS  = np.tile(lons, (ySize, 1))\n",
+    "\n",
+    "    dst_lat = op.join(out_dir, f'lat_{reg}.geo')\n",
+    "    dst_lon = op.join(out_dir, f'lon_{reg}.geo')\n",
+    "\n",
+    "    writeArrayToRaster(LATS, dst_lat, 0., 'GTiff', proj, gt)\n",
+    "    writeArrayToRaster(LONS, dst_lon, 0., 'GTiff', proj, gt)\n",
+    "\n",
+    "    logger.debug('Wrote: %s', dst_lat)\n",
+    "    logger.debug('Wrote: %s', dst_lon)\n",
+    "    return LATS, LONS"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "78874fc4",
+   "metadata": {},
+   "source": [
+    "## Hampton Roads Test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "9148bfce",
+   "metadata": {
+    "init_cell": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(3934, 2253) (3934, 2253)\n"
+     ]
+    }
+   ],
+   "source": [
+    "yaml_base = op.join(work_dir, 'example_yamls', 'raider.yaml')\n",
+    "gunw = op.join(work_dir, 'data', 'S1-GUNW-A-R-004-tops-20181219_20181113-230629-37654N_35778N-PP-ede0-v2_0_2.nc')\n",
+    "orb1 = op.join(work_dir, 'orbits', 'S1A_OPER_AUX_POEORB_OPOD_20181203T120749_V20181112T225942_20181114T005942.EOF')\n",
+    "orb2 = op.join(work_dir, 'orbits', 'S1A_OPER_AUX_POEORB_OPOD_20190108T120818_V20181218T225942_20181220T005942.EOF')\n",
+    "orb3 = op.join(work_dir, 'orbits', 'S1A_OPER_AUX_POEORB_OPOD_20210309T234202_V20181112T225942_20181114T005942.EOF')\n",
+    "makeLatLonGrid1(gunw, 'HR', op.join(work_dir, 'data'));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0f314c11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# first just use config file option"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "3264dfcc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Wrote new cfg file: /Users/buzzanga/Software_InSAR/RAiDER-docs_git/notebooks/RAiDER_ARIA/example_yamls/tmp.yaml\n",
+      "/Users/buzzanga/Miniconda3/envs/RAiDER/lib/python3.10/site-packages/rasterio/__init__.py:304: NotGeoreferencedWarning: Dataset has no geotransform, gcps, or rpcs. The identity matrix will be returned.\n",
+      "  dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)\n",
+      "Starting to run the weather model calculation\n",
+      "Time: 20181113\n",
+      "Beginning weather model pre-processing\n",
+      "\u001b[33;21mWARNING: Weather model already exists, please remove it (\"['/Users/buzzanga/Software_InSAR/RAiDER-docs_git/notebooks/RAiDER_ARIA/weather_files/ERA-5_2018_11_13_T23_06_29.nc']\") if you want to download a new one.\u001b[0m\n",
+      "Extent of the weather model is (xmin, ymin, xmax, ymax):-78.93, 35.38, -74.93, 37.88\n",
+      "Extent of the input is (xmin, ymin, xmax, ymax): -78.53, 35.78, -75.26, 37.65\n",
+      "\u001b[33;21mWARNING: The processed weather model file already exists, so I will use that.\u001b[0m\n",
+      "Output SNWE: [35.78, 37.660000000000004, -78.54, -75.24]\n",
+      "Output cube spacing: 0.02\n",
+      "/Users/buzzanga/Miniconda3/envs/RAiDER/lib/python3.10/site-packages/rasterio/__init__.py:304: NotGeoreferencedWarning: Dataset has no geotransform, gcps, or rpcs. The identity matrix will be returned.\n",
+      "  dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)\n",
+      "Using existing DEM: /Users/buzzanga/Software_InSAR/RAiDER-docs_git/notebooks/RAiDER_ARIA/GLO30_fullres_dem.tif\n",
+      "\u001b[31;21mERROR: Date 2018-11-13 23:06:29 failed\u001b[0m\n",
+      "Traceback (most recent call last):\n",
+      "  File \"rasterio/_base.pyx\", line 302, in rasterio._base.DatasetBase.__init__\n",
+      "  File \"rasterio/_base.pyx\", line 213, in rasterio._base.open_dataset\n",
+      "  File \"rasterio/_err.pyx\", line 217, in rasterio._err.exc_wrap_pointer\n",
+      "rasterio._err.CPLE_OpenFailedError: '/Users/buzzanga/Software_InSAR/RAiDER-docs_git/notebooks/RAiDER_ARIA/orbits/S1A_OPER_AUX_POEORB_OPOD_20210309T234202_V20181112T225942_20181114T005942.EOF' not recognized as a supported file format.\n",
+      "\n",
+      "During handling of the above exception, another exception occurred:\n",
+      "\n",
+      "Traceback (most recent call last):\n",
+      "  File \"/Users/buzzanga/Software_InSAR/RAiDER_git/tools/RAiDER/losreader.py\", line 111, in __call__\n",
+      "    LOS_enu = inc_hd_to_enu(*rio_open(self._file))\n",
+      "  File \"/Users/buzzanga/Software_InSAR/RAiDER_git/tools/RAiDER/utilFcns.py\", line 146, in rio_open\n",
+      "    with rasterio.open(fname) as src:\n",
+      "  File \"/Users/buzzanga/Miniconda3/envs/RAiDER/lib/python3.10/site-packages/rasterio/env.py\", line 444, in wrapper\n",
+      "    return f(*args, **kwds)\n",
+      "  File \"/Users/buzzanga/Miniconda3/envs/RAiDER/lib/python3.10/site-packages/rasterio/__init__.py\", line 304, in open\n",
+      "    dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)\n",
+      "  File \"rasterio/_base.pyx\", line 304, in rasterio._base.DatasetBase.__init__\n",
+      "rasterio.errors.RasterioIOError: '/Users/buzzanga/Software_InSAR/RAiDER-docs_git/notebooks/RAiDER_ARIA/orbits/S1A_OPER_AUX_POEORB_OPOD_20210309T234202_V20181112T225942_20181114T005942.EOF' not recognized as a supported file format.\n",
+      "\n",
+      "During handling of the above exception, another exception occurred:\n",
+      "\n",
+      "Traceback (most recent call last):\n",
+      "  File \"/Users/buzzanga/Software_InSAR/RAiDER_git/tools/RAiDER/cli/raider.py\", line 285, in main\n",
+      "    wet_delay, hydro_delay = tropo_delay(\n",
+      "  File \"/Users/buzzanga/Software_InSAR/RAiDER_git/tools/RAiDER/delay.py\", line 84, in tropo_delay\n",
+      "    wetDelay = los(wetDelay)\n",
+      "  File \"/Users/buzzanga/Software_InSAR/RAiDER_git/tools/RAiDER/losreader.py\", line 117, in __call__\n",
+      "    LOS_enu = state_to_los(svs,\n",
+      "  File \"/Users/buzzanga/Software_InSAR/RAiDER_git/tools/RAiDER/losreader.py\", line 524, in state_to_los\n",
+      "    los_ang, slant_range = get_radar_pos(target_llh, orb, out=\"lookangle\")\n",
+      "  File \"/Users/buzzanga/Software_InSAR/RAiDER_git/tools/RAiDER/losreader.py\", line 629, in get_radar_pos\n",
+      "    raise e\n",
+      "  File \"/Users/buzzanga/Software_InSAR/RAiDER_git/tools/RAiDER/losreader.py\", line 610, in get_radar_pos\n",
+      "    aztime, slant_range = isce.geometry.geo2rdr(\n",
+      "RuntimeError: geo2rdr failed to converge\n"
+     ]
+    }
+   ],
+   "source": [
+    "# ISCE lat/lon files (Hampton Roads)\n",
+    "# this downloads the DEM and intersects it\n",
+    "\n",
+    "grp = {'aoi_group': {'lat_file': f'{work_dir}/data/lat_HR.geo', 'lon_file': f'{work_dir}/data/lon_HR.geo'}, \n",
+    "       'height_group': {'dem': f'{work_dir}/GLO30_fullres_dem.tif'},\n",
+    "       'weather_model': 'ERA5',\n",
+    "       'date_group': {'date_start': '20181113'},\n",
+    "       'time_group': {'time': parse_time_GUNW(gunw)},\n",
+    "       'los_group' : {'orbit_file': orb3}\n",
+    "      }\n",
+    "\n",
+    "cfg = new_yaml_group(yaml_base, grp)\n",
+    "!raider.py {cfg}\n",
+    "\n",
+    "## store it for use later\n",
+    "# shutil.copy(f'{work_dir}/GLO30_fullres_dem.tif', f'{work_dir}/data/GLO30_fullres_dem_HR.tif')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "868da2c7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Driver: GTiff/GeoTIFF\n",
+      "Files: ERA5_wet_20181219T230629_ztd.GTiff\n",
+      "Size is 3934, 2253\n",
+      "Image Structure Metadata:\n",
+      "  INTERLEAVE=BAND\n",
+      "Corner Coordinates:\n",
+      "Upper Left  (    0.0,    0.0)\n",
+      "Lower Left  (    0.0, 2253.0)\n",
+      "Upper Right ( 3934.0,    0.0)\n",
+      "Lower Right ( 3934.0, 2253.0)\n",
+      "Center      ( 1967.0, 1126.5)\n",
+      "Band 1 Block=3934x1 Type=Float32, ColorInterp=Gray\n",
+      "  Minimum=0.055, Maximum=0.134, Mean=0.082, StdDev=0.022\n",
+      "  NoData Value=0\n",
+      "  Metadata:\n",
+      "    STATISTICS_MAXIMUM=0.13444668054581\n",
+      "    STATISTICS_MEAN=0.082404273144709\n",
+      "    STATISTICS_MINIMUM=0.055032920092344\n",
+      "    STATISTICS_STDDEV=0.021799223611822\n",
+      "    STATISTICS_VALID_PERCENT=100\n"
+     ]
+    }
+   ],
+   "source": [
+    "!gdalinfo ERA5_wet_20181219T230629_ztd.GTiff -stats"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d2eb9163",
+   "metadata": {},
+   "source": [
+    "## LA Test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f5722180",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "celltoolbar": "Initialization Cell",
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}