diff --git a/CHANGELOG.md b/CHANGELOG.md index 1af5f84..4fb7187 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,8 +17,6 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). * `compare.py` create new `assert_within_tolerance` function to use file-specific tolerances * `autorift_golden.json.j2` includes L5+5, L7+7, L7+8, and L8+7 pairs in different projections to test reprojection code * `--user-id` pytest CLI argument to allow finding products submitted by a different user than the authorized user - -### Added * `test_autorift.py` golden test for the autoRIFT plugin ### Changed @@ -28,6 +26,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). * InSAR Gamma tests so that they do not use per-image threshold and instead analyze metadata, coregistration, nodata coverage, and dataproduct quality * RTC and autoRIFT golden tests now sleep for 60 seconds between requests for job status * `conda-env.yml` has been renamed to `environment.yml` to follow standard naming conventions +* Refactored golden tests to move shared `golden_submission`, `golden_wait`, `golden_jobs_succeeds`, and `golden_tif_names` to `helpers.py` ## [0.1.0](https://github.com/ASFHyP3/hyp3-testing/compare/v0.0.0...v0.1.0) diff --git a/hyp3_testing/helpers.py b/hyp3_testing/helpers.py index 06487c7..68fc8d9 100644 --- a/hyp3_testing/helpers.py +++ b/hyp3_testing/helpers.py @@ -1,3 +1,4 @@ +import json import os from contextlib import contextmanager from glob import glob @@ -8,6 +9,8 @@ from hyp3_sdk.util import extract_zipped_product from remotezip import RemoteZip +from hyp3_testing import util + def freeze_job_parameters(job: Job) -> tuple: job_parameters = job.job_parameters @@ -107,3 +110,53 @@ def job_tifs(job_id, api, directory, keep=False): for ff in product_dir.rglob('*'): ff.unlink() product_dir.rmdir() + + +def golden_submission(comparison_environments: tuple[tuple[Path, str], tuple[Path, str]], job_template: str): + job_name = util.generate_job_name() + print(f'Job name: {job_name}') + + testing_parameters = util.render_template(job_template, name=job_name) + submission_payload = [{k: item[k] for k in ['name', 'job_parameters', 'job_type']} for item in testing_parameters] + + for dir_, api in comparison_environments: + dir_.mkdir(parents=True, exist_ok=True) + + hyp3 = HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) + jobs = hyp3.submit_prepared_jobs(submission_payload) + request_time = jobs.jobs[0].request_time.isoformat(timespec='seconds') + print(f'{dir_.name} request time: {request_time}') + + submission_details = {'name': job_name, 'request_time': request_time} + submission_report = dir_ / f'{dir_.name}_submission.json' + submission_report.write_text(json.dumps(submission_details)) + + +def golden_wait(comparison_environments: tuple[tuple[Path, str], tuple[Path, str]], job_name: str | None, user_id: str): + for dir_, api in comparison_environments: + if job_name is None: + submission_report = dir_ / f'{dir_.name}_submission.json' + submission_details = json.loads(submission_report.read_text()) + job_name = submission_details['name'] + + hyp3 = HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) + jobs = hyp3.find_jobs(name=job_name, user_id=user_id) + + assert len(jobs) > 0 # will throw if job_name not associated with user_id + + _ = hyp3.watch(jobs) + + +def golden_job_succeeds(jobs_info: dict): + main_succeeds = sum([value['main']['succeeded'] for value in jobs_info.values()]) + develop_succeeds = sum([value['develop']['succeeded'] for value in jobs_info.values()]) + assert main_succeeds != 0, 'Main jobs did not succeed' + assert develop_succeeds != 0, 'Develop jobs did not succeed' + assert main_succeeds == develop_succeeds, 'Main and develop job success counts do not match' + + +def golden_tif_names(jobs_info: dict): + for pair_information in jobs_info.values(): + main_normalized_files = pair_information['main']['normalized_files'] + develop_normalized_files = pair_information['develop']['normalized_files'] + assert main_normalized_files == develop_normalized_files diff --git a/hyp3_testing/isce2_compare.py b/hyp3_testing/isce2_compare.py new file mode 100644 index 0000000..3f82215 --- /dev/null +++ b/hyp3_testing/isce2_compare.py @@ -0,0 +1,67 @@ +import rioxarray # noqa: F401 +import xarray as xr +from osgeo import gdal + +from hyp3_testing import compare +from hyp3_testing.helpers import job_tifs + + +gdal.UseExceptions() + + +def _comparisons(main_ds, develop_ds, pixel_size): + compare.images_are_within_offset_threshold(main_ds, develop_ds, pixel_size=pixel_size, offset_threshold=5.0) + compare.maskes_are_within_similarity_threshold(main_ds, develop_ds, mask_rate=0.98) + compare.values_are_within_statistic(main_ds, develop_ds, confidence_level=0.99) + + +def compare_isce2_derived_products(comparison_environments, jobs_info, keep): + (main_dir, main_api), (develop_dir, develop_api) = comparison_environments + + failure_count = 0 + messages = [] + for pair, pair_information in jobs_info.items(): + with ( + job_tifs(pair_information['main']['job_id'], main_api, main_dir, keep) as main_tifs, + job_tifs(pair_information['develop']['job_id'], develop_api, develop_dir, keep) as develop_tifs, + ): + main_file_dir = main_dir / (main_product_name := pair_information['main']['dir']) + develop_file_dir = develop_dir / (develop_product_name := pair_information['develop']['dir']) + + compare.compare_product_files(main_file_dir, develop_file_dir) + + main_parameter_file = (main_file_dir / main_product_name).with_suffix('.txt') + develop_parameter_file = (develop_file_dir / develop_product_name).with_suffix('.txt') + + compare.compare_parameter_files(str(main_parameter_file), str(develop_parameter_file)) + + for main_tif, develop_tif in zip(main_tifs, develop_tifs): + comparison_header = '\n'.join(['-' * 80, str(main_tif), str(develop_tif), '-' * 80]) + + main_ds = xr.open_dataset(main_tif, engine='rasterio').band_data.data[0] + develop_ds = xr.open_dataset(develop_tif, engine='rasterio').band_data.data[0] + + try: + compare.compare_raster_info(main_tif, develop_tif) + + pixel_size = gdal.Info(str(main_tif), format='json')['geoTransform'][1] + # OpenCV does not support complex data, so we must compare each component as real values. + if main_ds.dtype in ('complex32', 'complex64'): + _comparisons(main_ds.real, develop_ds.real, pixel_size) + _comparisons(main_ds.imag, develop_ds.imag, pixel_size) + else: + _comparisons(main_ds, develop_ds, pixel_size) + + if '_unw_phase.tif' in str(main_tif): + compare.nodata_count_change_are_within_threshold(main_ds, develop_ds, threshold=0.01) + + if '_corr.tif' in str(main_tif): + compare.corr_average_decrease_within_threshold(main_ds, develop_ds, threshold=0.05) + + except compare.ComparisonFailure as e: + messages.append(f'{comparison_header}\n{e}') + failure_count += 1 + + if messages: + messages.insert(0, f'{failure_count} differences found!!') + raise compare.ComparisonFailure('\n\n'.join(messages)) diff --git a/tests/test_autorift.py b/tests/test_autorift.py index aff1330..9b0dbd3 100644 --- a/tests/test_autorift.py +++ b/tests/test_autorift.py @@ -7,7 +7,7 @@ import pytest import xarray as xr -from hyp3_testing import compare, helpers, util +from hyp3_testing import compare, helpers pytestmark = pytest.mark.golden @@ -15,22 +15,7 @@ @pytest.mark.nameskip def test_golden_submission(its_live_environments): - job_name = util.generate_job_name() - print(f'Job name: {job_name}') - - submission_payload = util.render_template('autorift_golden.json.j2', name=job_name) - - for dir_, api in its_live_environments: - dir_.mkdir(parents=True, exist_ok=True) - - hyp3 = hyp3_sdk.HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) - jobs = hyp3.submit_prepared_jobs(submission_payload) - request_time = jobs.jobs[0].request_time.isoformat(timespec='seconds') - print(f'{dir_.name} request time: {request_time}') - - submission_details = {'name': job_name, 'request_time': request_time} - submission_report = dir_ / f'{dir_.name}_submission.json' - submission_report.write_text(json.dumps(submission_details)) + helpers.golden_submission(its_live_environments, 'autorift_golden.json.j2') @pytest.mark.timeout(10800) # 3 hours diff --git a/tests/test_burst_insar.py b/tests/test_burst_insar.py index 5ac0b52..0a79c58 100644 --- a/tests/test_burst_insar.py +++ b/tests/test_burst_insar.py @@ -1,129 +1,33 @@ -import json -import os - -import hyp3_sdk.util import pytest -import rioxarray # noqa: F401 -import xarray as xr -from osgeo import gdal -from hyp3_testing import compare, util -from hyp3_testing.helpers import job_tifs +import hyp3_testing.helpers as helpers +from hyp3_testing.isce2_compare import compare_isce2_derived_products -gdal.UseExceptions() pytestmark = pytest.mark.golden @pytest.mark.nameskip def test_golden_submission(comparison_environments): - job_name = util.generate_job_name() - print(f'Job name: {job_name}') - - testing_parameters = util.render_template('insar_isce_burst_golden.json.j2', name=job_name) - submission_payload = [{k: item[k] for k in ['name', 'job_parameters', 'job_type']} for item in testing_parameters] - - for dir_, api in comparison_environments: - dir_.mkdir(parents=True, exist_ok=True) - - hyp3 = hyp3_sdk.HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) - jobs = hyp3.submit_prepared_jobs(submission_payload) - request_time = jobs.jobs[0].request_time.isoformat(timespec='seconds') - print(f'{dir_.name} request time: {request_time}') - - submission_details = {'name': job_name, 'request_time': request_time} - submission_report = dir_ / f'{dir_.name}_submission.json' - submission_report.write_text(json.dumps(submission_details)) + helpers.golden_submission(comparison_environments, 'insar_isce_burst_golden.json.j2') @pytest.mark.timeout(10800) # 180 minutes as InSAR jobs can take ~2.5 hrs @pytest.mark.dependency() def test_golden_wait(comparison_environments, job_name, user_id): - for dir_, api in comparison_environments: - if job_name is None: - submission_report = dir_ / f'{dir_.name}_submission.json' - submission_details = json.loads(submission_report.read_text()) - job_name = submission_details['name'] - - hyp3 = hyp3_sdk.HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) - jobs = hyp3.find_jobs(name=job_name, user_id=user_id) - - assert len(jobs) > 0 # will throw if job_name not associated with user_id - - _ = hyp3.watch(jobs) + helpers.golden_wait(comparison_environments, job_name, user_id) @pytest.mark.dependency(depends=['test_golden_wait']) def test_golden_job_succeeds(jobs_info): - main_succeeds = sum([value['main']['succeeded'] for value in jobs_info.values()]) - develop_succeeds = sum([value['develop']['succeeded'] for value in jobs_info.values()]) - assert main_succeeds != 0 - assert develop_succeeds != 0 - assert main_succeeds == develop_succeeds + helpers.golden_job_succeeds(jobs_info) @pytest.mark.dependency(depends=['test_golden_wait']) def test_golden_tif_names(jobs_info): - for pair_information in jobs_info.values(): - main_normalized_files = pair_information['main']['normalized_files'] - develop_normalized_files = pair_information['develop']['normalized_files'] - assert main_normalized_files == develop_normalized_files - - -def _comparisons(main_ds, develop_ds, pixel_size): - compare.images_are_within_offset_threshold(main_ds, develop_ds, pixel_size=pixel_size, offset_threshold=5.0) - compare.maskes_are_within_similarity_threshold(main_ds, develop_ds, mask_rate=0.98) - compare.values_are_within_statistic(main_ds, develop_ds, confidence_level=0.99) + helpers.golden_tif_names(jobs_info) @pytest.mark.dependency(depends=['test_golden_wait']) -def test_golden_burst_insar(comparison_environments, jobs_info, keep): - (main_dir, main_api), (develop_dir, develop_api) = comparison_environments - - failure_count = 0 - messages = [] - for pair, pair_information in jobs_info.items(): - with ( - job_tifs(pair_information['main']['job_id'], main_api, main_dir, keep) as main_tifs, - job_tifs(pair_information['develop']['job_id'], develop_api, develop_dir, keep) as develop_tifs, - ): - main_file_dir = main_dir / (main_product_name := pair_information['main']['dir']) - develop_file_dir = develop_dir / (develop_product_name := pair_information['develop']['dir']) - - compare.compare_product_files(main_file_dir, develop_file_dir) - - main_parameter_file = (main_file_dir / main_product_name).with_suffix('.txt') - develop_parameter_file = (develop_file_dir / develop_product_name).with_suffix('.txt') - - compare.compare_parameter_files(str(main_parameter_file), str(develop_parameter_file)) - - for main_tif, develop_tif in zip(main_tifs, develop_tifs): - comparison_header = '\n'.join(['-' * 80, str(main_tif), str(develop_tif), '-' * 80]) - - main_ds = xr.open_dataset(main_tif, engine='rasterio').band_data.data[0] - develop_ds = xr.open_dataset(develop_tif, engine='rasterio').band_data.data[0] - - try: - compare.compare_raster_info(main_tif, develop_tif) - - pixel_size = gdal.Info(str(main_tif), format='json')['geoTransform'][1] - # OpenCV does not support complex data, so we must compare each component as real values. - if main_ds.dtype in ('complex32', 'complex64'): - _comparisons(main_ds.real, develop_ds.real, pixel_size) - _comparisons(main_ds.imag, develop_ds.imag, pixel_size) - else: - _comparisons(main_ds, develop_ds, pixel_size) - - if '_unw_phase.tif' in str(main_tif): - compare.nodata_count_change_are_within_threshold(main_ds, develop_ds, threshold=0.01) - - if '_corr.tif' in str(main_tif): - compare.corr_average_decrease_within_threshold(main_ds, develop_ds, threshold=0.05) - - except compare.ComparisonFailure as e: - messages.append(f'{comparison_header}\n{e}') - failure_count += 1 - - if messages: - messages.insert(0, f'{failure_count} differences found!!') - raise compare.ComparisonFailure('\n\n'.join(messages)) +def test_golden_multi_burst_insar(comparison_environments, jobs_info, keep): + compare_isce2_derived_products(comparison_environments, jobs_info, keep) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index e86525a..e3ebca6 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,3 +1,5 @@ +import pytest + from hyp3_testing import helpers @@ -36,3 +38,38 @@ def test_find_files_in_products(tmp_path): (main_dir / 'b.tif', develop_dir / 'b.tif'), (main_dir / 'c.tif', develop_dir / 'c.tif'), ] + + +def test_golden_job_succeeds(): + jobs_info = {'pair1': {'main': {'succeeded': 1}, 'develop': {'succeeded': 1}}} + helpers.golden_job_succeeds(jobs_info) + + jobs_info = {'pair1': {'main': {'succeeded': 0}, 'develop': {'succeeded': 1}}} + with pytest.raises(AssertionError, match='Main jobs did not succeed'): + helpers.golden_job_succeeds(jobs_info) + + jobs_info = {'pair1': {'main': {'succeeded': 1}, 'develop': {'succeeded': 0}}} + with pytest.raises(AssertionError, match='Develop jobs did not succeed'): + helpers.golden_job_succeeds(jobs_info) + + jobs_info = {'pair1': {'main': {'succeeded': 1}, 'develop': {'succeeded': 2}}} + with pytest.raises(AssertionError, match='Main and develop job success counts do not match'): + helpers.golden_job_succeeds(jobs_info) + + +def test_golden_tif_names(): + jobs_info = { + 'pair1': { + 'main': {'normalized_files': ['file1', 'file2']}, + 'develop': {'normalized_files': ['file1', 'file2']}, + }, + 'pair2': { + 'main': {'normalized_files': ['file3', 'file4']}, + 'develop': {'normalized_files': ['file3', 'file4']}, + }, + } + helpers.golden_tif_names(jobs_info) + + jobs_info['pair2']['develop']['normalized_files'] = ['file3', 'file5'] + with pytest.raises(AssertionError): + helpers.golden_tif_names(jobs_info) diff --git a/tests/test_insar.py b/tests/test_insar.py index 7f9ceb4..1335496 100644 --- a/tests/test_insar.py +++ b/tests/test_insar.py @@ -1,13 +1,9 @@ -import json -import os - -import hyp3_sdk.util import pytest import rioxarray # noqa: F401 import xarray as xr from osgeo import gdal -from hyp3_testing import compare, util +from hyp3_testing import compare, helpers from hyp3_testing.helpers import job_tifs @@ -17,57 +13,23 @@ @pytest.mark.nameskip def test_golden_submission(comparison_environments): - job_name = util.generate_job_name() - print(f'Job name: {job_name}') - - testing_parameters = util.render_template('insar_gamma_golden.json.j2', name=job_name) - submission_payload = [{k: item[k] for k in ['name', 'job_parameters', 'job_type']} for item in testing_parameters] - - for dir_, api in comparison_environments: - dir_.mkdir(parents=True, exist_ok=True) - - hyp3 = hyp3_sdk.HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) - jobs = hyp3.submit_prepared_jobs(submission_payload) - request_time = jobs.jobs[0].request_time.isoformat(timespec='seconds') - print(f'{dir_.name} request time: {request_time}') - - submission_details = {'name': job_name, 'request_time': request_time} - submission_report = dir_ / f'{dir_.name}_submission.json' - submission_report.write_text(json.dumps(submission_details)) + helpers.golden_submission(comparison_environments, 'insar_gamma_golden.json.j2') @pytest.mark.timeout(10800) # 180 minutes as InSAR jobs can take ~2.5 hrs @pytest.mark.dependency() def test_golden_wait(comparison_environments, job_name, user_id): - for dir_, api in comparison_environments: - if job_name is None: - submission_report = dir_ / f'{dir_.name}_submission.json' - submission_details = json.loads(submission_report.read_text()) - job_name = submission_details['name'] - - hyp3 = hyp3_sdk.HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) - jobs = hyp3.find_jobs(name=job_name, user_id=user_id) - - assert len(jobs) > 0 # will throw if job_name not associated with user_id - - _ = hyp3.watch(jobs) + helpers.golden_wait(comparison_environments, job_name, user_id) @pytest.mark.dependency(depends=['test_golden_wait']) def test_golden_job_succeeds(jobs_info): - main_succeeds = sum([value['main']['succeeded'] for value in jobs_info.values()]) - develop_succeeds = sum([value['develop']['succeeded'] for value in jobs_info.values()]) - assert main_succeeds != 0 - assert develop_succeeds != 0 - assert main_succeeds == develop_succeeds + helpers.golden_job_succeeds(jobs_info) @pytest.mark.dependency(depends=['test_golden_wait']) def test_golden_tif_names(jobs_info): - for pair_information in jobs_info.values(): - main_normalized_files = pair_information['main']['normalized_files'] - develop_normalized_files = pair_information['develop']['normalized_files'] - assert main_normalized_files == develop_normalized_files + helpers.golden_tif_names(jobs_info) @pytest.mark.dependency(depends=['test_golden_wait']) diff --git a/tests/test_multi_burst_insar.py b/tests/test_multi_burst_insar.py index 3b9f162..7f0eded 100644 --- a/tests/test_multi_burst_insar.py +++ b/tests/test_multi_burst_insar.py @@ -1,129 +1,33 @@ -import json -import os - -import hyp3_sdk.util import pytest -import rioxarray # noqa: F401 -import xarray as xr -from osgeo import gdal -from hyp3_testing import compare, util -from hyp3_testing.helpers import job_tifs +import hyp3_testing.helpers as helpers +from hyp3_testing.isce2_compare import compare_isce2_derived_products -gdal.UseExceptions() pytestmark = pytest.mark.golden @pytest.mark.nameskip def test_golden_submission(comparison_environments): - job_name = util.generate_job_name() - print(f'Job name: {job_name}') - - testing_parameters = util.render_template('insar_isce_multi_burst_golden.json.j2', name=job_name) - submission_payload = [{k: item[k] for k in ['name', 'job_parameters', 'job_type']} for item in testing_parameters] - - for dir_, api in comparison_environments: - dir_.mkdir(parents=True, exist_ok=True) - - hyp3 = hyp3_sdk.HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) - jobs = hyp3.submit_prepared_jobs(submission_payload) - request_time = jobs.jobs[0].request_time.isoformat(timespec='seconds') - print(f'{dir_.name} request time: {request_time}') - - submission_details = {'name': job_name, 'request_time': request_time} - submission_report = dir_ / f'{dir_.name}_submission.json' - submission_report.write_text(json.dumps(submission_details)) + helpers.golden_submission(comparison_environments, 'insar_isce_multi_burst_golden.json.j2') @pytest.mark.timeout(10800) # 180 minutes as InSAR jobs can take ~2.5 hrs @pytest.mark.dependency() def test_golden_wait(comparison_environments, job_name, user_id): - for dir_, api in comparison_environments: - if job_name is None: - submission_report = dir_ / f'{dir_.name}_submission.json' - submission_details = json.loads(submission_report.read_text()) - job_name = submission_details['name'] - - hyp3 = hyp3_sdk.HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) - jobs = hyp3.find_jobs(name=job_name, user_id=user_id) - - assert len(jobs) > 0 # will throw if job_name not associated with user_id - - _ = hyp3.watch(jobs) + helpers.golden_wait(comparison_environments, job_name, user_id) @pytest.mark.dependency(depends=['test_golden_wait']) def test_golden_job_succeeds(jobs_info): - main_succeeds = sum([value['main']['succeeded'] for value in jobs_info.values()]) - develop_succeeds = sum([value['develop']['succeeded'] for value in jobs_info.values()]) - assert main_succeeds != 0 - assert develop_succeeds != 0 - assert main_succeeds == develop_succeeds + helpers.golden_job_succeeds(jobs_info) @pytest.mark.dependency(depends=['test_golden_wait']) def test_golden_tif_names(jobs_info): - for pair_information in jobs_info.values(): - main_normalized_files = pair_information['main']['normalized_files'] - develop_normalized_files = pair_information['develop']['normalized_files'] - assert main_normalized_files == develop_normalized_files - - -def _comparisons(main_ds, develop_ds, pixel_size): - compare.images_are_within_offset_threshold(main_ds, develop_ds, pixel_size=pixel_size, offset_threshold=5.0) - compare.maskes_are_within_similarity_threshold(main_ds, develop_ds, mask_rate=0.98) - compare.values_are_within_statistic(main_ds, develop_ds, confidence_level=0.99) + helpers.golden_tif_names(jobs_info) @pytest.mark.dependency(depends=['test_golden_wait']) def test_golden_multi_burst_insar(comparison_environments, jobs_info, keep): - (main_dir, main_api), (develop_dir, develop_api) = comparison_environments - - failure_count = 0 - messages = [] - for pair, pair_information in jobs_info.items(): - with ( - job_tifs(pair_information['main']['job_id'], main_api, main_dir, keep) as main_tifs, - job_tifs(pair_information['develop']['job_id'], develop_api, develop_dir, keep) as develop_tifs, - ): - main_file_dir = main_dir / (main_product_name := pair_information['main']['dir']) - develop_file_dir = develop_dir / (develop_product_name := pair_information['develop']['dir']) - - compare.compare_product_files(main_file_dir, develop_file_dir) - - main_parameter_file = (main_file_dir / main_product_name).with_suffix('.txt') - develop_parameter_file = (develop_file_dir / develop_product_name).with_suffix('.txt') - - compare.compare_parameter_files(str(main_parameter_file), str(develop_parameter_file)) - - for main_tif, develop_tif in zip(main_tifs, develop_tifs): - comparison_header = '\n'.join(['-' * 80, str(main_tif), str(develop_tif), '-' * 80]) - - main_ds = xr.open_dataset(main_tif, engine='rasterio').band_data.data[0] - develop_ds = xr.open_dataset(develop_tif, engine='rasterio').band_data.data[0] - - try: - compare.compare_raster_info(main_tif, develop_tif) - - pixel_size = gdal.Info(str(main_tif), format='json')['geoTransform'][1] - # OpenCV does not support complex data, so we must compare each component as real values. - if main_ds.dtype in ('complex32', 'complex64'): - _comparisons(main_ds.real, develop_ds.real, pixel_size) - _comparisons(main_ds.imag, develop_ds.imag, pixel_size) - else: - _comparisons(main_ds, develop_ds, pixel_size) - - if '_unw_phase.tif' in str(main_tif): - compare.nodata_count_change_are_within_threshold(main_ds, develop_ds, threshold=0.01) - - if '_corr.tif' in str(main_tif): - compare.corr_average_decrease_within_threshold(main_ds, develop_ds, threshold=0.05) - - except compare.ComparisonFailure as e: - messages.append(f'{comparison_header}\n{e}') - failure_count += 1 - - if messages: - messages.insert(0, f'{failure_count} differences found!!') - raise compare.ComparisonFailure('\n\n'.join(messages)) + compare_isce2_derived_products(comparison_environments, jobs_info, keep) diff --git a/tests/test_rtc.py b/tests/test_rtc.py index c95c7c7..af494db 100644 --- a/tests/test_rtc.py +++ b/tests/test_rtc.py @@ -1,13 +1,10 @@ -import json -import os from pathlib import Path -import hyp3_sdk.util import pytest import rioxarray # noqa: F401 import xarray as xr -from hyp3_testing import compare, util +from hyp3_testing import compare, helpers from hyp3_testing.helpers import job_tifs @@ -16,57 +13,23 @@ @pytest.mark.nameskip def test_golden_submission(comparison_environments): - job_name = util.generate_job_name() - print(f'Job name: {job_name}') - - testing_parameters = util.render_template('rtc_gamma_golden.json.j2', name=job_name) - submission_payload = [{k: item[k] for k in ['name', 'job_parameters', 'job_type']} for item in testing_parameters] - - for dir_, api in comparison_environments: - dir_.mkdir(parents=True, exist_ok=True) - - hyp3 = hyp3_sdk.HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) - jobs = hyp3.submit_prepared_jobs(submission_payload) - request_time = jobs.jobs[0].request_time.isoformat(timespec='seconds') - print(f'{dir_.name} request time: {request_time}') - - submission_details = {'name': job_name, 'request_time': request_time} - submission_report = dir_ / f'{dir_.name}_submission.json' - submission_report.write_text(json.dumps(submission_details)) + helpers.golden_submission(comparison_environments, 'rtc_gamma_golden.json.j2') @pytest.mark.timeout(5400) # 90 minutes as RTC jobs can take ~1.5 hr @pytest.mark.dependency() def test_golden_wait(comparison_environments, job_name, user_id): - for dir_, api in comparison_environments: - if job_name is None: - submission_report = dir_ / f'{dir_.name}_submission.json' - submission_details = json.loads(submission_report.read_text()) - job_name = submission_details['name'] - - hyp3 = hyp3_sdk.HyP3(api, os.environ.get('EARTHDATA_LOGIN_USER'), os.environ.get('EARTHDATA_LOGIN_PASSWORD')) - jobs = hyp3.find_jobs(name=job_name, user_id=user_id) - - assert len(jobs) > 0 # will throw if job_name not associated with user_id - - _ = hyp3.watch(jobs) + helpers.golden_wait(comparison_environments, job_name, user_id) @pytest.mark.dependency(depends=['test_golden_wait']) def test_golden_job_succeeds(jobs_info): - main_succeeds = sum([value['main']['succeeded'] for value in jobs_info.values()]) - develop_succeeds = sum([value['develop']['succeeded'] for value in jobs_info.values()]) - assert main_succeeds != 0 - assert develop_succeeds != 0 - assert main_succeeds == develop_succeeds + helpers.golden_job_succeeds(jobs_info) @pytest.mark.dependency(depends=['test_golden_wait']) def test_golden_tif_names(jobs_info): - for pair_information in jobs_info.values(): - main_normalized_files = pair_information['main']['normalized_files'] - develop_normalized_files = pair_information['develop']['normalized_files'] - assert main_normalized_files == develop_normalized_files + helpers.golden_tif_names(jobs_info) @pytest.mark.dependency(depends=['test_golden_wait'])