From c37c7cc8505e9df50eee3410b4a256f922bb4339 Mon Sep 17 00:00:00 2001 From: Dougal Dobie Date: Tue, 20 May 2025 14:31:07 +1000 Subject: [PATCH 1/5] Fixed webapp plots --- vast_pipeline/plots.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vast_pipeline/plots.py b/vast_pipeline/plots.py index b12845f8f..e16f5841a 100644 --- a/vast_pipeline/plots.py +++ b/vast_pipeline/plots.py @@ -84,6 +84,7 @@ def plot_lightcurve( # lightcurve required cols: taustart_ts, flux, flux_err_upper, flux_err_lower, forced lightcurve = pd.DataFrame(measurements_qs) + lightcurve['id'] = lightcurve['id'].astype(str) # remap method values to labels to make a better legend lightcurve["method"] = lightcurve.forced.map({True: "Forced", False: "Selavy"}) @@ -167,6 +168,13 @@ def plot_lightcurve( ) .reset_index() ) + + candidate_measurement_pairs_df = candidate_measurement_pairs_df.astype( + {'measurement_a_id': 'str', + 'measurement_b_id': 'str', + } + ) + g = nx.Graph() for _row in candidate_measurement_pairs_df.itertuples(index=False): g.add_edge(_row.measurement_a_id, _row.measurement_b_id) From 2da72d5cf1ae609f20df20432a9b6849451f3f42 Mon Sep 17 00:00:00 2001 From: Dougal Dobie Date: Tue, 20 May 2025 14:32:44 +1000 Subject: [PATCH 2/5] Updated changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6abcf4a4b..8e1aa87cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Added - - V2: Migrate pipeline to used a Dask.distributed.LocalCluster throughout [#816](https://github.com/askap-vast/vast-pipeline/pull/816) - V2: Add Dask.distributed support - V2: Use `django-postgres-copy` for database uploads [#803](https://github.com/askap-vast/vast-pipeline/pull/803) @@ -35,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Fixed +- V2: Fixed problem with webapp plots related to UUIDs [#844](https://github.com/askap-vast/vast-pipeline/pull/844) - V2: Pair metrics working with V2 dask LocalCluster changes [#817](https://github.com/askap-vast/vast-pipeline/pull/817) - V2: Fixed missing JS9 overlays on source webpage [#809](https://github.com/askap-vast/vast-pipeline/pull/809) - V2: Fix bug when deleting source tags from database which are referenced by multiple sources [#803](https://github.com/askap-vast/vast-pipeline/pull/803) @@ -58,6 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### List of PRs +- [#844](https://github.com/askap-vast/vast-pipeline/pull/844): fix: V2: Fixed problem with webapp plots related to UUIDs - [#833](https://github.com/askap-vast/vast-pipeline/pull/833): feat: V2: Limit associations upload to using num_io_workers - [#829](https://github.com/askap-vast/vast-pipeline/pull/829): feat: V2: Allow user specification of dask dashboard paramters and add some further logging to dask setup - [#817](https://github.com/askap-vast/vast-pipeline/pull/817): fix: V2: Updates to pairs calculation to make it work with Dask `LocalCluster`. From c73b24feaf05c62eb53200bf93c5a60858303367 Mon Sep 17 00:00:00 2001 From: Dougal Dobie Date: Tue, 20 May 2025 14:43:54 +1000 Subject: [PATCH 3/5] measurement UUID tweaks --- vast_pipeline/image/main.py | 7 +++--- ...ter_association_id_alter_measurement_id.py | 24 +++++++++++++++++++ vast_pipeline/models.py | 5 ++-- vast_pipeline/pipeline/forced_extraction.py | 10 ++++---- vast_pipeline/pipeline/loading.py | 3 ++- vast_pipeline/serializers.py | 2 +- vast_pipeline/urls.py | 5 ++++ 7 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 vast_pipeline/migrations/0003_alter_association_id_alter_measurement_id.py diff --git a/vast_pipeline/image/main.py b/vast_pipeline/image/main.py index 0a014636c..c6219f6b5 100644 --- a/vast_pipeline/image/main.py +++ b/vast_pipeline/image/main.py @@ -6,6 +6,7 @@ import logging import numpy as np import pandas as pd +import uuid from django.conf import settings from astropy.io import fits @@ -18,9 +19,7 @@ from vast_pipeline import models from vast_pipeline.survey.translators import tr_selavy -from vast_pipeline.utils.utils import ( - generate_shortuuid, UUID_LEN_MEAS, -) + from vast_pipeline.image.utils import get_fits_header @@ -341,7 +340,7 @@ def read_selavy(self, dj_image: models.Image) -> pd.DataFrame: df[key["name"]] = df[key["name"]].astype(key["dtype"]) # Add id column - df["id"] = df.apply(lambda _: generate_shortuuid(UUID_LEN_MEAS), axis=1) + df["id"] = df.apply(lambda _: str(uuid.uuid4()), axis=1) # do checks and fill in missing field for uploading sources # in DB (see fields in models.py -> Source model) diff --git a/vast_pipeline/migrations/0003_alter_association_id_alter_measurement_id.py b/vast_pipeline/migrations/0003_alter_association_id_alter_measurement_id.py new file mode 100644 index 000000000..453df72c4 --- /dev/null +++ b/vast_pipeline/migrations/0003_alter_association_id_alter_measurement_id.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.6 on 2025-03-30 23:19 + +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('vast_pipeline', '0002_q3c'), + ] + + operations = [ + migrations.AlterField( + model_name='association', + name='id', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='measurement', + name='id', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), + ), + ] diff --git a/vast_pipeline/models.py b/vast_pipeline/models.py index 8666970c0..e99ea8cdd 100644 --- a/vast_pipeline/models.py +++ b/vast_pipeline/models.py @@ -1,4 +1,5 @@ import numpy as np +import uuid from dataclasses import dataclass from itertools import combinations @@ -417,7 +418,7 @@ class Measurement(CommentableModel): Essentially a source single measurement in time. """ - id = ShortUUIDField(primary_key=True, editable=False, length=UUID_LEN_MEAS, alphabet=UUID_ALPHABET) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) image = models.ForeignKey( Image, null=True, on_delete=models.CASCADE, to_field="id" ) # first image seen in @@ -713,7 +714,7 @@ class Association(models.Model): some parameters """ - id = ShortUUIDField(primary_key=True, editable=False, length=UUID_LEN_MEAS, alphabet=UUID_ALPHABET) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) source = models.ForeignKey(Source, on_delete=models.CASCADE, to_field="id") meas = models.ForeignKey(Measurement, on_delete=models.CASCADE, to_field="id") diff --git a/vast_pipeline/pipeline/forced_extraction.py b/vast_pipeline/pipeline/forced_extraction.py index 3c3779c3e..809fab5c0 100644 --- a/vast_pipeline/pipeline/forced_extraction.py +++ b/vast_pipeline/pipeline/forced_extraction.py @@ -1,6 +1,8 @@ import os import logging import datetime +import uuid + import numpy as np import pandas as pd import dask.dataframe as dd @@ -20,11 +22,7 @@ from vast_pipeline.pipeline.loading import copy_upload_measurements from forced_phot import ForcedPhot -from ..utils.utils import ( - StopWatch, - generate_shortuuid, - UUID_LEN_MEAS -) +from ..utils.utils import StopWatch from vast_pipeline.image.utils import open_fits # NOTE: We check here to see if we're in a testing environment. @@ -486,7 +484,7 @@ def _update_forced_measurements(df: pd.DataFrame) -> pd.DataFrame: The forced extraction dataframe updated with defaults. """ df["name"] = df["name"] + f"_f_{p_run_id}" - df["id"] = df.apply(lambda _: generate_shortuuid(UUID_LEN_MEAS), axis=1) + df["id"] = df.apply(lambda _: str(uuid.uuid4()), axis=1) default_pos_err = settings.POS_DEFAULT_MIN_ERROR / 3600.0 df["ra_err"] = default_pos_err df["dec_err"] = default_pos_err diff --git a/vast_pipeline/pipeline/loading.py b/vast_pipeline/pipeline/loading.py index c7651762a..8a0aa3a84 100644 --- a/vast_pipeline/pipeline/loading.py +++ b/vast_pipeline/pipeline/loading.py @@ -9,6 +9,7 @@ from itertools import islice from django.db import transaction, connection, models from contextlib import closing +from uuid import uuid4 from vast_pipeline.image.main import SelavyImage from vast_pipeline.pipeline.model_generator import ( @@ -432,7 +433,7 @@ def copy_upload_associations( } def upload(df, Association, mapping, batch_size): - df["db_id"] = df.apply(lambda _: generate_shortuuid(UUID_LEN_MEAS), axis=1) + df["db_id"] = df.apply(lambda _: str(uuid4()), axis=1) copy_upload_model(df, Association, mapping=mapping, batch_size=batch_size) associations_df = associations_df[columns_to_upload].map_partitions(upload, diff --git a/vast_pipeline/serializers.py b/vast_pipeline/serializers.py index cc889cf8f..866ab5425 100644 --- a/vast_pipeline/serializers.py +++ b/vast_pipeline/serializers.py @@ -56,7 +56,7 @@ class Meta: class MeasurementSerializer(serializers.ModelSerializer): - id = serializers.CharField(read_only=True) + id = serializers.UUIDField(read_only=True) frequency = serializers.SerializerMethodField(read_only=True) def get_frequency(self, obj): diff --git a/vast_pipeline/urls.py b/vast_pipeline/urls.py index 9581e36e0..02e986b6f 100644 --- a/vast_pipeline/urls.py +++ b/vast_pipeline/urls.py @@ -38,6 +38,11 @@ name="image_detail", ), path("measurements/", views.MeasurementIndex, name="measurement_index"), + path( + "measurements//", + views.MeasurementDetail, + name="measurement_detail" + ), re_path( fr"^measurements/(?P[\w]{{{UUID_LEN_MEAS}}})(?:/(?P[\w]+))?/$", views.MeasurementDetail, From 29c928b2671b36b9b19ca95ef0bdd531f7087499 Mon Sep 17 00:00:00 2001 From: Dougal Dobie Date: Tue, 20 May 2025 14:46:27 +1000 Subject: [PATCH 4/5] PEP8 --- vast_pipeline/image/main.py | 2 +- vast_pipeline/pipeline/loading.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/vast_pipeline/image/main.py b/vast_pipeline/image/main.py index c6219f6b5..1ec11a261 100644 --- a/vast_pipeline/image/main.py +++ b/vast_pipeline/image/main.py @@ -15,7 +15,7 @@ from astropy.wcs.utils import proj_plane_pixel_scales from typing import Dict -from .utils import calc_condon_flux_errors, open_fits +from .utils import calc_condon_flux_errors from vast_pipeline import models from vast_pipeline.survey.translators import tr_selavy diff --git a/vast_pipeline/pipeline/loading.py b/vast_pipeline/pipeline/loading.py index 8a0aa3a84..0d5ef6e19 100644 --- a/vast_pipeline/pipeline/loading.py +++ b/vast_pipeline/pipeline/loading.py @@ -37,7 +37,6 @@ deg2hms, deg2dms, generate_shortuuid, - UUID_LEN_MEAS, UUID_LEN_SOURCE ) From ee4b8da46f5ca1a32401980d79468b469ee26a3a Mon Sep 17 00:00:00 2001 From: Dougal Dobie Date: Tue, 20 May 2025 15:02:02 +1000 Subject: [PATCH 5/5] Changelog updates --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73a923cee..740742e99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed +- V2: Change measurement and association DB indices to UUIDs [#844](https://github.com/askap-vast/vast-pipeline/pull/844) - V2: Limit associations upload to using num_io_workers [#833](https://github.com/askap-vast/vast-pipeline/pull/833) - V2: Allow user specification of dask dashboard paramters and add some further logging to dask setup [#829](https://github.com/askap-vast/vast-pipeline/pull/829) - V2: Replace all source, measurement, run etc. IDs with UUID indices. [#803](https://github.com/askap-vast/vast-pipeline/pull/803) @@ -35,7 +36,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Fixed -- V2: Fixed problem with webapp plots related to UUIDs [#844](https://github.com/askap-vast/vast-pipeline/pull/844) - V2: Pair metrics working with V2 dask LocalCluster changes [#817](https://github.com/askap-vast/vast-pipeline/pull/817) - V2: Fixed missing JS9 overlays on source webpage [#809](https://github.com/askap-vast/vast-pipeline/pull/809) - V2: Fix bug when deleting source tags from database which are referenced by multiple sources [#803](https://github.com/askap-vast/vast-pipeline/pull/803) @@ -59,7 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### List of PRs -- [#844](https://github.com/askap-vast/vast-pipeline/pull/844): fix: V2: Fixed problem with webapp plots related to UUIDs +- [#844](https://github.com/askap-vast/vast-pipeline/pull/844): fix: V2: Change measurements and association DB indices to UUIDs - [#843](https://github.com/askap-vast/vast-pipeline/pull/843): feat: V2: Enable specification of Dask worker memory limits - [#833](https://github.com/askap-vast/vast-pipeline/pull/833): feat: V2: Limit associations upload to using num_io_workers - [#829](https://github.com/askap-vast/vast-pipeline/pull/829): feat: V2: Allow user specification of dask dashboard paramters and add some further logging to dask setup