diff --git a/CHANGELOG.md b/CHANGELOG.md index 541e13fb4..740742e99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed +- V2: Change measurement and association DB indices to UUIDs [#844](https://github.com/askap-vast/vast-pipeline/pull/844) - V2: Limit associations upload to using num_io_workers [#833](https://github.com/askap-vast/vast-pipeline/pull/833) - V2: Allow user specification of dask dashboard paramters and add some further logging to dask setup [#829](https://github.com/askap-vast/vast-pipeline/pull/829) - V2: Replace all source, measurement, run etc. IDs with UUID indices. [#803](https://github.com/askap-vast/vast-pipeline/pull/803) @@ -58,6 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### List of PRs +- [#844](https://github.com/askap-vast/vast-pipeline/pull/844): fix: V2: Change measurements and association DB indices to UUIDs - [#843](https://github.com/askap-vast/vast-pipeline/pull/843): feat: V2: Enable specification of Dask worker memory limits - [#833](https://github.com/askap-vast/vast-pipeline/pull/833): feat: V2: Limit associations upload to using num_io_workers - [#829](https://github.com/askap-vast/vast-pipeline/pull/829): feat: V2: Allow user specification of dask dashboard paramters and add some further logging to dask setup diff --git a/vast_pipeline/image/main.py b/vast_pipeline/image/main.py index 1a0f7aab5..1ec11a261 100644 --- a/vast_pipeline/image/main.py +++ b/vast_pipeline/image/main.py @@ -6,6 +6,7 @@ import logging import numpy as np import pandas as pd +import uuid from django.conf import settings from astropy.io import fits @@ -18,9 +19,7 @@ from vast_pipeline import models from vast_pipeline.survey.translators import tr_selavy -from vast_pipeline.utils.utils import ( - generate_shortuuid, UUID_LEN_MEAS, -) + from vast_pipeline.image.utils import get_fits_header @@ -341,7 +340,7 @@ def read_selavy(self, dj_image: models.Image) -> pd.DataFrame: df[key["name"]] = df[key["name"]].astype(key["dtype"]) # Add id column - df["id"] = df.apply(lambda _: generate_shortuuid(UUID_LEN_MEAS), axis=1) + df["id"] = df.apply(lambda _: str(uuid.uuid4()), axis=1) # do checks and fill in missing field for uploading sources # in DB (see fields in models.py -> Source model) diff --git a/vast_pipeline/migrations/0003_alter_association_id_alter_measurement_id.py b/vast_pipeline/migrations/0003_alter_association_id_alter_measurement_id.py new file mode 100644 index 000000000..453df72c4 --- /dev/null +++ b/vast_pipeline/migrations/0003_alter_association_id_alter_measurement_id.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.6 on 2025-03-30 23:19 + +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('vast_pipeline', '0002_q3c'), + ] + + operations = [ + migrations.AlterField( + model_name='association', + name='id', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='measurement', + name='id', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), + ), + ] diff --git a/vast_pipeline/models.py b/vast_pipeline/models.py index 8666970c0..e99ea8cdd 100644 --- a/vast_pipeline/models.py +++ b/vast_pipeline/models.py @@ -1,4 +1,5 @@ import numpy as np +import uuid from dataclasses import dataclass from itertools import combinations @@ -417,7 +418,7 @@ class Measurement(CommentableModel): Essentially a source single measurement in time. """ - id = ShortUUIDField(primary_key=True, editable=False, length=UUID_LEN_MEAS, alphabet=UUID_ALPHABET) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) image = models.ForeignKey( Image, null=True, on_delete=models.CASCADE, to_field="id" ) # first image seen in @@ -713,7 +714,7 @@ class Association(models.Model): some parameters """ - id = ShortUUIDField(primary_key=True, editable=False, length=UUID_LEN_MEAS, alphabet=UUID_ALPHABET) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) source = models.ForeignKey(Source, on_delete=models.CASCADE, to_field="id") meas = models.ForeignKey(Measurement, on_delete=models.CASCADE, to_field="id") diff --git a/vast_pipeline/pipeline/forced_extraction.py b/vast_pipeline/pipeline/forced_extraction.py index 3c3779c3e..809fab5c0 100644 --- a/vast_pipeline/pipeline/forced_extraction.py +++ b/vast_pipeline/pipeline/forced_extraction.py @@ -1,6 +1,8 @@ import os import logging import datetime +import uuid + import numpy as np import pandas as pd import dask.dataframe as dd @@ -20,11 +22,7 @@ from vast_pipeline.pipeline.loading import copy_upload_measurements from forced_phot import ForcedPhot -from ..utils.utils import ( - StopWatch, - generate_shortuuid, - UUID_LEN_MEAS -) +from ..utils.utils import StopWatch from vast_pipeline.image.utils import open_fits # NOTE: We check here to see if we're in a testing environment. @@ -486,7 +484,7 @@ def _update_forced_measurements(df: pd.DataFrame) -> pd.DataFrame: The forced extraction dataframe updated with defaults. """ df["name"] = df["name"] + f"_f_{p_run_id}" - df["id"] = df.apply(lambda _: generate_shortuuid(UUID_LEN_MEAS), axis=1) + df["id"] = df.apply(lambda _: str(uuid.uuid4()), axis=1) default_pos_err = settings.POS_DEFAULT_MIN_ERROR / 3600.0 df["ra_err"] = default_pos_err df["dec_err"] = default_pos_err diff --git a/vast_pipeline/pipeline/loading.py b/vast_pipeline/pipeline/loading.py index c7651762a..0d5ef6e19 100644 --- a/vast_pipeline/pipeline/loading.py +++ b/vast_pipeline/pipeline/loading.py @@ -9,6 +9,7 @@ from itertools import islice from django.db import transaction, connection, models from contextlib import closing +from uuid import uuid4 from vast_pipeline.image.main import SelavyImage from vast_pipeline.pipeline.model_generator import ( @@ -36,7 +37,6 @@ deg2hms, deg2dms, generate_shortuuid, - UUID_LEN_MEAS, UUID_LEN_SOURCE ) @@ -432,7 +432,7 @@ def copy_upload_associations( } def upload(df, Association, mapping, batch_size): - df["db_id"] = df.apply(lambda _: generate_shortuuid(UUID_LEN_MEAS), axis=1) + df["db_id"] = df.apply(lambda _: str(uuid4()), axis=1) copy_upload_model(df, Association, mapping=mapping, batch_size=batch_size) associations_df = associations_df[columns_to_upload].map_partitions(upload, diff --git a/vast_pipeline/plots.py b/vast_pipeline/plots.py index b12845f8f..e16f5841a 100644 --- a/vast_pipeline/plots.py +++ b/vast_pipeline/plots.py @@ -84,6 +84,7 @@ def plot_lightcurve( # lightcurve required cols: taustart_ts, flux, flux_err_upper, flux_err_lower, forced lightcurve = pd.DataFrame(measurements_qs) + lightcurve['id'] = lightcurve['id'].astype(str) # remap method values to labels to make a better legend lightcurve["method"] = lightcurve.forced.map({True: "Forced", False: "Selavy"}) @@ -167,6 +168,13 @@ def plot_lightcurve( ) .reset_index() ) + + candidate_measurement_pairs_df = candidate_measurement_pairs_df.astype( + {'measurement_a_id': 'str', + 'measurement_b_id': 'str', + } + ) + g = nx.Graph() for _row in candidate_measurement_pairs_df.itertuples(index=False): g.add_edge(_row.measurement_a_id, _row.measurement_b_id) diff --git a/vast_pipeline/serializers.py b/vast_pipeline/serializers.py index cc889cf8f..866ab5425 100644 --- a/vast_pipeline/serializers.py +++ b/vast_pipeline/serializers.py @@ -56,7 +56,7 @@ class Meta: class MeasurementSerializer(serializers.ModelSerializer): - id = serializers.CharField(read_only=True) + id = serializers.UUIDField(read_only=True) frequency = serializers.SerializerMethodField(read_only=True) def get_frequency(self, obj): diff --git a/vast_pipeline/urls.py b/vast_pipeline/urls.py index 9581e36e0..02e986b6f 100644 --- a/vast_pipeline/urls.py +++ b/vast_pipeline/urls.py @@ -38,6 +38,11 @@ name="image_detail", ), path("measurements/", views.MeasurementIndex, name="measurement_index"), + path( + "measurements//", + views.MeasurementDetail, + name="measurement_detail" + ), re_path( fr"^measurements/(?P[\w]{{{UUID_LEN_MEAS}}})(?:/(?P[\w]+))?/$", views.MeasurementDetail,