diff --git a/pyproject.toml b/pyproject.toml index 0156c2b38..179f92dbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,11 +55,14 @@ packages = [ members = ["python/loris_*"] [tool.uv.sources] -loris_bids_importer = { workspace = true } -loris_bids_utils = { workspace = true } -loris_dicom_importer = { workspace = true } -loris_ephys_chunker = { workspace = true } -loris_utils = { workspace = true } +loris_bids_importer = { workspace = true } +loris_bids_utils = { workspace = true } +loris_dicom_importer = { workspace = true } +loris_ephys_chunker = { workspace = true } +loris_ephys_visualizer_module = { workspace = true } +loris_meegqc_module = { workspace = true } +loris_server = { workspace = true } +loris_utils = { workspace = true } [tool.ruff] src = ["python"] @@ -68,8 +71,8 @@ line-length = 120 preview = true [tool.ruff.lint] -ignore = ["E202", "E203", "E221", "E241", "E251", "E272"] -select = ["E", "EXE", "F", "I", "N", "RUF", "UP", "W"] +ignore = ["E202", "E203", "E221", "E241", "E251", "E272", "FAST003"] +select = ["E", "EXE", "F", "FAST", "I", "N", "RUF", "UP", "W"] [tool.ruff.lint.pycodestyle] max-doc-length = 100 @@ -77,6 +80,7 @@ max-doc-length = 100 [tool.ruff.lint.per-file-ignores] # ORM models often have very long lines. "python/lib/db/models/*.py" = ["E501"] +"python/loris_meegqc_module/src/loris_meegqc_module/database/models/*.py" = ["E501"] # The strict type checking configuration is used to type check only the modern (typed) modules. An # additional basic type checking configuration to type check legacy modules can be found in the diff --git a/python/lib/config.py b/python/lib/config.py index 60430a284..4a4779ab5 100644 --- a/python/lib/config.py +++ b/python/lib/config.py @@ -7,6 +7,14 @@ from lib.logging import log_error_exit +def get_jwt_secret_key_config(env: Env) -> str: + """ + Get the LORIS JWT secret key from the in-database configuration. + """ + + return _get_config_value(env, 'JWTKey') + + def get_patient_id_dicom_header_config(env: Env) -> Literal['PatientID', 'PatientName']: """ Get the DICOM header in which to look for the patient ID from the in-database configuration, or diff --git a/python/lib/db/queries/meg_ctf_head_shape.py b/python/lib/db/queries/meg_ctf_head_shape.py new file mode 100644 index 000000000..f96ffee64 --- /dev/null +++ b/python/lib/db/queries/meg_ctf_head_shape.py @@ -0,0 +1,12 @@ +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.orm import Session as Database + +from lib.db.models.meg_ctf_head_shape_file import DbMegCtfHeadShapeFile + + +def try_get_meg_ctf_head_shape_file_with_path(db: Database, path: Path) -> DbMegCtfHeadShapeFile | None: + return db.execute(select(DbMegCtfHeadShapeFile) + .where(DbMegCtfHeadShapeFile.path == path) + ).scalar_one_or_none() diff --git a/python/lib/db/queries/user.py b/python/lib/db/queries/user.py new file mode 100644 index 000000000..3ec2e496f --- /dev/null +++ b/python/lib/db/queries/user.py @@ -0,0 +1,24 @@ +from sqlalchemy import select +from sqlalchemy.orm import Session as Database + +from lib.db.models.user import DbUser + + +def try_get_user_with_id(db: Database, user_id: int) -> DbUser | None: + """ + Get a user from the database using its ID, or return `None` if no user is found. + """ + + return db.execute(select(DbUser) + .where(DbUser.id == user_id) + ).scalar_one_or_none() + + +def try_get_user_with_username(db: Database, username: str) -> DbUser | None: + """ + Get a user from the database using its username, or return `None` if no user is found. + """ + + return db.execute(select(DbUser) + .where(DbUser.username == username) + ).scalar_one_or_none() diff --git a/python/lib/physio/file.py b/python/lib/physio/file.py index e1891da3f..ebf9b5c90 100644 --- a/python/lib/physio/file.py +++ b/python/lib/physio/file.py @@ -3,6 +3,7 @@ from pathlib import Path from lib.db.models.imaging_file_type import DbImagingFileType +from lib.db.models.meg_ctf_head_shape_file import DbMegCtfHeadShapeFile from lib.db.models.physio_file import DbPhysioFile from lib.db.models.physio_modality import DbPhysioModality from lib.db.models.physio_output_type import DbPhysioOutputType @@ -18,6 +19,7 @@ def insert_physio_file( modality: DbPhysioModality, output_type: DbPhysioOutputType, acquisition_time: datetime | None, + head_shape_file: DbMegCtfHeadShapeFile | None = None, ) -> DbPhysioFile: """ Insert a physiological file into the database. @@ -31,6 +33,7 @@ def insert_physio_file( output_type_id = output_type.id, acquisition_time = acquisition_time, inserted_by_user = getpass.getuser(), + head_shape_file_id = head_shape_file.id if head_shape_file is not None else None, ) env.db.add(file) diff --git a/python/lib/user.py b/python/lib/user.py new file mode 100644 index 000000000..a37e5f744 --- /dev/null +++ b/python/lib/user.py @@ -0,0 +1,29 @@ +from lib.db.models.project import DbProject +from lib.db.models.session import DbSession +from lib.db.models.site import DbSite +from lib.db.models.user import DbUser +from lib.env import Env + + +def can_user_access_project(_: Env, user: DbUser, project: DbProject) -> bool: + """ + Check whether a user has access to a project. + """ + + return project in user.projects + + +def can_user_access_site(_: Env, user: DbUser, site: DbSite) -> bool: + """ + Check whether a user has access to a site. + """ + + return site in user.sites + + +def can_user_access_session(env: Env, user: DbUser, session: DbSession) -> bool: + """ + Check whether a user has access to a session. + """ + + return can_user_access_site(env, user, session.site) and can_user_access_project(env, user, session.project) diff --git a/python/loris_bids_importer/src/loris_bids_importer/acquisitions.py b/python/loris_bids_importer/src/loris_bids_importer/acquisitions.py index 41a663772..da5d9b258 100644 --- a/python/loris_bids_importer/src/loris_bids_importer/acquisitions.py +++ b/python/loris_bids_importer/src/loris_bids_importer/acquisitions.py @@ -5,14 +5,14 @@ from lib.logging import log, log_error from loris_bids_utils.info import BidsAcquisitionInfo -from loris_bids_importer.env import BidsImportEnv +from loris_bids_importer.importer import BidsImporter T = TypeVar('T') def import_bids_acquisitions( env: Env, - import_env: BidsImportEnv, + import_env: BidsImporter, acquisitions: list[tuple[T, BidsAcquisitionInfo]], importer: Callable[[T, BidsAcquisitionInfo], None] ): @@ -32,6 +32,7 @@ def import_bids_acquisitions( log(env, f"Successfully imported acquisition '{bids_info.name}'.") import_env.imported_acquisitions_count += 1 except Exception as exception: + import traceback log_error( env, ( @@ -40,4 +41,5 @@ def import_bids_acquisitions( "Skipping." ) ) + print(traceback.format_exc()) import_env.failed_acquisitions_count += 1 diff --git a/python/loris_bids_importer/src/loris_bids_importer/args.py b/python/loris_bids_importer/src/loris_bids_importer/args.py deleted file mode 100644 index b4d8f549d..000000000 --- a/python/loris_bids_importer/src/loris_bids_importer/args.py +++ /dev/null @@ -1,14 +0,0 @@ -from dataclasses import dataclass -from pathlib import Path -from typing import Literal - - -@dataclass -class Args: - source_bids_path: Path - type: Literal[None, 'raw', 'derivative'] - bids_validation: bool - create_candidate: bool - create_session: bool - copy: bool - verbose: bool diff --git a/python/loris_bids_importer/src/loris_bids_importer/channels.py b/python/loris_bids_importer/src/loris_bids_importer/channels.py index f75ce456a..b08b7afc8 100644 --- a/python/loris_bids_importer/src/loris_bids_importer/channels.py +++ b/python/loris_bids_importer/src/loris_bids_importer/channels.py @@ -14,12 +14,12 @@ from loris_utils.error import group_errors, group_errors_tuple from loris_bids_importer.copy_files import get_loris_bids_file_path -from loris_bids_importer.env import BidsImportEnv +from loris_bids_importer.importer import BidsImporter def insert_bids_channels_file( env: Env, - import_env: BidsImportEnv, + import_env: BidsImporter, physio_file: DbPhysioFile, session: DbSession, acquisition: BidsAcquisitionInfo, diff --git a/python/loris_bids_importer/src/loris_bids_importer/copy_files.py b/python/loris_bids_importer/src/loris_bids_importer/copy_files.py index ac9f068b5..ee295ea7e 100644 --- a/python/loris_bids_importer/src/loris_bids_importer/copy_files.py +++ b/python/loris_bids_importer/src/loris_bids_importer/copy_files.py @@ -9,7 +9,7 @@ from loris_bids_utils.files.participants import BidsParticipantsTsvFile from loris_bids_utils.files.scans import BidsScansTsvFile -from loris_bids_importer.env import BidsImportEnv +from loris_bids_importer.importer import BidsImporter def get_loris_bids_dataset_path(env: Env, dataset_description: BidsDatasetDescriptionJsonFile) -> Path: @@ -31,21 +31,21 @@ def get_loris_bids_dataset_path(env: Env, dataset_description: BidsDatasetDescri return loris_bids_path -def get_loris_bids_root_file_path(import_env: BidsImportEnv, file_path: Path) -> Path: +def get_loris_bids_root_file_path(importer: BidsImporter, file_path: Path) -> Path: """ Get the path of a BIDS file relative to the LORIS data directory, maintaining the same relative path in the LORIS BIDS dataset as within the source BIDS dataset. """ # In the import is run in no-copy mode, return the original file path. - if import_env.loris_bids_path is None: - return file_path.relative_to(import_env.data_dir_path) + if importer.loris_bids_path is None: + return file_path.relative_to(importer.data_dir_path) - return import_env.loris_bids_path / file_path.relative_to(import_env.source_bids_path) + return importer.loris_bids_path / file_path.relative_to(importer.args.source_bids_path) def get_loris_bids_file_path( - import_env: BidsImportEnv, + importer: BidsImporter, session: DbSession, data_type: str, file_path: Path, @@ -56,20 +56,20 @@ def get_loris_bids_file_path( """ # In the import is run in no-copy mode, return the original file path. - if import_env.loris_bids_path is None: - return file_path.relative_to(import_env.data_dir_path) + if importer.loris_bids_path is None: + return file_path.relative_to(importer.data_dir_path) # If the file is a derivative, the path is unpredictable, so return a copy of that path in the # LORIS BIDS dataset. if derivative: - return import_env.loris_bids_path / file_path.relative_to(import_env.source_bids_path) + return importer.loris_bids_path / file_path.relative_to(importer.args.source_bids_path) # Otherwise, normalize the subject and session directory names using the LORIS session # information. loris_file_name = get_loris_bids_file_name(file_path.name, session) return ( - import_env.loris_bids_path + importer.loris_bids_path / f'sub-{session.candidate.psc_id}' / f'ses-{session.visit_label}' / data_type @@ -91,34 +91,43 @@ def get_loris_bids_file_name(file_name: str, session: DbSession) -> str: return f'sub-{session.candidate.psc_id}_ses-{session.visit_label}_{file_name}' -def get_loris_scans_path(import_env: BidsImportEnv, scans_file: BidsScansTsvFile, session: DbSession) -> Path: +def get_loris_scans_path(importer: BidsImporter, scans_file: BidsScansTsvFile, session: DbSession) -> Path: """ Get the path of a `scans.tsv` file in LORIS, relative to the LORIS data directory. """ # In the import is run in no-copy mode, return the original file path. - if import_env.loris_bids_path is None: - return scans_file.path.relative_to(import_env.data_dir_path) + if importer.loris_bids_path is None: + return scans_file.path.relative_to(importer.data_dir_path) loris_file_name = get_loris_bids_file_name(scans_file.path.name, session) return ( - import_env.loris_bids_path + importer.loris_bids_path / f'sub-{session.candidate.psc_id}' / f'ses-{session.visit_label}' / loris_file_name ) -def copy_loris_bids_file(import_env: BidsImportEnv, file_path: Path, loris_file_path: Path): +def copy_loris_bids_file(importer: BidsImporter, file_path: Path, loris_file_path: Path): """ Copy a BIDS file to the LORIS data directory, unless the no-copy mode is enabled. """ + input_bids_file_path = file_path.relative_to(importer.args.source_bids_path) + + if importer.loris_bids_path is not None: + output_bids_file_path = loris_file_path.relative_to(importer.loris_bids_path) + else: + output_bids_file_path = input_bids_file_path + + importer.files_dict[input_bids_file_path] = output_bids_file_path + # Do not copy the file in no-copy mode. - if import_env.loris_bids_path is None: + if importer.loris_bids_path is None: return - full_loris_file_path = import_env.data_dir_path / loris_file_path + full_loris_file_path = importer.data_dir_path / loris_file_path if full_loris_file_path.exists(): raise Exception(f"File '{loris_file_path}' already exists in the LORIS data directory.") @@ -130,31 +139,31 @@ def copy_loris_bids_file(import_env: BidsImportEnv, file_path: Path, loris_file_ shutil.copytree(file_path, full_loris_file_path) -def copy_bids_static_files(import_env: BidsImportEnv): +def copy_bids_static_files(importer: BidsImporter): """ Copy the static files of the source BIDS dataset to the LORIS BIDS dataset. """ # Do not copy files in no-copy mode. - if import_env.loris_bids_path is None: + if importer.loris_bids_path is None: return for file_name in ['README', 'dataset_description.json']: - source_file_path = import_env.source_bids_path / file_name + source_file_path = importer.args.source_bids_path / file_name if not source_file_path.is_file(): continue - loris_file_path = import_env.loris_bids_path / file_name + loris_file_path = importer.loris_bids_path / file_name # Do not copy the file if it is already present during an incremental import. - if (import_env.data_dir_path / loris_file_path).is_file(): + if (importer.data_dir_path / loris_file_path).is_file(): continue - copy_loris_bids_file(import_env, source_file_path, loris_file_path) + copy_loris_bids_file(importer, source_file_path, loris_file_path) def copy_bids_participants_file( - import_env: BidsImportEnv, + importer: BidsImporter, participants_file: BidsParticipantsTsvFile, loris_participants_path: Path, ): @@ -164,10 +173,10 @@ def copy_bids_participants_file( """ # Do not copy the file in no-copy mode. - if import_env.loris_bids_path is None: + if importer.loris_bids_path is None: return - participants_path = import_env.data_dir_path / loris_participants_path + participants_path = importer.data_dir_path / loris_participants_path if participants_path.exists(): participants_file.merge(BidsParticipantsTsvFile(participants_path)) @@ -175,16 +184,16 @@ def copy_bids_participants_file( participants_file.write(participants_path) -def copy_bids_scans_file(import_env: BidsImportEnv, scans_file: BidsScansTsvFile, loris_scans_path: Path): +def copy_bids_scans_file(importer: BidsImporter, scans_file: BidsScansTsvFile, loris_scans_path: Path): """ Copy some `scans.tsv` rows into a LORIS `scans.tsv` file, creating it if necessary. """ # Do not copy the file in no-copy mode. - if import_env.loris_bids_path is None: + if importer.loris_bids_path is None: return - scans_path = import_env.data_dir_path / loris_scans_path + scans_path = importer.data_dir_path / loris_scans_path if scans_path.exists(): scans_file.merge(BidsScansTsvFile(scans_path)) diff --git a/python/loris_bids_importer/src/loris_bids_importer/eeg/main.py b/python/loris_bids_importer/src/loris_bids_importer/eeg/main.py index 0ea7714fe..14d5a76d0 100644 --- a/python/loris_bids_importer/src/loris_bids_importer/eeg/main.py +++ b/python/loris_bids_importer/src/loris_bids_importer/eeg/main.py @@ -2,10 +2,8 @@ import json import os -import sys from pathlib import Path -import lib.exitcode import lib.utilities as utilities from lib.config import get_ephys_visualization_enabled_config from lib.db.models.physio_file import DbPhysioFile @@ -28,16 +26,17 @@ from loris_bids_importer.archive import import_physio_event_archive, import_physio_file_archive from loris_bids_importer.channels import insert_bids_channels_file -from loris_bids_importer.copy_files import copy_loris_bids_file, get_loris_bids_file_path, get_loris_scans_path +from loris_bids_importer.copy_files import copy_loris_bids_file, get_loris_bids_file_path from loris_bids_importer.eeg.physiological import Physiological -from loris_bids_importer.env import BidsImportEnv from loris_bids_importer.events import insert_bids_event_dict_file, insert_bids_events_file from loris_bids_importer.file_type import get_check_bids_imaging_file_type_from_extension +from loris_bids_importer.importer import BidsImporter from loris_bids_importer.physio import ( get_check_bids_physio_file_hash, get_check_bids_physio_modality, get_check_bids_physio_output_type, ) +from loris_bids_importer.scans import add_bids_scans_file_parameters class Eeg: @@ -46,22 +45,18 @@ class Eeg: into the database by calling the loris_bids_importer.eeg.physiological class. """ - def __init__(self, env: Env, import_env: BidsImportEnv, bids_layout, bids_info: BidsDataTypeInfo, - session: DbSession, db, dataset_tag_dict, dataset_type): + def __init__(self, env: Env, importer: BidsImporter, bids_layout, bids_info: BidsDataTypeInfo, + session: DbSession, db, dataset_tag_dict): """ Constructor method for the Eeg class. - :param bids_reader : dictionary with BIDS reader information - :type bids_reader : dict + :param bids_layout : PyBIDS layout :param bids_info : the BIDS data type information :param session : The LORIS session the EEG datasets are linked to :param db : Database class object :type db : object :param info : The BIDS import pipeline information :param dataset_tag_dict : Dict of dataset-inherited HED tags - :type dataset_tag_dict : dict - :param dataset_type : raw | derivative. Type of the dataset - :type dataset_type : string """ self.env = env @@ -71,7 +66,7 @@ def __init__(self, env: Env, import_env: BidsImportEnv, bids_layout, bids_info: # load the LORIS BIDS import root directory where the eeg files will # be copied - self.info = import_env + self.info = importer self.data_dir = self.info.data_dir_path # load bids subject, visit and modality @@ -99,13 +94,14 @@ def __init__(self, env: Env, import_env: BidsImportEnv, bids_layout, bids_info: self.scans_file = BidsScansTsvFile(Path(scans_file_path)) # register the data into LORIS - if (dataset_type and dataset_type == 'raw'): - self.register_data(detect=False) - elif (dataset_type and dataset_type == 'derivative'): - self.register_data(derivatives=True, detect=False) - else: - self.register_data() - self.register_data(derivatives=True) + match importer.args.type: + case 'raw': + self.register_data(detect=False) + case 'derivative': + self.register_data(derivatives=True, detect=False) + case None: + self.register_data() + self.register_data(derivatives=True) env.db.commit() @@ -306,17 +302,8 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True): if self.scans_file is not None: scan_info = self.scans_file.get_row(eeg_file_path) if scan_info is not None: - try: - eeg_acq_time = scan_info.get_acquisition_time() - eeg_file_data['age_at_scan'] = scan_info.get_age_at_scan() - except Exception as error: - print(f"ERROR: {error}") - sys.exit(lib.exitcode.PROGRAM_EXECUTION_FAILURE) - - loris_scans_path = get_loris_scans_path(self.info, self.scans_file, self.session) - eeg_file_data['scans_tsv_file'] = loris_scans_path - scans_blake2 = compute_file_blake2b_hash(self.scans_file.path) - eeg_file_data['physiological_scans_tsv_file_bake2hash'] = scans_blake2 + eeg_acq_time = scan_info.get_acquisition_time() + add_bids_scans_file_parameters(self.info, self.session, self.scans_file, scan_info, eeg_file_data) # if file type is set and fdt file exists, append fdt path to the # eeg_file_data dictionary diff --git a/python/loris_bids_importer/src/loris_bids_importer/env.py b/python/loris_bids_importer/src/loris_bids_importer/env.py deleted file mode 100644 index d1b074e80..000000000 --- a/python/loris_bids_importer/src/loris_bids_importer/env.py +++ /dev/null @@ -1,47 +0,0 @@ -from dataclasses import dataclass -from pathlib import Path - - -@dataclass -class BidsImportEnv: - """ - Information about a specific BIDS import pipeline run. - """ - - data_dir_path: Path - """ - The LORIS data directory path. - """ - - source_bids_path: Path - """ - The source BIDS directory path. - """ - - loris_bids_path: Path | None - """ - The LORIS BIDS directory path for this import, relative to the LORIS data directory. - """ - - imported_acquisitions_count: int = 0 - """ - The number of successfully imported BIDS acquisitions. - """ - - ignored_acquisitions_count: int = 0 - """ - The number of ignored BIDS acquisition imports. - """ - - failed_acquisitions_count: int = 0 - """ - The number of failed BIDS acquisition imports. - """ - - @property - def processed_acquisitions_count(self) -> int: - """ - The total number of processed BIDS acquisitions. - """ - - return self.imported_acquisitions_count + self.ignored_acquisitions_count + self.failed_acquisitions_count diff --git a/python/loris_bids_importer/src/loris_bids_importer/events.py b/python/loris_bids_importer/src/loris_bids_importer/events.py index a3e962ba8..234ab4998 100644 --- a/python/loris_bids_importer/src/loris_bids_importer/events.py +++ b/python/loris_bids_importer/src/loris_bids_importer/events.py @@ -24,12 +24,12 @@ from loris_utils.crypto import compute_file_blake2b_hash from loris_bids_importer.copy_files import copy_loris_bids_file, get_loris_bids_root_file_path -from loris_bids_importer.env import BidsImportEnv +from loris_bids_importer.importer import BidsImporter def import_bids_root_event_dict_file( env: Env, - import_env: BidsImportEnv, + import_env: BidsImporter, project: DbProject, bids_event_dict_file: BidsJsonFile, ) -> tuple[DbPhysioEventFile, dict[str, dict[str, list[list[TagGroupMember]]]]]: diff --git a/python/loris_bids_importer/src/loris_bids_importer/importer.py b/python/loris_bids_importer/src/loris_bids_importer/importer.py new file mode 100644 index 000000000..8c4d0d434 --- /dev/null +++ b/python/loris_bids_importer/src/loris_bids_importer/importer.py @@ -0,0 +1,96 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + + +@dataclass +class BidsImporterArgs: + """ + The arguments given to the BIDS importer. + """ + + source_bids_path: Path + """ + The path of the source BIDS dataset to import. + """ + + type: Literal['raw', 'derivative', None] + """ + The type of the BIDS dataset to import. + """ + + bids_validation: bool + """ + Whether to validate the BIDS dataset. + """ + + create_candidate: bool + """ + Whether to create candidates in LORIS. + """ + + create_session: bool + """ + Whether to create sessions in LORIS. + """ + + copy: bool + """ + Whether to copy the BIDS dataset into the LORIS data directory. + """ + + verbose: bool + """ + Whether to enable verbose output. + """ + + +@dataclass +class BidsImporter: + """ + Information about the current BIDS importer run. + """ + + args: BidsImporterArgs + """ + The arguments given to the BIDS importer. + """ + + data_dir_path: Path + """ + The LORIS data directory path. + """ + + loris_bids_path: Path | None + """ + The LORIS BIDS directory path for this import, relative to the LORIS data directory. + """ + + files_dict: dict[Path, Path] + """ + A dictionary mapping the original BIDS file paths to their corresponding paths in the LORIS data + directory. Both paths are relative to their respective BIDS dataset root directories. + """ + + imported_acquisitions_count: int = 0 + """ + The number of successfully imported BIDS acquisitions. + """ + + ignored_acquisitions_count: int = 0 + """ + The number of ignored BIDS acquisition imports. + """ + + failed_acquisitions_count: int = 0 + """ + The number of failed BIDS acquisition imports. + """ + + @property + def processed_acquisitions_count(self) -> int: + """ + The total number of processed BIDS acquisitions. + """ + + return self.imported_acquisitions_count + self.ignored_acquisitions_count + self.failed_acquisitions_count diff --git a/python/loris_bids_importer/src/loris_bids_importer/main.py b/python/loris_bids_importer/src/loris_bids_importer/main.py index 538011ac2..b0c070a61 100644 --- a/python/loris_bids_importer/src/loris_bids_importer/main.py +++ b/python/loris_bids_importer/src/loris_bids_importer/main.py @@ -1,3 +1,4 @@ +from importlib.metadata import entry_points from typing import Any from lib.config import get_data_dir_path_config, get_default_bids_visit_label_config @@ -7,10 +8,10 @@ from lib.db.queries.session import try_get_session_with_cand_id_visit_label from lib.env import Env from lib.logging import log, log_error, log_error_exit, log_warning +from loris_bids_utils.meg.reader import BidsMegDataTypeReader from loris_bids_utils.mri.reader import BidsMriDataTypeReader from loris_bids_utils.reader import BidsDatasetReader, BidsDataTypeReader, BidsSessionReader -from loris_bids_importer.args import Args from loris_bids_importer.copy_files import ( copy_bids_participants_file, copy_bids_scans_file, @@ -20,21 +21,20 @@ get_loris_scans_path, ) from loris_bids_importer.eeg.main import Eeg -from loris_bids_importer.env import BidsImportEnv from loris_bids_importer.events import import_bids_root_event_dict_file +from loris_bids_importer.importer import BidsImporter, BidsImporterArgs +from loris_bids_importer.meg.ctf import import_bids_meg_data_type from loris_bids_importer.mri.main import import_bids_mri_data_type from loris_bids_importer.print import print_bids_import_summary, print_bids_info from loris_bids_importer.validation.sessions import validate_bids_sessions from loris_bids_importer.validation.subjects import validate_bids_subjects -def import_bids_dataset(env: Env, args: Args, legacy_db: Database): +def import_bids_dataset(env: Env, args: BidsImporterArgs, legacy_db: Database): """ Read the provided BIDS dataset and import it into LORIS. """ - data_dir_path = get_data_dir_path_config(env) - log(env, "Parsing BIDS dataset...") bids = BidsDatasetReader(args.source_bids_path, args.type == 'derivative', args.bids_validation) @@ -79,15 +79,18 @@ def import_bids_dataset(env: Env, args: Args, legacy_db: Database): else: loris_bids_path = None - import_env = BidsImportEnv( - data_dir_path = data_dir_path, - loris_bids_path = loris_bids_path.relative_to(data_dir_path) if loris_bids_path is not None else None, - source_bids_path = args.source_bids_path, + data_dir_path = get_data_dir_path_config(env) + + importer = BidsImporter( + data_dir_path = data_dir_path, + args = args, + loris_bids_path = loris_bids_path.relative_to(data_dir_path) if loris_bids_path is not None else None, + files_dict = {}, ) # Copy the static BIDS files. - copy_bids_static_files(import_env) + copy_bids_static_files(importer) # Get the BIDS event dictionary. @@ -97,7 +100,7 @@ def import_bids_dataset(env: Env, args: Args, legacy_db: Database): else: _, dataset_tag_dict = import_bids_root_event_dict_file( env, - import_env, + importer, single_project, bids.event_dict_file, ) @@ -105,23 +108,34 @@ def import_bids_dataset(env: Env, args: Args, legacy_db: Database): # Copy the `participants.tsv` file rows. if bids.participants_file is not None: - loris_participants_path = get_loris_bids_root_file_path(import_env, bids.participants_file.path) - copy_bids_participants_file(import_env, bids.participants_file, loris_participants_path) + loris_participants_path = get_loris_bids_root_file_path(importer, bids.participants_file.path) + copy_bids_participants_file(importer, bids.participants_file, loris_participants_path) # Process each session directory. for bids_session in bids.sessions: - import_bids_session(env, import_env, args, bids_session, dataset_tag_dict, legacy_db) + import_bids_session(env, importer, bids_session, dataset_tag_dict, legacy_db) + + # Run custom importers. + + for entry_point in entry_points(group='loris-bids-importer.loaders'): + print(f"Loading importer '{entry_point.name}'") + try: + custom_importer = entry_point.load() + except Exception as exception: + log_error(env, f"Error loading importer '{entry_point.name}'. Skipping. Full error:\n{exception}") + continue + + custom_importer(env, importer, bids) # Print import summary. - print_bids_import_summary(env, import_env) + print_bids_import_summary(env, importer) def import_bids_session( env: Env, - import_env: BidsImportEnv, - args: Args, + importer: BidsImporter, bids_session: BidsSessionReader, dataset_tag_dict: dict[Any, Any], legacy_db: Database, @@ -155,8 +169,8 @@ def import_bids_session( try: # Read the scans.tsv property to raise an exception if the file is incorrect. if bids_session.scans_file is not None: - loris_scans_path = get_loris_scans_path(import_env, bids_session.scans_file, session) - copy_bids_scans_file(import_env, bids_session.scans_file, loris_scans_path) + loris_scans_path = get_loris_scans_path(importer, bids_session.scans_file, session) + copy_bids_scans_file(importer, bids_session.scans_file, loris_scans_path) except Exception as exception: log_warning( env, @@ -166,13 +180,12 @@ def import_bids_session( # Process each data type directory. for data_type in bids_session.data_types: - import_bids_data_type(env, import_env, args, session, data_type, dataset_tag_dict, legacy_db) + import_bids_data_type(env, importer, session, data_type, dataset_tag_dict, legacy_db) def import_bids_data_type( env: Env, - import_env: BidsImportEnv, - args: Args, + importer: BidsImporter, session: DbSession, data_type: BidsDataTypeReader, dataset_tag_dict: dict[Any, Any], @@ -189,15 +202,16 @@ def import_bids_data_type( match data_type: case BidsMriDataTypeReader(): - import_bids_mri_data_type(env, import_env, session, data_type) + import_bids_mri_data_type(env, importer, session, data_type) + case BidsMegDataTypeReader(): + import_bids_meg_data_type(env, importer, session, data_type) case BidsDataTypeReader(): - import_bids_eeg_data_type_files(env, import_env, args, session, data_type, dataset_tag_dict, legacy_db) + import_bids_eeg_data_type_files(env, importer, session, data_type, dataset_tag_dict, legacy_db) def import_bids_eeg_data_type_files( env: Env, - import_env: BidsImportEnv, - args: Args, + importer: BidsImporter, session: DbSession, data_type: BidsDataTypeReader, dataset_tag_dict: dict[Any, Any], @@ -210,13 +224,12 @@ def import_bids_eeg_data_type_files( try: Eeg( env = env, - import_env = import_env, + importer = importer, bids_layout = data_type.session.subject.dataset.layout, bids_info = data_type.info, db = legacy_db, session = session, dataset_tag_dict = dataset_tag_dict, - dataset_type = args.type, ) except Exception as exception: log_error( @@ -227,4 +240,4 @@ def import_bids_eeg_data_type_files( "Skipping." ) ) - import_env.failed_acquisitions_count += 1 + importer.failed_acquisitions_count += 1 diff --git a/python/loris_bids_importer/src/loris_bids_importer/meg/ctf.py b/python/loris_bids_importer/src/loris_bids_importer/meg/ctf.py new file mode 100644 index 000000000..513f41ef5 --- /dev/null +++ b/python/loris_bids_importer/src/loris_bids_importer/meg/ctf.py @@ -0,0 +1,209 @@ +from pathlib import Path + +from lib.config import ( + get_data_dir_path_config, + get_ephys_archive_dir_path_config, + get_ephys_visualization_enabled_config, +) +from lib.db.models.meg_ctf_head_shape_file import DbMegCtfHeadShapeFile +from lib.db.models.session import DbSession +from lib.db.queries.hed_schema_node import get_all_hed_schema_nodes +from lib.db.queries.meg_ctf_head_shape import try_get_meg_ctf_head_shape_file_with_path +from lib.db.queries.physio_file import try_get_physio_file_with_path +from lib.env import Env +from lib.logging import log, log_warning +from lib.physio.chunking import create_physio_channels_chunks +from lib.physio.events import EventDictFileSource +from lib.physio.file import insert_physio_file +from lib.physio.parameters import insert_physio_file_parameter +from loris_bids_utils.info import BidsAcquisitionInfo +from loris_bids_utils.meg.acquisition import MegAcquisition +from loris_bids_utils.meg.reader import BidsMegDataTypeReader +from loris_utils.error import group_errors_tuple +from loris_utils.path import add_path_extension + +from loris_bids_importer.acquisitions import import_bids_acquisitions +from loris_bids_importer.channels import insert_bids_channels_file +from loris_bids_importer.copy_files import copy_loris_bids_file, get_loris_bids_file_path +from loris_bids_importer.events import insert_bids_event_dict_file, insert_bids_events_file +from loris_bids_importer.file_type import get_check_bids_imaging_file_type +from loris_bids_importer.importer import BidsImporter +from loris_bids_importer.meg.ctf_head_shape import insert_head_shape_file +from loris_bids_importer.physio import ( + get_check_bids_physio_file_hash, + get_check_bids_physio_modality, + get_check_bids_physio_output_type, +) + + +def import_bids_meg_data_type( + env: Env, + importer: BidsImporter, + session: DbSession, + data_type: BidsMegDataTypeReader, +): + if data_type.head_shape_file is not None: + head_shape_file_path = get_loris_bids_file_path( + importer, + session, + data_type.name, + data_type.head_shape_file.path, + ) + + head_shape_file = try_get_meg_ctf_head_shape_file_with_path(env.db, head_shape_file_path) + if head_shape_file is None: + head_shape_file = insert_head_shape_file(env, data_type.head_shape_file, head_shape_file_path) + copy_loris_bids_file(importer, data_type.head_shape_file.path, head_shape_file_path) + else: + head_shape_file = None + + import_bids_acquisitions( + env, + importer, + data_type.acquisitions, + lambda acquisition, bids_info: import_bids_meg_acquisition( + env, + importer, + session, + acquisition, + bids_info, + head_shape_file, + ), + ) + + +def import_bids_meg_acquisition( + env: Env, + importer: BidsImporter, + session: DbSession, + acquisition: MegAcquisition, + bids_info: BidsAcquisitionInfo, + head_shape_file: DbMegCtfHeadShapeFile | None, +): + # MEG CTF directories should not be renamed, + if importer.loris_bids_path is not None: + loris_file_path = ( + importer.loris_bids_path + / f'sub-{session.candidate.psc_id}' + / f'ses-{session.visit_label}' + / 'meg' + / acquisition.ctf_path.name + ) + else: + loris_file_path = acquisition.ctf_path.relative_to(importer.data_dir_path) + + loris_file = try_get_physio_file_with_path(env.db, loris_file_path) + if loris_file is not None: + log(env, f"File '{loris_file_path}' is already registered in LORIS. Skipping.") + importer.files_dict[acquisition.ctf_path.relative_to(importer.args.source_bids_path)] = ( + loris_file_path.relative_to(importer.loris_bids_path) + if importer.loris_bids_path is not None + else loris_file_path.relative_to(importer.args.source_bids_path) + ) + importer.ignored_acquisitions_count += 1 + return + + modality, output_type, file_type, file_hash = group_errors_tuple( + f"Error while checking database information for MEG acquisition '{bids_info.name}'.", + lambda: get_check_bids_physio_modality(env, bids_info.data_type), + lambda: get_check_bids_physio_output_type(env, importer.args.type or 'raw'), + lambda: get_check_bids_imaging_file_type(env, 'ctf'), + lambda: get_check_bids_physio_file_hash(env, acquisition.ctf_path), + ) + + # The files to copy to LORIS, with the source path on the left and the LORIS path on the right. + files_to_copy: list[tuple[Path, Path]] = [] + + files_to_copy.append((acquisition.ctf_path, loris_file_path)) + + check_bids_meg_metadata_files(env, acquisition, bids_info) + + physio_file = insert_physio_file( + env, + session, + loris_file_path, + file_type, + modality, + output_type, + bids_info.scan_row.get_acquisition_time() if bids_info.scan_row is not None else None, + head_shape_file, + ) + + insert_physio_file_parameter(env, physio_file, 'physiological_file_blake2b_hash', file_hash) + for name, value in acquisition.sidecar_file.data.items(): + insert_physio_file_parameter(env, physio_file, name, value) + + if acquisition.events_file is not None: + hed_union = get_all_hed_schema_nodes(env.db) + + loris_events_file_path = get_loris_bids_file_path( + importer, session, bids_info.data_type, acquisition.events_file.path + ) + + insert_bids_events_file(env, physio_file, acquisition.events_file, loris_events_file_path, {}, {}, hed_union) + files_to_copy.append((acquisition.events_file.path, loris_events_file_path)) + if acquisition.events_file.dictionary is not None: + loris_event_dict_file_path = get_loris_bids_file_path( + importer, session, bids_info.data_type, acquisition.events_file.dictionary.path + ) + + insert_bids_event_dict_file( + env, + EventDictFileSource.from_file(physio_file), + acquisition.events_file.dictionary, + loris_event_dict_file_path, + ) + + files_to_copy.append((acquisition.events_file.dictionary.path, loris_event_dict_file_path)) + + if acquisition.channels_file is not None: + insert_bids_channels_file(env, importer, physio_file, session, bids_info, acquisition.channels_file) + loris_channels_file_path = get_loris_bids_file_path( + importer, session, bids_info.data_type, acquisition.channels_file.path + ) + files_to_copy.append((acquisition.channels_file.path, loris_channels_file_path)) + + for source_path, destination_path in files_to_copy: + copy_loris_bids_file(importer, source_path, destination_path) + + env.db.commit() + + log(env, f"MEG file successfully imported with ID: {physio_file.id}.") + + if get_ephys_visualization_enabled_config(env): + log(env, "Creating visualization chunks...") + create_physio_channels_chunks(env, physio_file) + + env.db.commit() + + +def check_bids_meg_metadata_files(env: Env, acquisition: MegAcquisition, bids_info: BidsAcquisitionInfo): + """ + Check for the presence of BIDS metadata files for the BIDS MEG acquisition and warn the user if + that is not the case. + """ + + if acquisition.channels_file is None: + log_warning(env, f"No channels file found for acquisition '{bids_info.name}'.") + + if acquisition.events_file is None: + log_warning(env, f"No events file found for acquisition '{bids_info.name}'.") + + if acquisition.events_file is not None and acquisition.events_file.dictionary is not None: + log_warning(env, f"No events dictionary file found for acquisition '{bids_info.name}'.") + + +def get_ctf_archive_path(env: Env, loris_ctf_path: Path) -> Path: + """ + Get the path of a CTF archive. + """ + + archive_rel_path = add_path_extension(loris_ctf_path, 'tgz') + archive_dir_path = get_ephys_archive_dir_path_config(env) + if archive_dir_path is not None: + data_dir_path = get_data_dir_path_config(env) + archive_path = archive_dir_path / 'ctf' / archive_rel_path.name + archive_path.parent.mkdir(exist_ok=True, parents=True) + return (archive_path).relative_to(data_dir_path) + else: + return archive_rel_path diff --git a/python/loris_bids_importer/src/loris_bids_importer/meg/ctf_head_shape.py b/python/loris_bids_importer/src/loris_bids_importer/meg/ctf_head_shape.py new file mode 100644 index 000000000..60ef8371a --- /dev/null +++ b/python/loris_bids_importer/src/loris_bids_importer/meg/ctf_head_shape.py @@ -0,0 +1,40 @@ +from decimal import Decimal +from pathlib import Path + +from lib.db.models.meg_ctf_head_shape_file import DbMegCtfHeadShapeFile +from lib.db.models.meg_ctf_head_shape_point import DbMegCtfHeadShapePoint +from lib.env import Env +from loris_bids_utils.meg.head_shape import MegCtfHeadShapeFile +from loris_utils.crypto import compute_file_blake2b_hash + + +def insert_head_shape_file( + env: Env, + head_shape_file: MegCtfHeadShapeFile, + loris_head_shape_file_path: Path, +) -> DbMegCtfHeadShapeFile: + """ + Insert a MEG CTF head shape file into the LORIS database. + """ + + blake2b_hash = compute_file_blake2b_hash(head_shape_file.path) + + db_head_shape_file = DbMegCtfHeadShapeFile( + path = loris_head_shape_file_path, + blake2b_hash = blake2b_hash, + ) + + env.db.add(db_head_shape_file) + env.db.flush() + + for name, point in head_shape_file.points.items(): + env.db.add(DbMegCtfHeadShapePoint( + file_id = db_head_shape_file.id, + name = name, + x = Decimal(point.x), + y = Decimal(point.y), + z = Decimal(point.z), + )) + + env.db.flush() + return db_head_shape_file diff --git a/python/loris_bids_importer/src/loris_bids_importer/mri/main.py b/python/loris_bids_importer/src/loris_bids_importer/mri/main.py index 829b6988a..4cab5ed58 100644 --- a/python/loris_bids_importer/src/loris_bids_importer/mri/main.py +++ b/python/loris_bids_importer/src/loris_bids_importer/mri/main.py @@ -20,8 +20,8 @@ from loris_bids_importer.acquisitions import import_bids_acquisitions from loris_bids_importer.copy_files import copy_loris_bids_file, get_loris_bids_file_path -from loris_bids_importer.env import BidsImportEnv from loris_bids_importer.file_type import get_check_bids_imaging_file_type_from_extension +from loris_bids_importer.importer import BidsImporter from loris_bids_importer.mri.sidecar import add_bids_mri_sidecar_file_parameters from loris_bids_importer.scans import add_bids_scans_file_parameters @@ -44,7 +44,7 @@ def import_bids_mri_data_type( env: Env, - import_env: BidsImportEnv, + import_env: BidsImporter, session: DbSession, data_type: BidsMriDataTypeReader, ): @@ -68,7 +68,7 @@ def import_bids_mri_data_type( def import_bids_mri_acquisition( env: Env, - import_env: BidsImportEnv, + import_env: BidsImporter, session: DbSession, acquisition: MriAcquisition, bids_info: BidsAcquisitionInfo, @@ -139,7 +139,7 @@ def import_bids_mri_acquisition( file_parameters['file_blake2b_hash'] = file_hash if bids_info.scans_file is not None and bids_info.scan_row is not None: - add_bids_scans_file_parameters(bids_info.scans_file, bids_info.scan_row, file_parameters) + add_bids_scans_file_parameters(import_env, session, bids_info.scans_file, bids_info.scan_row, file_parameters) for aux_file_type, aux_file_path in aux_file_paths: aux_file_hash = compute_file_blake2b_hash(aux_file_path) diff --git a/python/loris_bids_importer/src/loris_bids_importer/physio.py b/python/loris_bids_importer/src/loris_bids_importer/physio.py index c85c6f153..d80b13c95 100644 --- a/python/loris_bids_importer/src/loris_bids_importer/physio.py +++ b/python/loris_bids_importer/src/loris_bids_importer/physio.py @@ -5,7 +5,7 @@ from lib.db.queries.physio import try_get_physio_modality_with_name, try_get_physio_output_type_with_name from lib.db.queries.physio_file import try_get_physio_file_with_hash from lib.env import Env -from loris_utils.crypto import compute_file_blake2b_hash +from loris_utils.crypto import compute_directory_blake2b_hash, compute_file_blake2b_hash def get_check_bids_physio_modality(env: Env, data_type_name: str) -> DbPhysioModality: @@ -40,7 +40,10 @@ def get_check_bids_physio_file_hash(env: Env, file_path: Path) -> str: registered in the database. """ - file_hash = compute_file_blake2b_hash(file_path) + if file_path.is_dir(): + file_hash = compute_directory_blake2b_hash(file_path) + else: + file_hash = compute_file_blake2b_hash(file_path) file = try_get_physio_file_with_hash(env.db, file_hash) if file is not None: diff --git a/python/loris_bids_importer/src/loris_bids_importer/print.py b/python/loris_bids_importer/src/loris_bids_importer/print.py index 8d82be18d..8e87bbdd7 100644 --- a/python/loris_bids_importer/src/loris_bids_importer/print.py +++ b/python/loris_bids_importer/src/loris_bids_importer/print.py @@ -2,7 +2,7 @@ from lib.logging import log from loris_bids_utils.reader import BidsDatasetReader -from loris_bids_importer.env import BidsImportEnv +from loris_bids_importer.importer import BidsImporter def print_bids_info(env: Env, bids: BidsDatasetReader): @@ -23,7 +23,7 @@ def print_bids_info(env: Env, bids: BidsDatasetReader): log(env, f"- {data_type_name}") -def print_bids_import_summary(env: Env, import_env: BidsImportEnv): +def print_bids_import_summary(env: Env, import_env: BidsImporter): """ Print a summary of this BIDS import process. """ diff --git a/python/loris_bids_importer/src/loris_bids_importer/scans.py b/python/loris_bids_importer/src/loris_bids_importer/scans.py index 2523560d6..fc39fca0d 100644 --- a/python/loris_bids_importer/src/loris_bids_importer/scans.py +++ b/python/loris_bids_importer/src/loris_bids_importer/scans.py @@ -1,10 +1,16 @@ from typing import Any +from lib.db.models.session import DbSession from loris_bids_utils.files.scans import BidsScansTsvFile, BidsScanTsvRow from loris_utils.crypto import compute_file_blake2b_hash +from loris_bids_importer.copy_files import get_loris_scans_path +from loris_bids_importer.env import BidsImportEnv + def add_bids_scans_file_parameters( + import_env: BidsImportEnv, + session: DbSession, scans_file: BidsScansTsvFile, scan_row: BidsScanTsvRow, file_parameters: dict[str, Any], @@ -14,7 +20,7 @@ def add_bids_scans_file_parameters( dictionary. """ - file_parameters['scan_acquisition_time'] = scan_row.get_acquisition_time() - file_parameters['age_at_scan'] = scan_row.get_age_at_scan() - file_parameters['scans_tsv_file'] = scans_file.path - file_parameters['scans_tsv_file_bake2hash'] = compute_file_blake2b_hash(scans_file.path) + file_parameters['scan_acquisition_time'] = scan_row.get_acquisition_time() + file_parameters['age_at_scan'] = scan_row.get_age_at_scan() + file_parameters['scans_tsv_file'] = get_loris_scans_path(import_env, scans_file, session) + file_parameters['scans_tsv_file_blake2hash'] = compute_file_blake2b_hash(scans_file.path) diff --git a/python/loris_bids_importer/src/loris_bids_importer/scripts/import_bids_dataset.py b/python/loris_bids_importer/src/loris_bids_importer/scripts/import_bids_dataset.py index 4a3cccca8..fb7262a52 100755 --- a/python/loris_bids_importer/src/loris_bids_importer/scripts/import_bids_dataset.py +++ b/python/loris_bids_importer/src/loris_bids_importer/scripts/import_bids_dataset.py @@ -7,12 +7,12 @@ from lib.logging import log_error_exit from lib.lorisgetopt import LorisGetOpt -from loris_bids_importer.args import Args +from loris_bids_importer.importer import BidsImporterArgs from loris_bids_importer.main import import_bids_dataset -def pack_args(options_dict: dict[str, Any]) -> Args: - return Args( +def pack_args(options_dict: dict[str, Any]) -> BidsImporterArgs: + return BidsImporterArgs( source_bids_path = Path(options_dict['directory']['value']), type = options_dict['type']['value'], bids_validation = not options_dict['no-bids-validation']['value'], diff --git a/python/loris_bids_utils/src/loris_bids_utils/eeg/reader.py b/python/loris_bids_utils/src/loris_bids_utils/eeg/reader.py new file mode 100644 index 000000000..6a1d3ad9b --- /dev/null +++ b/python/loris_bids_utils/src/loris_bids_utils/eeg/reader.py @@ -0,0 +1,11 @@ +from collections.abc import Sequence +from functools import cached_property + +from loris_bids_utils.info import BidsAcquisitionInfo +from loris_bids_utils.reader import BidsDataTypeReader + + +class BidsEegDataTypeReader(BidsDataTypeReader): + @cached_property + def acquisitions(self) -> Sequence[BidsAcquisitionInfo]: + return [] diff --git a/python/loris_bids_utils/src/loris_bids_utils/files/scans.py b/python/loris_bids_utils/src/loris_bids_utils/files/scans.py index e26e4b652..26f32f562 100644 --- a/python/loris_bids_utils/src/loris_bids_utils/files/scans.py +++ b/python/loris_bids_utils/src/loris_bids_utils/files/scans.py @@ -63,7 +63,10 @@ def get_row(self, file_path: Path) -> BidsScanTsvRow | None: Get the row corresponding to the given file path. """ - return find(self.rows, lambda row: file_path.name == row.data['filename']) + # According to the specification, the 'filename' column is the path of the acquisition file + # relative to the directory in which the scans.tsv file is located. + relative_path = file_path.relative_to(self.path.parent) + return find(self.rows, lambda row: str(relative_path) == row.data['filename']) def set_row(self, scan: BidsScanTsvRow): """ diff --git a/python/loris_bids_utils/src/loris_bids_utils/meg/acquisition.py b/python/loris_bids_utils/src/loris_bids_utils/meg/acquisition.py new file mode 100644 index 000000000..90f66b8e9 --- /dev/null +++ b/python/loris_bids_utils/src/loris_bids_utils/meg/acquisition.py @@ -0,0 +1,37 @@ + +import re +from pathlib import Path + +from loris_utils.path import remove_path_extension + +from loris_bids_utils.eeg.channels import BidsEegChannelsTsvFile +from loris_bids_utils.files.events import BidsEventsTsvFile +from loris_bids_utils.meg.head_shape import MegCtfHeadShapeFile +from loris_bids_utils.meg.sidecar import BidsMegSidecarJsonFile + + +class MegAcquisition: + ctf_path: Path + sidecar_file: BidsMegSidecarJsonFile + channels_file: BidsEegChannelsTsvFile | None + events_file: BidsEventsTsvFile | None + head_shape_file: MegCtfHeadShapeFile | None + + def __init__(self, ctf_path: Path, head_shape_file: MegCtfHeadShapeFile | None): + self.ctf_path = ctf_path + + path = remove_path_extension(ctf_path) + + sidecar_path = path.with_suffix('.json') + if not sidecar_path.exists(): + raise Exception("No MEG JSON sidecar file.") + + self.sidecar_file = BidsMegSidecarJsonFile(sidecar_path) + + channels_path = path.parent / re.sub(r'_meg$', '_channels.tsv', path.name) + self.channels_file = BidsEegChannelsTsvFile(channels_path) if channels_path.exists() else None + + events_path = path.parent / re.sub(r'_meg$', '_events.tsv', path.name) + self.events_file = BidsEventsTsvFile(events_path) if events_path.exists() else None + + self.head_shape_file = head_shape_file diff --git a/python/loris_bids_utils/src/loris_bids_utils/meg/head_shape.py b/python/loris_bids_utils/src/loris_bids_utils/meg/head_shape.py new file mode 100644 index 000000000..3b0584422 --- /dev/null +++ b/python/loris_bids_utils/src/loris_bids_utils/meg/head_shape.py @@ -0,0 +1,110 @@ +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +VecF = np.ndarray[tuple[int], np.dtype[np.float64]] + + +@dataclass +class MegCtfHeadShapePoint: + """ + A point in a MEG CTF `headshape.pos` file. + """ + + x: float + y: float + z: float + + def scale(self, factor: float) -> 'MegCtfHeadShapePoint': + """ + Scale the point coordinates by a factor. Notably useful to convert the head shape point from + one unit to another. + """ + + return MegCtfHeadShapePoint( + x = self.x * factor, + y = self.y * factor, + z = self.z * factor, + ) + + def to_numpy(self) -> VecF: + """ + Convert the point to a numpy array. + """ + + return np.array([self.x, self.y, self.z], dtype=np.float64) + + +@dataclass +class MegCtfHeadShapeFile: + """ + A MEG CTF `headshape.pos` file. + """ + + path: Path + """ + The path of this head shape file. + """ + + points: dict[str, MegCtfHeadShapePoint] + """ + The points of this head shape file. + """ + + @staticmethod + def read(path: Path) -> 'MegCtfHeadShapeFile': + """ + Read and parse a MEG CTF head shape file. + """ + + with path.open() as file: + lines = file.readlines() + + points: dict[str, MegCtfHeadShapePoint] = {} + # The first line simply gives the number of points. + for line in lines[1:]: + parts = line.split() + # The first column contains the sensor name or index. + # The second column may or may not be empty. + # The last three columns contain the point coordimates. + points[parts[0]] = MegCtfHeadShapePoint(float(parts[-3]), float(parts[-2]), float(parts[-1])) + + return MegCtfHeadShapeFile( + path = path, + points = points, + ) + + def scale(self, factor: float) -> 'MegCtfHeadShapeFile': + """ + Create a new head shape file with all points scaled by a factor. + """ + + return MegCtfHeadShapeFile( + path = self.path, + points = {name: point.scale(factor) for name, point in self.points.items()}, + ) + + @property + def nasion(self) -> MegCtfHeadShapePoint | None: + """ + Get the nasion fiducial point. + """ + + return self.points.get('NAS') or self.points.get('Nasion') + + @property + def lpa(self) -> MegCtfHeadShapePoint | None: + """ + Get the LPA fiducial point. + """ + + return self.points.get('LPA') + + @property + def rpa(self) -> MegCtfHeadShapePoint | None: + """ + Get the RPA fiducial point. + """ + + return self.points.get('RPA') diff --git a/python/loris_bids_utils/src/loris_bids_utils/meg/reader.py b/python/loris_bids_utils/src/loris_bids_utils/meg/reader.py new file mode 100644 index 000000000..683c35fba --- /dev/null +++ b/python/loris_bids_utils/src/loris_bids_utils/meg/reader.py @@ -0,0 +1,75 @@ +import re +from collections.abc import Iterator +from dataclasses import dataclass +from functools import cached_property +from pathlib import Path + +from loris_bids_utils.info import BidsAcquisitionInfo +from loris_bids_utils.meg.acquisition import MegAcquisition +from loris_bids_utils.meg.head_shape import MegCtfHeadShapeFile +from loris_bids_utils.reader import BidsDataTypeReader +from loris_bids_utils.utils import get_pybids_file_path, try_get_pybids_value + + +@dataclass +class BidsMegDataTypeReader(BidsDataTypeReader): + path: Path + + @cached_property + def acquisitions(self) -> list[tuple[MegAcquisition, BidsAcquisitionInfo]]: + """ + The MEG acquisitions found in the MEG data type. + """ + + acquisitions: list[tuple[MegAcquisition, BidsAcquisitionInfo]] = [] + for ctf_name in find_dir_meg_acquisition_names(self.path): + scan_row = self.session.scans_file.get_row(self.path / ctf_name) \ + if self.session.scans_file is not None else None + + acquisition = MegAcquisition(self.path / ctf_name, self.head_shape_file) + + info = BidsAcquisitionInfo( + subject = self.session.subject.label, + participant_row = self.session.subject.participant_row, + session = self.session.label, + scans_file = self.session.scans_file, + data_type = self.name, + scan_row = scan_row, + name = ctf_name, + suffix = 'meg', + ) + + acquisitions.append((acquisition, info)) + + return acquisitions + + @cached_property + def head_shape_file(self) -> MegCtfHeadShapeFile | None: + """ + The MEG CTF file of this acquisition if it exists. + """ + + head_shape_file = try_get_pybids_value( + self.session.subject.dataset.layout, + subject=self.session.subject.label, + session=self.session.label, + datatype=self.name, + suffix='headshape', + extension='.pos', + ) + + if head_shape_file is None: + return None + + return MegCtfHeadShapeFile.read(get_pybids_file_path(head_shape_file)) + + +def find_dir_meg_acquisition_names(dir_path: Path) -> Iterator[str]: + """ + Iterate over the Path objects of the NIfTI files found in a directory. + """ + + for item_path in dir_path.iterdir(): + name_match = re.search(r'.+_meg\.ds$', item_path.name) + if name_match is not None: + yield name_match.group(0) diff --git a/python/loris_bids_utils/src/loris_bids_utils/meg/sidecar.py b/python/loris_bids_utils/src/loris_bids_utils/meg/sidecar.py new file mode 100644 index 000000000..706054855 --- /dev/null +++ b/python/loris_bids_utils/src/loris_bids_utils/meg/sidecar.py @@ -0,0 +1,12 @@ +from loris_bids_utils.json import BidsJsonFile + + +class BidsMegSidecarJsonFile(BidsJsonFile): + """ + Class representing a BIDS EEG or iEEG sidecar JSON file. + + Documentation: + - https://bids-specification.readthedocs.io/en/stable/modality-specific-files/magnetoencephalography.html#sidecar-json-_megjson + """ + + pass diff --git a/python/loris_bids_utils/src/loris_bids_utils/reader.py b/python/loris_bids_utils/src/loris_bids_utils/reader.py index 3d010b3bb..14c6f05d6 100644 --- a/python/loris_bids_utils/src/loris_bids_utils/reader.py +++ b/python/loris_bids_utils/src/loris_bids_utils/reader.py @@ -16,6 +16,7 @@ # Circular imports if TYPE_CHECKING: + from loris_bids_utils.meg.reader import BidsMegDataTypeReader from loris_bids_utils.mri.reader import BidsMriDataTypeReader PYBIDS_IGNORE = ['.git', 'code/', 'log/', 'sourcedata/'] @@ -297,13 +298,38 @@ def eeg_data_types(self) -> list['BidsDataTypeReader']: ) ] + @cached_property + def meg_data_types(self) -> list['BidsMegDataTypeReader']: + """ + Get the MEG data type directory readers of this session. + """ + + from loris_bids_utils.meg.reader import BidsMegDataTypeReader + + return [ + BidsMegDataTypeReader( + session=self, + name=data_type, # type: ignore + path=( + self.subject.dataset.path + / f'sub-{self.subject.label}' + / (f'ses-{self.label}' if self.label is not None else '') + / data_type # type: ignore + ), + ) for data_type in self.subject.dataset.layout.get_datatypes( # type: ignore + subject=self.subject.label, + session=self.label, + datatype=['meg'], + ) + ] + @cached_property def data_types(self) -> Sequence['BidsDataTypeReader']: """ Get all the data type directory readers of this session. """ - return self.eeg_data_types + self.mri_data_types + return self.eeg_data_types + self.meg_data_types + self.mri_data_types @cached_property def info(self) -> BidsSessionInfo: diff --git a/python/loris_bids_utils/src/loris_bids_utils/tsv.py b/python/loris_bids_utils/src/loris_bids_utils/tsv.py index c38cf1e2b..e15ae33a6 100644 --- a/python/loris_bids_utils/src/loris_bids_utils/tsv.py +++ b/python/loris_bids_utils/src/loris_bids_utils/tsv.py @@ -3,6 +3,9 @@ from typing import Generic, TypeVar from loris_utils.parse import nullify_empty_string +from loris_utils.path import replace_path_extension + +from loris_bids_utils.json import BidsJsonFile class BidsTsvRow: @@ -27,12 +30,19 @@ class BidsTsvFile(Generic[T]): """ path: Path + dictionary: BidsJsonFile | None rows: list[T] def __init__(self, model: type[T], path: Path): self.path = path self.rows = [] + dictionary_path = replace_path_extension(self.path, 'json') + if dictionary_path.exists(): + self.dictionary = BidsJsonFile(dictionary_path) + else: + self.dictionary = None + # The 'utf-8-sig' encoding is used to support some datasets where metadata files may contain # a byte-order mark (BOM). with open(self.path, encoding='utf-8-sig') as file: diff --git a/python/loris_bids_utils/src/loris_bids_utils/utils.py b/python/loris_bids_utils/src/loris_bids_utils/utils.py index 9d633f697..d3aaed347 100644 --- a/python/loris_bids_utils/src/loris_bids_utils/utils.py +++ b/python/loris_bids_utils/src/loris_bids_utils/utils.py @@ -13,7 +13,7 @@ def try_get_pybids_value(layout: BIDSLayout, **args: Any) -> Any | None: values are found. """ - match layout.get(args): # type: ignore + match layout.get(**args): # type: ignore case []: return None case [value]: # type: ignore diff --git a/python/loris_ephys_chunker/pyproject.toml b/python/loris_ephys_chunker/pyproject.toml index d29278854..b4f41cb25 100644 --- a/python/loris_ephys_chunker/pyproject.toml +++ b/python/loris_ephys_chunker/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "numpy", "protobuf>=3.0.0", "scipy", + "loris-server", ] [project.scripts] diff --git a/python/loris_ephys_chunker/src/loris_ephys_chunker/scripts/ctf_to_chunks.py b/python/loris_ephys_chunker/src/loris_ephys_chunker/scripts/ctf_to_chunks.py index ab6887c0f..3fbca0c52 100755 --- a/python/loris_ephys_chunker/src/loris_ephys_chunker/scripts/ctf_to_chunks.py +++ b/python/loris_ephys_chunker/src/loris_ephys_chunker/scripts/ctf_to_chunks.py @@ -12,15 +12,23 @@ def load_channels(path: Path) -> RawCTF: - return mne.io.read_raw_ctf( # type: ignore + raw = mne.io.read_raw_ctf( # type: ignore path, - preload=False, # CTF raw channel names can contain suffixes that causes them to mismatch their # corresponding `channels.tsv` entries, the following flag removes these suffixes. clean_names=True, verbose=False, ) + # Apply third-order software gradient compensation to remove environmental noise. + # CTF systems use reference sensors to measure ambient magnetic fields (building vibrations, + # distant equipment, etc.). This subtraction algorithm cancels this noise from the MEG + # channels. Grade 3 is the highest order and standard for analysis/visualization. + # Without this, raw channel values reflect environmental noise (millions of fT) + # instead of actual brain signals (tens to hundreds of fT). + raw.apply_gradient_compensation(3) # type: ignore + return raw + def main(): parser = argparse.ArgumentParser( diff --git a/python/loris_ephys_chunker/src/loris_ephys_chunker/scripts/eeglab_to_chunks.py b/python/loris_ephys_chunker/src/loris_ephys_chunker/scripts/eeglab_to_chunks.py index 65758cec2..d1512cc7f 100755 --- a/python/loris_ephys_chunker/src/loris_ephys_chunker/scripts/eeglab_to_chunks.py +++ b/python/loris_ephys_chunker/src/loris_ephys_chunker/scripts/eeglab_to_chunks.py @@ -39,7 +39,6 @@ def main(): eeg = mne_eeglab._check_load_mat(path, None) # type: ignore eeglab_info = mne_eeglab._get_info(eeg, eog=(), montage_units="auto") # type: ignore channel_names = cast(list[str], eeglab_info[0]['ch_names']) - if args.channel_index < 0: sys.exit("Channel index must be a positive integer") diff --git a/python/loris_ephys_visualizer_module/README.md b/python/loris_ephys_visualizer_module/README.md new file mode 100644 index 000000000..7c33ba0ac --- /dev/null +++ b/python/loris_ephys_visualizer_module/README.md @@ -0,0 +1,3 @@ +# LORIS electrophysiology visualizer module + +The LORIS electrophysiology visualizer module. diff --git a/python/loris_ephys_visualizer_module/pyproject.toml b/python/loris_ephys_visualizer_module/pyproject.toml new file mode 100644 index 000000000..742f6895d --- /dev/null +++ b/python/loris_ephys_visualizer_module/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "loris-ephys-visualizer-module" +version = "27.0.0" +description = "LORIS electrophysiology visualizer module" +readme = "README.md" +requires-python = ">= 3.11" +dependencies = [ + "fastapi", + "mne", + "mne-bids", + "numpy", + "uvicorn[standard]", +] + +[project.entry-points."loris-server.loaders"] +ephys-visualizer = "loris_ephys_visualizer_module.server:load" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/loris_ephys_visualizer_module"] + +[tool.ruff] +extend = "../../pyproject.toml" +src = ["src"] diff --git a/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/dependencies.py b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/dependencies.py new file mode 100644 index 000000000..21bd0aea7 --- /dev/null +++ b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/dependencies.py @@ -0,0 +1,27 @@ + +from typing import Annotated + +from fastapi import Depends, HTTPException +from lib.db.models.physio_file import DbPhysioFile +from lib.db.queries.physio_file import try_get_physio_file_with_id +from lib.user import can_user_access_session +from loris_server.dependencies import EnvDep, UserDep + + +def get_physio_file(env: EnvDep, user: UserDep, physio_file_id: int) -> DbPhysioFile: + """ + Get a physiological file from the query parameters or raise an HTTP 404 error if that file + does not exist in LORIS or the user does not have the permissions to access it. + """ + + physio_file = try_get_physio_file_with_id(env.db, physio_file_id) + if physio_file is None: + raise HTTPException(status_code=404, detail="Electrophysiology file not found or not accessible.") + + if not can_user_access_session(env, user, physio_file.session): + raise HTTPException(status_code=404, detail="Electrophysiology file not found or not accessible.") + + return physio_file + + +PhysioFileDep = Annotated[DbPhysioFile, Depends(get_physio_file)] diff --git a/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/endpoints/meg_head_shape.py b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/endpoints/meg_head_shape.py new file mode 100644 index 000000000..a73631c1a --- /dev/null +++ b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/endpoints/meg_head_shape.py @@ -0,0 +1,135 @@ +from dataclasses import dataclass + +import mne.io +import numpy as np +from fastapi import HTTPException +from lib.config import get_data_dir_path_config +from lib.db.models.physio_file import DbPhysioFile +from lib.env import Env +from loris_bids_utils.meg.head_shape import MegCtfHeadShapeFile +from mne.io.ctf.ctf import RawCTF +from pydantic import BaseModel + + +class MegHeadShapePoint(BaseModel): + x: float + y: float + z: float + + +class MegHeadShapeResponse(BaseModel): + points: dict[str, MegHeadShapePoint] + + +def get_meg_head_shape(env: Env, physio_file: DbPhysioFile) -> MegHeadShapeResponse: + """ + Get head shape points aligned to the MEG sensor coordinate system. + """ + + if physio_file.type != 'ctf': + raise HTTPException(status_code=404, detail="Electrophysiology file is not an MEG file.") + if physio_file.head_shape_file is None: + raise HTTPException(status_code=404, detail="Headshape file not found.") + + data_dir_path = get_data_dir_path_config(env) + + # 1. Read the raw head shape (already in CTF head coordinates, cm → m) + head_shape_path = data_dir_path / physio_file.head_shape_file.path + head_shape_file = MegCtfHeadShapeFile.read(head_shape_path) + + # 2. Read the MEG data to get MNE's fiducial positions + raw_ctf = mne.io.read_raw_ctf(data_dir_path / physio_file.path) # type: ignore + + # 3. Align head shape to MNE's head coordinates + aligned_points = align_head_shape_to_mne(head_shape_file, raw_ctf) + + # 4. Return as response + response_points: dict[str, MegHeadShapePoint] = {} + for name, point in aligned_points.items(): + response_points[name] = MegHeadShapePoint( + x = float(point[0]), + y = float(point[1]), + z = float(point[2]), + ) + + return MegHeadShapeResponse(points=response_points) + + +VecF = np.ndarray[tuple[int], np.dtype[np.float64]] + + +@dataclass +class Fiducials: + nasion: VecF + lpa: VecF + rpa: VecF + + +def get_mne_raw_fiducials(raw: RawCTF) -> Fiducials: + positions = raw.get_montage().get_positions() # type: ignore + + return Fiducials( + nasion = positions['nasion'], # type: ignore + lpa = positions['lpa'], # type: ignore + rpa = positions['rpa'], # type: ignore + ) + + +def get_head_shape_fiducials(head_shape_file: MegCtfHeadShapeFile) -> Fiducials: + if head_shape_file.nasion is None or head_shape_file.lpa is None or head_shape_file.rpa is None: + raise Exception("Could not find head shape fiducial points.") + + return Fiducials( + nasion = head_shape_file.nasion.to_numpy(), + lpa = head_shape_file.lpa.to_numpy(), + rpa = head_shape_file.rpa.to_numpy(), + ) + + +def align_head_shape_to_mne(head_shape_file: MegCtfHeadShapeFile, raw_ctf: RawCTF) -> dict[str, VecF]: + """ + Align head shape points (in CTF head coordinates) to MNE's head coordinates using the three + cardinal fiducials: Nasion, LPA, RPA. + """ + + # Convert head shape points from centimeters to meters to match MNE units. + head_shape_file = head_shape_file.scale(1 / 100) + + head_shape_fiducials = get_head_shape_fiducials(head_shape_file) + mne_fiducials = get_mne_raw_fiducials(raw_ctf) + + # List fiducial points in the same order. + source_points = np.array([ + head_shape_fiducials.nasion, + head_shape_fiducials.lpa, + head_shape_fiducials.rpa, + ]) + + target_points = np.array([ + mne_fiducials.nasion, + mne_fiducials.lpa, + mne_fiducials.rpa, + ]) + + # Remove translation. + source_centroid = np.mean(source_points, axis=0) + target_centroid = np.mean(target_points, axis=0) + source_centered = source_points - source_centroid + target_centered = target_points - target_centroid + + # Compute rotation matrix using SVD (Kabsch algorithm). + h = source_centered.T @ target_centered + u, _, vt = np.linalg.svd(h) + r = vt.T @ u.T + + # Special reflection case. + if np.linalg.det(r) < 0: + vt[-1, :] *= -1 + r = vt.T @ u.T + + # Apply transformation to all points. + aligned_points: dict[str, VecF] = {} + for name, point in head_shape_file.points.items(): + aligned_points[name] = (r @ (point.to_numpy() - source_centroid)) + target_centroid + + return aligned_points diff --git a/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/endpoints/meg_sensors.py b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/endpoints/meg_sensors.py new file mode 100644 index 000000000..a6b302774 --- /dev/null +++ b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/endpoints/meg_sensors.py @@ -0,0 +1,89 @@ +from typing import Any, cast + +import mne.io +import numpy as np +import numpy.typing as npt +from fastapi import HTTPException +from lib.config import get_data_dir_path_config +from lib.db.models.physio_file import DbPhysioFile +from lib.env import Env +from mne.io.constants import FIFF +from pydantic import BaseModel + +from loris_ephys_visualizer_module.jsonize import jsonize + + +def get_ephys_unit_symbol(unit_code: int) -> str | None: + match unit_code: + case FIFF.FIFF_UNIT_V: # type: ignore + # Used by EEG electrodes. + return 'V' + case FIFF.FIFF_UNIT_SEC: # type: ignore + # Used by MEG system clock. + return 's' + case FIFF.FIFF_UNIT_T: # type: ignore + # Used by MEG magnetometers. + return 'T' + case FIFF.FIFF_UNIT_T_M: # type: ignore + # Used by MEG gradiometers. + return 'T/m' + case _: + return None + + +class MegSensorPoint(BaseModel): + x: float + y: float + z: float + unit: str | None + type: Any + + +class MegSensorsResponse(BaseModel): + sensors: dict[str, MegSensorPoint] + + +def get_meg_sensors(env: Env, physio_file: DbPhysioFile) -> MegSensorsResponse: + """ + Get the head MEG sensors of a LORIS MEG file. + """ + + if physio_file.type != 'ctf': + raise HTTPException(status_code=404, detail="Electrophysiology file is not an MEG file.") + + data_dir_path = get_data_dir_path_config(env) + + raw = mne.io.read_raw_ctf(data_dir_path / physio_file.path) # type: ignore + + # Get the transformation from the device to the head coordinates system. + dev_head_t = raw.info.get('dev_head_t') # type: ignore + if dev_head_t is None: + raise HTTPException(status_code=500, detail="No device-to-head transformation found in the CTF file.") + + # The transformation matrix is a 4x4 array. + transform = cast(npt.NDArray[np.float64], dev_head_t['trans']) + + sensors: dict[str, MegSensorPoint] = {} + for channel in raw.info["chs"]: # type: ignore + channel_loc = cast(list[float], channel['loc']) + + # Sensor position in device coordinates (meters) + device_pos = np.array([ + channel_loc[0], + channel_loc[1], + channel_loc[2], + 1.0 # Homogeneous coordinates + ]) + + # Transform to head coordinates + head_pos = transform @ device_pos + + sensors[channel['ch_name']] = MegSensorPoint( + x = float(head_pos[0]), + y = float(head_pos[1]), + z = float(head_pos[2]), + unit = get_ephys_unit_symbol(channel['unit']), # type: ignore + type = jsonize(channel), + ) + + return MegSensorsResponse(sensors=sensors) diff --git a/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/endpoints/topographic_map.py b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/endpoints/topographic_map.py new file mode 100644 index 000000000..3fdc584d8 --- /dev/null +++ b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/endpoints/topographic_map.py @@ -0,0 +1,97 @@ +from io import BytesIO +from typing import cast + +import matplotlib.pyplot as plt +import mne +import numpy as np +import numpy.typing as npt +from fastapi import HTTPException +from fastapi.responses import StreamingResponse +from lib.config import get_data_dir_path_config +from lib.db.models.physio_file import DbPhysioFile +from lib.env import Env +from mne.io import BaseRaw + + +def read_physio_file_mne_raw(env: Env, physio_file: DbPhysioFile) -> BaseRaw | None: + """ + Get the MNE raw object of a LORIS electrophysiology file if that file type is supported. + """ + + data_dir_path = get_data_dir_path_config(env) + + match physio_file.type: + case 'ctf': + raw = mne.io.read_raw_ctf(data_dir_path / physio_file.path) # type: ignore + # raw.pick('meg', exclude='ref_meg') + raw.pick_types(meg=True, ref_meg=False) + return raw + case 'edf': + raw = mne.io.read_raw_edf(data_dir_path / physio_file.path) # type: ignore + raw.set_montage('biosemi128') # type: ignore + raw.pick('eeg') # type: ignore + return raw + case 'set': + raw = mne.io.read_raw_eeglab(data_dir_path / physio_file.path) # type: ignore + raw.pick('eeg') # type: ignore + return raw + case _: + return None + + +def get_topographic_map( + env: Env, + physio_file: DbPhysioFile, + t_min: float | None, + t_max: float | None, + l_freq: float | None, + h_freq: float | None, +) -> StreamingResponse: + """ + Get the topographic map of a LORIS electrophysiology file. + """ + + raw = read_physio_file_mne_raw(env, physio_file) + if raw is None: + raise HTTPException(status_code=500, detail="Electrophysiology file type not supported.") + + # Crop the MNE raw according to the time window. + raw.crop(tmin=t_min if t_min is not None else 0.0, tmax=t_max) # type: ignore + + # Apply frequency filters if specified. + if l_freq is not None or h_freq is not None: + # Load the signal data for filtering. + raw.load_data() # type: ignore + + # Filter the signal data with the provided low and high pass. + raw.filter( # type: ignore + l_freq=l_freq, + h_freq=h_freq, + picks='all', + method='fir', + phase='zero', + verbose=False, + ) + + # Get the mean signal values of the channels over time. + data_raw = cast(npt.NDArray[np.float64], raw.get_data().mean(axis=1)) # type: ignore + + # Plot the topographic map. + figure, axes = plt.subplots() # type: ignore + _, _ = mne.viz.plot_topomap( # type: ignore + data_raw, + raw.info, # type: ignore + axes=axes, + show=False, + contours=0, + ) + + # Write the figure to a buffer for streaming. + buffer = BytesIO() + figure.savefig(buffer, format='png', dpi=150, bbox_inches='tight') # type: ignore + plt.close(figure) + + # Reset the stream position to the start of the buffer. + buffer.seek(0) + + return StreamingResponse(buffer, media_type='image/png') diff --git a/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/jsonize.py b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/jsonize.py new file mode 100644 index 000000000..08868c0cb --- /dev/null +++ b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/jsonize.py @@ -0,0 +1,71 @@ +import math +import uuid +from datetime import date, datetime +from decimal import Decimal +from typing import Any + +import numpy as np + +JsonPrimitive = str | int | float | bool | None +JsonValue = JsonPrimitive | dict[str, 'JsonValue'] | list['JsonValue'] + + +def jsonize(value: Any) -> JsonValue: + """ + Recursively convert a value to a JSON-like value. + """ + + if value is None or isinstance(value, (str, int, bool)): + return value + + # Handle float special cases + if isinstance(value, float): + if math.isinf(value) or math.isnan(value): + return str(value) + return value + + # Handle numpy types + if isinstance(value, np.ndarray): + if value.dtype.kind == 'f': # type: ignore + if np.any(np.isinf(value)) or np.any(np.isnan(value)): # type: ignore + return [ + str(x) + if (isinstance(x, float) and (math.isinf(x) or math.isnan(x))) else jsonize(x) + for x in value.tolist() + ] + + return value.tolist() + + if isinstance(value, np.integer): + return int(value) # type: ignore + + if isinstance(value, np.floating): + if np.isinf(value) or np.isnan(value): # type: ignore + return str(value) # type: ignore + + return float(value) # type: ignore + + if isinstance(value, np.bool_): + return bool(value) # type: ignore + + # Handle datetime/dates + if isinstance(value, (datetime, date)): + return value.isoformat() + + # Handle Decimal + if isinstance(value, Decimal): + return float(value) + + # Handle UUID + if isinstance(value, uuid.UUID): + return str(value) + + # Handle iterables (list, tuple, set) + if isinstance(value, (list, tuple, set)): + return [jsonize(item) for item in value] # type: ignore + + # Handle dictionaries + if isinstance(value, dict): + return {str(k): jsonize(v) for k, v in value.items()} # type: ignore + + raise Exception(value) diff --git a/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/server.py b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/server.py new file mode 100644 index 000000000..c7199cfc8 --- /dev/null +++ b/python/loris_ephys_visualizer_module/src/loris_ephys_visualizer_module/server.py @@ -0,0 +1,35 @@ +from fastapi import APIRouter, FastAPI +from loris_server.dependencies import EnvDep + +from loris_ephys_visualizer_module.dependencies import PhysioFileDep +from loris_ephys_visualizer_module.endpoints.meg_head_shape import MegHeadShapeResponse, get_meg_head_shape +from loris_ephys_visualizer_module.endpoints.meg_sensors import MegSensorsResponse, get_meg_sensors +from loris_ephys_visualizer_module.endpoints.topographic_map import get_topographic_map + +router = APIRouter(prefix='/ephys') + + +@router.get('/{physio_file_id}/topographic-map') +def topographic_map( + env: EnvDep, + physio_file: PhysioFileDep, + tmin: float | None = None, + tmax: float | None = None, + lfreq: float | None = None, + hfreq: float | None = None, +): + return get_topographic_map(env, physio_file, tmin, tmax, lfreq, hfreq) + + +@router.get('/{physio_file_id}/meg/sensors', response_model=MegSensorsResponse) +def meg_sensors(env: EnvDep, physio_file: PhysioFileDep): + return get_meg_sensors(env, physio_file) + + +@router.get('/{physio_file_id}/meg/headshape', response_model=MegHeadShapeResponse) +def meg_head_shape(env: EnvDep, physio_file: PhysioFileDep): + return get_meg_head_shape(env, physio_file) + + +def load(api: FastAPI): + return api.include_router(router) diff --git a/python/loris_meegqc_module/README.md b/python/loris_meegqc_module/README.md new file mode 100644 index 000000000..ae61081f4 --- /dev/null +++ b/python/loris_meegqc_module/README.md @@ -0,0 +1,22 @@ +# LORIS MEEGqc module + +## Description + +This module provides LORIS support for the [MEEGqc](https://ancplaboldenburg.github.io/megqc_documentation/) EEG/MEG quality control tool. + +## Installation + +This is an optional module not installed with LORIS by default. It can be installed using the following command from the root LORIS Python directory: + +```sh +pip install python/loris_module_meegqc +``` + +## Features + +Here are the features provided by this module: +- Import MEEGqc derivatives from a BIDS dataset. +- Serve MEEGqc endpoints for the LORIS electrophysiology browser. + +Here are the features not provided by this module yet: +- Run MEEGqc on imported data. diff --git a/python/loris_meegqc_module/pyproject.toml b/python/loris_meegqc_module/pyproject.toml new file mode 100644 index 000000000..b584c39e7 --- /dev/null +++ b/python/loris_meegqc_module/pyproject.toml @@ -0,0 +1,30 @@ +[project] +name = "loris-meegqc-module" +version = "27.0.0" +description = "The LORIS MEEGqc module." +readme = "README.md" +requires-python = ">= 3.11" +dependencies = [ + "fastapi", + "loris-bids-importer", + "loris-bids-utils", + "loris-server", + "loris-utils", +] + +[project.entry-points."loris-bids-importer.loaders"] +meegqc = "loris_meegqc_module.importer.main:import_meegqc_derivatives" + +[project.entry-points."loris-server.loaders"] +meegqc = "loris_meegqc_module.server.main:load" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/loris_meegqc_module"] + +[tool.ruff] +extend = "../../pyproject.toml" +src = ["src"] diff --git a/python/loris_meegqc_module/src/loris_meegqc_module/database/models/meegqc_file.py b/python/loris_meegqc_module/src/loris_meegqc_module/database/models/meegqc_file.py new file mode 100644 index 000000000..32e2728ed --- /dev/null +++ b/python/loris_meegqc_module/src/loris_meegqc_module/database/models/meegqc_file.py @@ -0,0 +1,47 @@ +from pathlib import Path + +from lib.db.base import Base +from lib.db.decorators.string_path import StringPath +from lib.db.models.physio_file import DbPhysioFile +from sqlalchemy import ForeignKey +from sqlalchemy.dialects.mysql import INTEGER, VARCHAR +from sqlalchemy.orm import Mapped, mapped_column, relationship + + +class DbMeegqcFile(Base): + """ + A MEEGqc file. + """ + + __tablename__ = 'meegqc_file' + + id: Mapped[int] = mapped_column('ID', INTEGER(unsigned=True), primary_key=True, autoincrement=True) + """ + ID of the MEEGqc file. + """ + + acquisition_file_id: Mapped[int] = mapped_column('AcquisitionFileID', INTEGER(unsigned=True), ForeignKey('physiological_file.PhysiologicalFileID')) + """ + ID of the acquisition file associated with the MEEGqc file. + """ + + path: Mapped[Path] = mapped_column('Path', StringPath, unique=True) + """ + Path of the MEEGqc file relative to the LORIS data directory. + """ + + blake2b_hash: Mapped[str] = mapped_column('Blake2bHash', VARCHAR(255)) + """ + Blake2B hash of the MEEGqc file, which may be used to check that the on-disk file data matches + the file registered in the LORIS database. + """ + + category: Mapped[str] = mapped_column('Category', VARCHAR(255)) + """ + Category of the MEEGqc file, which may be 'calculation', 'reports', or 'summary_reports'. + """ + + acquisition_file: Mapped['DbPhysioFile'] = relationship('DbPhysioFile') + """ + Acquisition file associated with the MEEGqc file. + """ diff --git a/python/loris_meegqc_module/src/loris_meegqc_module/database/queries/meegqc_file.py b/python/loris_meegqc_module/src/loris_meegqc_module/database/queries/meegqc_file.py new file mode 100644 index 000000000..20cec8d45 --- /dev/null +++ b/python/loris_meegqc_module/src/loris_meegqc_module/database/queries/meegqc_file.py @@ -0,0 +1,63 @@ +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.orm import Session as Database + +from loris_meegqc_module.database.models.meegqc_file import DbMeegqcFile + + +def try_get_meegqc_file_with_path(db: Database, path: Path) -> DbMeegqcFile | None: + """ + Get an MEEGqc file from the database using its path, or return `None` if no MEEGqc file was + found. + """ + + return db.execute(select(DbMeegqcFile) + .where(DbMeegqcFile.path == path) + ).scalar_one_or_none() + + +def get_meegqc_files_with_acquisition_file_id(db: Database, acquisition_file_id: int) -> list[DbMeegqcFile]: + """ + Get the MEEGqc files associated with an acquisition file using its ID. + """ + + return list(db.execute(select(DbMeegqcFile) + .where(DbMeegqcFile.acquisition_file_id == acquisition_file_id) + .order_by(DbMeegqcFile.category, DbMeegqcFile.path) + ).scalars()) + + +def get_meegqc_files_with_acquisition_file_id_category( + db: Database, + acquisition_file_id: int, + category: str, +) -> list[DbMeegqcFile]: + """ + Get the MEEGqc files of a given category associated with an acquisition file using its ID. + """ + + return list(db.execute(select(DbMeegqcFile) + .where( + DbMeegqcFile.acquisition_file_id == acquisition_file_id, + DbMeegqcFile.category == category, + ) + .order_by(DbMeegqcFile.category, DbMeegqcFile.path) + ).scalars()) + + +def try_get_meegqc_file_with_id_acquisition_file_id( + db: Database, + meegqc_file_id: int, + acquisition_file_id: int, +) -> DbMeegqcFile | None: + """ + Get an MEEGqc file using its ID and acquisition file ID, or return `None` if no file was found. + """ + + return db.execute(select(DbMeegqcFile) + .where( + DbMeegqcFile.id == meegqc_file_id, + DbMeegqcFile.acquisition_file_id == acquisition_file_id, + ) + ).scalar_one_or_none() diff --git a/python/loris_meegqc_module/src/loris_meegqc_module/importer/main.py b/python/loris_meegqc_module/src/loris_meegqc_module/importer/main.py new file mode 100644 index 000000000..2be21826e --- /dev/null +++ b/python/loris_meegqc_module/src/loris_meegqc_module/importer/main.py @@ -0,0 +1,92 @@ +import re +from pathlib import Path + +from lib.db.models.physio_file import DbPhysioFile +from lib.db.queries.physio_file import try_get_physio_file_with_path +from lib.env import Env +from lib.logging import log, log_error, log_warning +from loris_bids_importer.copy_files import copy_loris_bids_file, get_loris_bids_root_file_path +from loris_bids_importer.importer import BidsImporter +from loris_bids_utils.reader import BidsDatasetReader +from loris_utils.crypto import compute_file_blake2b_hash +from loris_utils.fs import iter_all_dir_files +from loris_utils.iter import find + +from loris_meegqc_module.database.models.meegqc_file import DbMeegqcFile +from loris_meegqc_module.database.queries.meegqc_file import try_get_meegqc_file_with_path + + +def import_meegqc_derivatives(env: Env, importer: BidsImporter, bids_dataset: BidsDatasetReader): + print("Running MEEGQC importer") + + meegqc_path = bids_dataset.path / 'derivatives' / 'Meg_QC' + if not meegqc_path.exists(): + log(env, "No MEEGqc derivatives found in the BIDS dataset. Skipping.") + return + + for category in ['calculation', 'summary_reports', 'reports']: + category_path = meegqc_path / category + if not category_path.exists(): + log_warning(env, f"No MEEGqc files found for category '{category}'.") + continue + + import_meegqc_files(env, importer, category_path, category) + + +def import_meegqc_files(env: Env, importer: BidsImporter, category_path: Path, category: str): + print(f"Importing MEEGqc files for category '{category}'") + + for file_path in iter_all_dir_files(category_path): + try: + import_meegqc_file(env, importer, file_path, category) + except Exception as exception: + log_error(env, f"Error while importing MEEGqc file '{file_path}'. Error message:\n{exception}") + + +def import_meegqc_file(env: Env, importer: BidsImporter, meegqc_file_path: Path, category: str): + log(env, f"Importing MEEGqc {category} file '{meegqc_file_path}'.") + + full_file_path = importer.args.source_bids_path / meegqc_file_path + + acquisition_file = find_acquisition_file(env, importer, full_file_path) + + blake2b_hash = compute_file_blake2b_hash(full_file_path) + + loris_file_path = get_loris_bids_root_file_path(importer, full_file_path) + + current_meegqc_file = try_get_meegqc_file_with_path(env.db, loris_file_path) + if current_meegqc_file is not None: + log(env, f"A MEEGqc file with path {loris_file_path} already exists in the database. Skipping.") + return + + copy_loris_bids_file(importer, meegqc_file_path, loris_file_path) + + env.db.add(DbMeegqcFile( + acquisition_file_id = acquisition_file.id, + path = loris_file_path, + category = category, + blake2b_hash = blake2b_hash, + )) + + env.db.commit() + + +def find_acquisition_file(env: Env, importer: BidsImporter, meegqc_file_path: Path) -> DbPhysioFile: + # TODO: Use a general BIDS file name abstraction. + meegqc_file_pattern = re.sub(r'_run-(\d)', r'_run-0+\1', meegqc_file_path.stem) + meegqc_file_pattern = re.sub(r'_desc-.+_meg', r'(_.*)?_meg', meegqc_file_pattern) + entry = find(importer.files_dict.items(), lambda entry: re.match(meegqc_file_pattern, entry[0].stem) is not None) + if entry is None: + raise Exception(f"TODO 1 cannot match {meegqc_file_path.stem} ({meegqc_file_pattern})") + + file_path = ( + importer.loris_bids_path / entry[1] + if importer.loris_bids_path is not None + else (importer.args.source_bids_path / entry[1]).relative_to(importer.data_dir_path) + ) + + file = try_get_physio_file_with_path(env.db, file_path) + if file is None: + raise Exception(f"TODO 2 {entry}") + + return file diff --git a/python/loris_meegqc_module/src/loris_meegqc_module/install.py b/python/loris_meegqc_module/src/loris_meegqc_module/install.py new file mode 100644 index 000000000..4a65c6096 --- /dev/null +++ b/python/loris_meegqc_module/src/loris_meegqc_module/install.py @@ -0,0 +1,11 @@ +from lib.config_file import load_config +from lib.make_env import make_env + +from loris_meegqc_module.database.models.meegqc_file import DbMeegqcFile + +# TODO: This script is only used for testing purposes and should not be committed to the final PR. +config = load_config(None) +env = make_env('install-loris-meegqc-module', {}, config, False) + +# Create only the User table +DbMeegqcFile.__table__.create(env.db_engine) # type: ignore diff --git a/python/loris_meegqc_module/src/loris_meegqc_module/server/dependencies.py b/python/loris_meegqc_module/src/loris_meegqc_module/server/dependencies.py new file mode 100644 index 000000000..7e28af8d4 --- /dev/null +++ b/python/loris_meegqc_module/src/loris_meegqc_module/server/dependencies.py @@ -0,0 +1,25 @@ +from typing import Annotated + +from fastapi import Depends, HTTPException +from lib.db.models.physio_file import DbPhysioFile +from lib.db.queries.physio_file import try_get_physio_file_with_id +from lib.user import can_user_access_session +from loris_server.dependencies import EnvDep, UserDep + + +def get_physio_file(env: EnvDep, user: UserDep, physio_file_id: int) -> DbPhysioFile: + """ + Get a physiological file or raise an HTTP 404 error if it does not exist or cannot be accessed. + """ + + physio_file = try_get_physio_file_with_id(env.db, physio_file_id) + if physio_file is None: + raise HTTPException(status_code=404, detail="Electrophysiology file not found or not accessible.") + + if not can_user_access_session(env, user, physio_file.session): + raise HTTPException(status_code=404, detail="Electrophysiology file not found or not accessible.") + + return physio_file + + +PhysioFileDep = Annotated[DbPhysioFile, Depends(get_physio_file)] diff --git a/python/loris_meegqc_module/src/loris_meegqc_module/server/endpoints.py b/python/loris_meegqc_module/src/loris_meegqc_module/server/endpoints.py new file mode 100644 index 000000000..74249b70a --- /dev/null +++ b/python/loris_meegqc_module/src/loris_meegqc_module/server/endpoints.py @@ -0,0 +1,108 @@ +from pathlib import Path + +from fastapi import HTTPException +from fastapi.responses import FileResponse +from lib.config import get_data_dir_path_config +from lib.db.models.physio_file import DbPhysioFile +from lib.env import Env +from loris_server.utils import TempZipResponse, guess_mime_type +from pydantic import BaseModel + +from loris_meegqc_module.database.queries.meegqc_file import ( + get_meegqc_files_with_acquisition_file_id, + get_meegqc_files_with_acquisition_file_id_category, + try_get_meegqc_file_with_id_acquisition_file_id, +) + + +class MeegqcFileResponse(BaseModel): + id: int + name: str + category: str + blake2b_hash: str + + +class MeegqcFilesResponse(BaseModel): + files: list[MeegqcFileResponse] + + +def list_meegqc_files(env: Env, acquisition_file: DbPhysioFile) -> MeegqcFilesResponse: + """ + List the MEEGqc files associated with a given acquisition file. + """ + + meegqc_files = get_meegqc_files_with_acquisition_file_id(env.db, acquisition_file.id) + + return MeegqcFilesResponse( + files=[ + MeegqcFileResponse( + id = meegqc_file.id, + name = meegqc_file.path.name, + category = meegqc_file.category, + blake2b_hash = meegqc_file.blake2b_hash, + ) + for meegqc_file in meegqc_files + ], + ) + + +def get_meegqc_file(env: Env, acquisition_file: DbPhysioFile, meegqc_file_id: int) -> FileResponse: + """ + Get an MEEGqc file. + """ + + meegqc_file = try_get_meegqc_file_with_id_acquisition_file_id(env.db, meegqc_file_id, acquisition_file.id) + if meegqc_file is None: + raise HTTPException(status_code=404, detail="MEEGqc file not found or not accessible.") + + data_dir_path = get_data_dir_path_config(env) + + meegqc_file_path = data_dir_path / meegqc_file.path + + media_type = guess_mime_type(meegqc_file.path) + + return FileResponse( + meegqc_file_path, + filename = meegqc_file.path.name, + media_type = media_type, + content_disposition_type = 'inline', + ) + + +def get_meegqc_files_archive( + env: Env, + acquisition_file: DbPhysioFile, + category: str | None, +) -> FileResponse: + """ + Get the MEEGqc files associated with an acquisition file as a downloadable archive. + """ + + if category is None: + meegqc_files = get_meegqc_files_with_acquisition_file_id(env.db, acquisition_file.id) + else: + meegqc_files = get_meegqc_files_with_acquisition_file_id_category(env.db, acquisition_file.id, category) + + if meegqc_files == []: + raise HTTPException(status_code=404, detail="MEEGqc files not found.") + + data_dir_path = get_data_dir_path_config(env) + + meegqc_file_paths = [data_dir_path / meegqc_file.path for meegqc_file in meegqc_files] + + meegqc_archive_name = get_meegqc_archive_name(acquisition_file.path, category) + + return TempZipResponse(meegqc_file_paths, meegqc_archive_name) + + +def get_meegqc_archive_name(acquisition_file_path: Path, category: str | None) -> str: + """ + Get the name of a new MEEGqc file archive. + """ + + archive_name = f'{acquisition_file_path.stem}_meegqc' + + if category is not None: + archive_name += f'_{category}' + + return f'{archive_name}.zip' diff --git a/python/loris_meegqc_module/src/loris_meegqc_module/server/main.py b/python/loris_meegqc_module/src/loris_meegqc_module/server/main.py new file mode 100644 index 000000000..fe945bf5a --- /dev/null +++ b/python/loris_meegqc_module/src/loris_meegqc_module/server/main.py @@ -0,0 +1,32 @@ +from fastapi import APIRouter, FastAPI +from fastapi.responses import FileResponse +from loris_server.dependencies import EnvDep + +from loris_meegqc_module.server.dependencies import PhysioFileDep +from loris_meegqc_module.server.endpoints import ( + MeegqcFilesResponse, + get_meegqc_file, + get_meegqc_files_archive, + list_meegqc_files, +) + +router = APIRouter(prefix='/ephys') + + +@router.get('/{physio_file_id}/meegqc/files', response_model=MeegqcFilesResponse) +def meegqc_files(env: EnvDep, physio_file: PhysioFileDep): + return list_meegqc_files(env, physio_file) + + +@router.get('/{physio_file_id}/meegqc/files/archive', response_class=FileResponse) +def meegqc_files_archive(env: EnvDep, physio_file: PhysioFileDep, category: str | None = None): + return get_meegqc_files_archive(env, physio_file, category) + + +@router.get('/{physio_file_id}/meegqc/files/{meegqc_file_id}', response_class=FileResponse) +def meegqc_file(env: EnvDep, physio_file: PhysioFileDep, meegqc_file_id: int): + return get_meegqc_file(env, physio_file, meegqc_file_id) + + +def load(api: FastAPI): + return api.include_router(router) diff --git a/python/loris_server/README.md b/python/loris_server/README.md new file mode 100644 index 000000000..88d13d719 --- /dev/null +++ b/python/loris_server/README.md @@ -0,0 +1,39 @@ +# LORIS Python server + +## Installation + +This package can be installed with the following command (from the LORIS Python root directory): + +```sh +pip install python/loris_server +``` + +## Deployment + +The LORIS Python server can be deployed as a standard Linux service, this can be done using a service file such as `/etc/systemd/system/loris-server.service`, with a content such as the following: + +```ini +[Unit] +Description=LORIS Python server +After=network.target + +[Service] +User=lorisadmin +Group=lorisadmin +WorkingDirectory=/opt/loris/bin/mri +ExecStart=/bin/bash -c 'source environment && exec run-loris-server' +Restart=always +RestartSec=5 +Environment="PYTHONUNBUFFERED=1" + +[Install] +WantedBy=multi-user.target +``` + +The LORIS Python server can then be used as any Linux service with commands such as the following: +- `systemctl start loris-server` to start the server. +- `systemctl stop loris-server` to stop the server. +- `systemctl restart loris-server` to restart the server. +- `journalctl -u loris-server` to view the server logs. +- `journalctl -u loris-server -f` to view the server logs in real-time. +- `journalctl -u loris-server -p err` to view only the server error logs. diff --git a/python/loris_server/pyproject.toml b/python/loris_server/pyproject.toml new file mode 100644 index 000000000..6328be7fe --- /dev/null +++ b/python/loris_server/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "loris-server" +version = "27.0.0" +description = "LORIS server" +readme = "README.md" +requires-python = ">= 3.11" +dependencies = [ + "fastapi", + "loris-utils", + "uvicorn[standard]", + "pyjwt", +] + +[project.scripts] +run-loris-server = "loris_server.cli.run_loris_server:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/loris_server"] + +[tool.ruff] +extend = "../../pyproject.toml" +src = ["src"] diff --git a/python/loris_server/src/loris_server/api.py b/python/loris_server/src/loris_server/api.py new file mode 100644 index 000000000..f81c24e88 --- /dev/null +++ b/python/loris_server/src/loris_server/api.py @@ -0,0 +1,28 @@ +import os +from importlib.metadata import entry_points + +from fastapi import FastAPI +from lib.config_file import load_config + +from loris_server.endpoints.health import health + +# Get the LORIS configuration values from the environment. +config_file_name = os.environ.get('LORIS_CONFIG_FILE') +dev_mode = os.environ.get('LORIS_DEV_MODE') == 'true' + +# Load the LORIS configuration. +config = load_config(config_file_name) + +# Create the API object. +api = FastAPI(title="LORIS server", debug=dev_mode) + +# Attach the LORIS configuration to the API state. +api.state.config = config + +# Add the health check route to the API. +api.add_api_route('/health', health, methods=['GET']) + +# Load the modules registered into the LORIS server. +for module in entry_points(group='loris-server.loaders'): + print(f"Loading server module '{module.name}'") + module.load()(api) diff --git a/python/loris_server/src/loris_server/cli/run_loris_server.py b/python/loris_server/src/loris_server/cli/run_loris_server.py new file mode 100644 index 000000000..04b6d814b --- /dev/null +++ b/python/loris_server/src/loris_server/cli/run_loris_server.py @@ -0,0 +1,53 @@ +import argparse +import os + +import uvicorn + + +def main(): + parser = argparse.ArgumentParser( + description="Start the LORIS server", + ) + + parser.add_argument( + '--config', + help='Name of the LORIS configuration file') + + parser.add_argument( + '--dev', + action='store_true', + help="Run in development mode with hot reload" + ) + + parser.add_argument( + '--host', + default='127.0.0.1', + help="Host to bind to (default: 127.0.0.1)" + ) + + parser.add_argument( + '--port', + type=int, + default=8000, + help="Port to bind to (default: 8000)" + ) + + args = parser.parse_args() + + if args.config is not None: + os.environ['LORIS_CONFIG_FILE'] = args.config + + if args.dev: + os.environ['LORIS_DEV_MODE'] = 'true' + + uvicorn.run( + 'loris_server.api:api', + host=args.host, + port=args.port, + reload=args.dev, + log_level='debug' if args.dev else 'info' + ) + + +if __name__ == '__main__': + main() diff --git a/python/loris_server/src/loris_server/dependencies.py b/python/loris_server/src/loris_server/dependencies.py new file mode 100644 index 000000000..04b81779b --- /dev/null +++ b/python/loris_server/src/loris_server/dependencies.py @@ -0,0 +1,59 @@ +from typing import Annotated + +import jwt +from fastapi import Depends, HTTPException, Request +from lib.config import get_jwt_secret_key_config +from lib.db.models.user import DbUser +from lib.db.queries.user import try_get_user_with_id +from lib.env import Env +from lib.make_env import make_env + + +def get_server_env(request: Request) -> Env: + """ + Get the LORIS environment. + """ + + config = request.app.state.config + if config is None: + raise RuntimeError("Server configuration not initialized.") + + # Create the LORIS environment object for this request. + return make_env('server', {}, config, False) + + +EnvDep = Annotated[Env, Depends(get_server_env)] + + +def get_user(env: EnvDep, request: Request) -> DbUser: + """ + Get the LORIS user that issued the request. + """ + + bearer = request.headers.get('Authorization') + if bearer is None: + raise HTTPException(status_code=401, detail="Authorization header is missing.") + + token = bearer.removeprefix('Bearer ').strip() + if token == '': + raise HTTPException(status_code=401, detail="Authorization token is missing.") + + secret_key = get_jwt_secret_key_config(env) + + payload = jwt.decode(token, secret_key, algorithms=['HS256']) # type: ignore + user_info = payload.get('user') + if user_info is None: + raise HTTPException(status_code=401, detail="Login information is incorrect.") + + user_id = user_info.get('userID') + if user_id is None: + raise HTTPException(status_code=401, detail="Login information is incorrect.") + + user = try_get_user_with_id(env.db, user_id) + if user is None: + raise HTTPException(status_code=401, detail="Login information is incorrect.") + + return user + + +UserDep = Annotated[DbUser, Depends(get_user)] diff --git a/python/loris_server/src/loris_server/endpoints/health.py b/python/loris_server/src/loris_server/endpoints/health.py new file mode 100644 index 000000000..5cefb7143 --- /dev/null +++ b/python/loris_server/src/loris_server/endpoints/health.py @@ -0,0 +1,5 @@ +from fastapi.responses import PlainTextResponse + + +def health(): + return PlainTextResponse("It works!") diff --git a/python/loris_server/src/loris_server/utils.py b/python/loris_server/src/loris_server/utils.py new file mode 100644 index 000000000..cc95af409 --- /dev/null +++ b/python/loris_server/src/loris_server/utils.py @@ -0,0 +1,51 @@ +from collections.abc import Sequence +from mimetypes import guess_type +from pathlib import Path +from tempfile import NamedTemporaryFile + +from fastapi import HTTPException +from fastapi.responses import FileResponse +from loris_utils.archive import create_zip_archive_with_files +from starlette.background import BackgroundTask + + +def guess_mime_type(path: Path) -> str: + """ + Guess the MIME type of a file based on its path. + """ + + # Describe TSV files as plain text so that the client can directly visualize them in their web + # browser. + if path.suffix == '.tsv': + return 'text/plain' + + media_type, _ = guess_type(path.name) + if media_type is not None: + return media_type + + # Return unknown files as binary files for the client to download, who can then use the + # appropriate application to visualize them. + return 'application/octet-stream' + + +class TempZipResponse(FileResponse): + """ + Build a temporary zip archive for a set of files or directories and return it as a response. + """ + + def __init__(self, paths: Sequence[Path], filename: str): + with NamedTemporaryFile(prefix='loris_archive_', suffix='.zip', delete=False) as temp_archive: + archive_path = Path(temp_archive.name) + + try: + create_zip_archive_with_files(archive_path, paths) + except Exception as exception: + archive_path.unlink() + raise HTTPException(status_code=500, detail="Could not create archive.") from exception + + super().__init__( + archive_path, + filename=filename, + media_type='application/zip', + background=BackgroundTask(lambda: archive_path.unlink()), + ) diff --git a/python/loris_tus_server/README.md b/python/loris_tus_server/README.md new file mode 100644 index 000000000..07548f3e3 --- /dev/null +++ b/python/loris_tus_server/README.md @@ -0,0 +1,9 @@ +# LORIS TUS Server + +This module provides TUS-based resumable file upload functionality for the LORIS server. + +## Features + +- Resumable file uploads using the TUS protocol +- Large file support +- Proof-of-concept implementation diff --git a/python/loris_tus_server/pyproject.toml b/python/loris_tus_server/pyproject.toml new file mode 100644 index 000000000..29135213c --- /dev/null +++ b/python/loris_tus_server/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "loris-tus-server" +version = "27.0.0" +description = "LORIS TUS server for large file uploads" +readme = "README.md" +requires-python = ">= 3.11" +dependencies = [ + "fastapi", + "tuspyserver", + "uvicorn[standard]", +] + +[project.entry-points."loris-server.loaders"] +tus = "loris_tus_server.main:load" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/loris_tus_server"] + +[tool.ruff] +extend = "../../pyproject.toml" +src = ["src"] \ No newline at end of file diff --git a/python/loris_tus_server/src/loris_tus_server/main.py b/python/loris_tus_server/src/loris_tus_server/main.py new file mode 100644 index 000000000..a1a4d0f0e --- /dev/null +++ b/python/loris_tus_server/src/loris_tus_server/main.py @@ -0,0 +1,28 @@ +import os +from pathlib import Path + +from fastapi import APIRouter, FastAPI +from tuspyserver import create_tus_router # type: ignore + + +def get_upload_dir() -> Path: + """Get the upload directory from environment or use default""" + upload_dir = os.environ.get('TUS_UPLOAD_DIR', '/data/bic/tmp/tus') + return Path(upload_dir) + + +def load(api: FastAPI): + """Load the TUS server endpoints into the main LORIS server""" + upload_dir = get_upload_dir() + upload_dir.mkdir(parents=True, exist_ok=True) + + # Initialize TUS router with the upload directory + # tus_router = TusRouter(upload_dir=str(upload_dir)) + + router: APIRouter = create_tus_router( # type: ignore + prefix='/ephys/upload', + files_dir=str(upload_dir), + ) + + # Include TUS routes in the FastAPI app + api.include_router(router) # type: ignore diff --git a/python/loris_utils/src/loris_utils/crypto.py b/python/loris_utils/src/loris_utils/crypto.py index 84c25910c..64d1aff4d 100644 --- a/python/loris_utils/src/loris_utils/crypto.py +++ b/python/loris_utils/src/loris_utils/crypto.py @@ -1,19 +1,55 @@ import hashlib +from hashlib import blake2b from pathlib import Path -def compute_file_blake2b_hash(file_path: Path | str) -> str: +def compute_file_blake2b_hash(file_path: Path) -> str: """ Compute the BLAKE2b hash of a file. """ + hash = blake2b() + update_file_blake2b_hash(Path(file_path), hash) + return hash.hexdigest() + + +def compute_directory_blake2b_hash(dir_path: Path) -> str: + """ + Compute the BLAKE2b hash of a directory. + """ + + hash = blake2b() + update_directory_blake2b_hash(dir_path, hash) + return hash.hexdigest() + + +def update_file_blake2b_hash(file_path: Path, hash: blake2b): + """ + Update a BLAKE2b hash with the contents of a file. + """ + # Since the file given to this function may be large, we read it in chunks to avoid running # out of memory. - hash = hashlib.blake2b() with open(file_path, 'rb') as file: while chunk := file.read(1048576): hash.update(chunk) - return hash.hexdigest() + + +def update_directory_blake2b_hash(dir_path: Path, hash: blake2b): + """ + Update a BLAKE2b hash with the contents of a directory. + """ + + # The paths are sorted to ensure the hash is deterministic regardless of iteration order. + for path in sorted(dir_path.iterdir()): + # The file name is included in the hash to ensure the directory structure is reflected in + # the hash. + hash.update(path.name.encode()) + # Symlinks are currently not included in the hash. + if path.is_file(): + update_file_blake2b_hash(path, hash) + elif path.is_dir(): + update_directory_blake2b_hash(path, hash) def compute_file_md5_hash(file_path: Path | str) -> str: diff --git a/python/tests/integration/scripts/test_import_bids_dataset.py b/python/tests/integration/scripts/test_import_bids_dataset.py index 15fcb9fe7..71cb7fca5 100644 --- a/python/tests/integration/scripts/test_import_bids_dataset.py +++ b/python/tests/integration/scripts/test_import_bids_dataset.py @@ -1,3 +1,4 @@ +from datetime import datetime from pathlib import Path from lib.db.queries.bids_event_dataset_mapping import get_bids_event_dataset_mappings_with_project_id @@ -27,9 +28,7 @@ def test_import_eeg_bids_dataset(): # Check the return code. assert process.returncode == 0 - assert process.stderr == ( - "WARNING: No 'scans.tsv' file found, 'scans.tsv' data will be ignored.\n" - ) + assert process.stderr == "" # Check that the candidate and sessions are present in the database. candidate = try_get_candidate_with_psc_id(db, 'OTT166') @@ -47,6 +46,7 @@ def test_import_eeg_bids_dataset(): assert len(file.channels) == 128 assert len(file.event_files) == 1 assert len(file.task_events) == 3185 + assert file.acquisition_time == datetime(2025, 10, 10, 15, 1, 10) assert file.archive is not None assert file.archive.path == Path('bids_imports/Face13_BIDSVersion_1.1.0/sub-OTT166/ses-V1/eeg/sub-OTT166_ses-V1_task-faceO_eeg.tgz') # noqa: E501 assert file.event_archive is not None @@ -82,6 +82,10 @@ def test_import_eeg_bids_dataset(): 'channel_file_blake2b_hash': '7b91e3650086ef50ecc00f1c50e17e7ad8dc39c484536bbc2423af4be7d2b50a3a0010f840d457fec68fbfb3e136edf4d616a31bab0ca09ed686f555727341dd', # noqa: E501 'event_file_blake2b_hash': '532aa0b52749eb9ee52c2bbb65fa7b1d00d7126cb9a4e10bd4b9dbb4c5527b06e30acdaf17d5806e81d3ce8ad224a9f456e27aba1bf8b92fd43522837c7ffec7', # noqa: E501 'electrophysiology_chunked_dataset_path': 'chunks/Face13_BIDSVersion_1.1.0_chunks/sub-OTT166_ses-V1_task-faceO_eeg.chunks', # noqa: E501 + 'scan_acquisition_time': '2025-10-10 15:01:10.720100+00:00', + 'age_at_scan': 'None', + 'scans_tsv_file': 'bids_imports/Face13_BIDSVersion_1.1.0/sub-OTT166/ses-V1/sub-OTT166_ses-V1_scans.tsv', + 'scans_tsv_file_blake2hash': '4d9b695ba6b35257531b96375ca15c36179b08360f373c46425d958e406132b84ad029113ed4be5e458e762a8af0792ddde5127b5044778c8c8705d8df8f8621', # noqa: E501 } # Check that the event files has been inserted in the database. @@ -97,6 +101,7 @@ def test_import_eeg_bids_dataset(): 'README': None, 'sub-OTT166': { 'ses-V1': { + 'sub-OTT166_ses-V1_scans.tsv': None, 'eeg': { 'sub-OTT166_ses-V1_task-faceO_channels.tsv': None, 'sub-OTT166_ses-V1_task-faceO_eeg.edf': None, diff --git a/upload.py b/upload.py new file mode 100644 index 000000000..24e13bc22 --- /dev/null +++ b/upload.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +""" +TUS Resumable File Upload Script with Progress Display +""" + +import sys +import os +import time +from tusclient import client + +def upload_with_progress(file_path, tus_url): + """Upload file with manual progress tracking using stop_at""" + tus_client = client.TusClient(tus_url) + + # Create uploader + uploader = tus_client.uploader( + file_path, + chunk_size=1024 * 1024 # 1 MB chunks + ) + + file_size = os.path.getsize(file_path) + total_uploaded = 0 + + print(f"File size: {file_size / (1024**3):.2f} GB") + print("Starting upload...") + print("-" * 50) + + start_time = time.time() + + # Upload in segments and track progress + while total_uploaded < file_size: + # Calculate next stopping point (show progress every 10MB or at end) + next_stop = min(total_uploaded + (10 * 1024 * 1024), file_size) + + # Upload up to next_stop bytes + uploader.upload(stop_at=next_stop) + + # Update progress + total_uploaded = next_stop + percent = (total_uploaded / file_size) * 100 + + # Calculate speed + elapsed = time.time() - start_time + if elapsed > 0: + speed_mb = (total_uploaded / (1024 * 1024)) / elapsed + eta_seconds = ((file_size - total_uploaded) / (1024 * 1024)) / speed_mb if speed_mb > 0 else 0 + + # Progress bar + bar_length = 30 + filled = int(bar_length * total_uploaded // file_size) + bar = '█' * filled + '░' * (bar_length - filled) + + print(f"\r{bar} {percent:.1f}% | {total_uploaded/(1024**2):.1f}/{file_size/(1024**2):.1f} MB | " + f"{speed_mb:.1f} MB/s | ETA: {eta_seconds:.0f}s", end='', flush=True) + else: + print(f"\rProgress: {percent:.1f}% ({total_uploaded}/{file_size} bytes)", end='', flush=True) + + print("\n" + "-" * 50) + print("Upload complete!") + +def main(): + if len(sys.argv) != 3: + print("Usage: python tus_upload.py ") + print("Example: python tus_upload.py ./myvideo.mp4 http://localhost:8080/files/") + sys.exit(1) + + file_path = sys.argv[1] + tus_url = sys.argv[2] + + if not os.path.exists(file_path): + print(f"Error: File '{file_path}' not found.") + sys.exit(1) + + if not tus_url.endswith('/'): + tus_url += '/' + + print(f"Uploading: {file_path}") + print(f"TUS Server: {tus_url}") + print("-" * 50) + + try: + upload_with_progress(file_path, tus_url) + print(f"SUCCESS: File uploaded to {tus_url}") + except ImportError: + print("ERROR: 'tuspy' library not installed.") + print("Please install it with: pip install tuspy") + sys.exit(1) + except Exception as e: + print(f"\nERROR: Upload failed - {str(e)}") + sys.exit(1) + +if __name__ == "__main__": + main()